Declare the files to load from an external configuration file

15 files changed, 709 insertions, 232 deletions

diff --git a/.ocamlformat b/.ocamlformat
index 72fc0fd..eb55759 100644
--- a/.ocamlformat
+++ b/.ocamlformat
@@ -1,3 +1,4 @@
+ocaml-version = 5.3

 profile = default

 parens-tuple = always

 sequence-style = terminator

diff --git a/bin/importer.ml b/bin/importer.ml
index 0da2ab7..260d83b 100644
--- a/bin/importer.ml
+++ b/bin/importer.ml
@@ -33,6 +33,7 @@ module Args = struct
 
   let load_conf : string -> ImporterSyntax.t =
    fun file ->
+    let dirname = Filename.dirname file in
     match Filename.extension file with
     | _ -> (
         let (conf : (ImporterSyntax.t, string) result) =
@@ -51,7 +52,16 @@ module Args = struct
 
                 Error error_msg
           in
-          ImportConf.t_of_toml configuration_file
+
+          ImportConf.t_of_toml
+            ~context:
+              {
+                checkFile =
+                  (fun f -> Sys.file_exists (Filename.concat dirname f));
+                loadFile =
+                  (fun f -> Otoml.Parser.from_file (Filename.concat dirname f));
+              }
+            configuration_file
         in
         match conf with
         | Error e ->
@@ -238,14 +248,6 @@ let () =
   let sqlfile = Filename.concat dirname (prefix ^ ".sqlite") in
   let conf = { conf with mapping_date = creation_date sqlfile } in
 
-  (* Ensure that all the files exists *)
-  List.iter process_order ~f:(fun (mapping : Analyse.t) ->
-      let source = Analyse.table mapping in
-      (* First, check *)
-      if not (Sys.file_exists source.Table.file) then begin
-        ignore @@ exists @@ Filename.concat dirname source.Table.file
-      end);
-
   (* The configuration is loaded and valid, we create the errors log file *)
   let log_error = ImportErrors.log ~with_bom:conf.bom prefix dirname in
 
diff --git a/examples/dataset.toml b/examples/dataset.toml
new file mode 100644
index 0000000..dd72cd2
--- /dev/null
+++ b/examples/dataset.toml
@@ -0,0 +1,2 @@
+[files]
+    source = "financial.xlsx"
diff --git a/examples/importer.toml b/examples/importer.toml
index 08e9e25..a8ee199 100644
--- a/examples/importer.toml
+++ b/examples/importer.toml
@@ -1,23 +1,25 @@
+dataset = "dataset.toml"
+
 [source]
-  file = "financial.xlsx"
   name = "source"
+  # The file is looked up in the dataset
 
-[externals.target]
+[externals.source-target]
   intern_key = ":source.A ^ '-suffix'"
   extern_key = ":A ^ '-suffix'"
-  file = "financial.xlsx"
   allow_missing = false
+  # The file is looked up in the dataset
 
 [externals.a_financial]
-  intern_key = ":target.A"
-  extern_key = ":O" # This key is here to generate errors
+  intern_key = ":source-target.A"
+  extern_key = ":O"
   file = "financial.xlsx"
   allow_missing = false
 
 [sheet]
   columns = [
-    ":target.A ^ '\\''",  # Ensure the quote is escaped before sending to the sql engine
-    "join('-', :A, :target.E, :B)",
+    ":source-target.A ^ '\\''",  # Ensure the quote is escaped before sending to the sql engine
+    "join('-', :A, :source-target.E, :B)",
     ":C",
     "counter([:C], [:A])", 
     "sum(:F, [:B, :C, :D], [:B])",
@@ -40,4 +42,3 @@
 
   ]
   sort = []
-  uniq = []
diff --git a/lib/configuration/expression_parser.mly b/lib/configuration/expression_parser.mly
index 1761cce..9b97637 100644
--- a/lib/configuration/expression_parser.mly
+++ b/lib/configuration/expression_parser.mly
@@ -34,20 +34,25 @@ column_expr:
 
 path_: 
     | COLUMN
-      column = IDENT    
-        { ImportExpression.T.Path 
-             ImportDataTypes.Path.{ alias = None
-            ; column = ImportDataTypes.Path.column_of_string column
-            }
-        }
-
-    | COLUMN
-      table = IDENT
-      DOT
-      column = IDENT    
-      { ImportExpression.T.Path
-             ImportDataTypes.Path.{ alias = Some table 
-            ; column = ImportDataTypes.Path.column_of_string column}
+      (* The dot character is required as a separator between the table and the
+         colum, like in [:table.XX] but the table name can also contains [.]
+         (this is allowed in the toml configuration syntax, as long as the
+         identifier is quoted.
+
+         So we have to handle cases like [:foo.bar.XX]
+       *)
+      path = separated_nonempty_list(DOT, IDENT)
+      { let reversed_path = List.rev path in
+        (* reversed_path is nonempty, and we can take head and tail safely *)
+        let tail =  List.tl reversed_path in
+        let alias = match tail with
+        | [] -> None
+        | tl -> Some (String.concat "." (List.rev tl))
+        in
+
+        ImportExpression.T.Path
+             ImportDataTypes.Path.{ alias
+            ; column = ImportDataTypes.Path.column_of_string (List.hd reversed_path)}
       }
 
 column_: 
diff --git a/lib/configuration/importConf.ml b/lib/configuration/importConf.ml
index 2df24bd..4b49686 100644
--- a/lib/configuration/importConf.ml
+++ b/lib/configuration/importConf.ml
@@ -1,14 +1,22 @@
 module TomlReader = Read_conf.Make (Helpers.Toml.Decode)
 
-let t_of_toml : Otoml.t -> (ImporterSyntax.t, string) result =
- fun toml ->
+type loader_context = TomlReader.loader_context = {
+  checkFile : string -> bool;
+  loadFile : string -> Otoml.t;
+}
+
+let t_of_toml :
+    context:loader_context -> Otoml.t -> (ImporterSyntax.t, string) result =
+ fun ~context toml ->
   let version =
     Otoml.find_or ~default:ImporterSyntax.latest_version toml
       (Otoml.get_integer ~strict:false)
       [ "version" ]
   in
   match version with
-  | n when n = ImporterSyntax.latest_version -> TomlReader.read toml
+  | n when n = ImporterSyntax.latest_version -> begin
+      TomlReader.read context toml
+    end
   | _ ->
       Printf.eprintf "Unsuported version : %d\n" version;
       exit 1
diff --git a/lib/configuration/importConf.mli b/lib/configuration/importConf.mli
index d2f65f2..7234499 100644
--- a/lib/configuration/importConf.mli
+++ b/lib/configuration/importConf.mli
@@ -1,4 +1,12 @@
-val t_of_toml : Otoml.t -> (ImporterSyntax.t, string) result
+type loader_context = {
+  checkFile : string -> bool;
+  loadFile : string -> Otoml.t;
+}
+
+val t_of_toml :
+  context:loader_context -> Otoml.t -> (ImporterSyntax.t, string) result
+(** [fileChecker] is called when a file is declared in the configuration. An
+    arror will be raised if the computation return false *)
 
 val expression_from_string :
   string -> (ImportDataTypes.Path.t ImportExpression.T.t, string) result
diff --git a/lib/configuration/read_conf.ml b/lib/configuration/read_conf.ml
index d406b0e..c3c78cc 100644
--- a/lib/configuration/read_conf.ml
+++ b/lib/configuration/read_conf.ml
@@ -178,14 +178,61 @@ module Make (S : Decoders.Decode.S) = struct
   let ( >>= ) = S.( >>= )
   let ( >|= ) = S.( >|= )
 
-  class loader =
+  type loader_context = {
+    checkFile : string -> bool;
+    loadFile : string -> S.value;
+  }
+
+  type dataSet = S.value * (string * string) list
+
+  class loader (context : loader_context) =
     object (self)
+      method path_checker : string S.decoder -> string S.decoder =
+        fun check ->
+          Decoders.Decoder.bind
+            (fun path ->
+              if context.checkFile path then Decoders.Decoder.pure path
+              else Decoders.Decoder.fail "Expected a path to an existing file")
+            check
+      (** Check if a file given in the configuration exists: an error is raised
+          if the function [checkFile] retun false.
+
+          In the unit tests, the effect is mapped to a function returning alway
+          true. *)
+
+      method keep : type a. a S.decoder -> (S.value * a) S.decoder =
+        fun decoder value -> S.map (fun v -> (value, v)) decoder value
+      (** [keep decoder] transform a decoder and keep the initial value with the
+          decoded value.
+
+          This helps to build custom error message, if we want to report an
+          error from a different place. *)
+
+      method load_resources : dataSet option S.decoder =
+        (* Do not accept dash in the keys, as the caracter is used to alias
+           the same file in differents mappings *)
+        let no_dash_decoder =
+          let* s = S.string in
+          match String.exists s ~f:(Char.equal '-') with
+          | true -> S.fail "Expected a key without '-'"
+          | false -> S.succeed s
+        in
+        let list_files_decoders =
+          self#keep (S.key_value_pairs' no_dash_decoder S.string)
+        in
+        let get_field = S.field "files" list_files_decoders in
+
+        let* result = S.map context.loadFile (self#path_checker S.string) in
+        let files = get_field result in
+        let dataSetResult = Result.map (fun v -> Some v) files in
+        S.from_result dataSetResult
+      (** Load an external file containing the list of files to include.*)
+
       method parse_expression : type a.
           ?groups:a ImportExpression.T.t list ->
           eq:(a -> a -> bool) ->
           a ExpressionParser.path_builder ->
-          S.value ->
-          (a ImportExpression.T.t, S.value Decoders.Error.t) result =
+          a ImportExpression.T.t S.decoder =
         fun ?(groups = []) ~eq path ->
           S.string >>= fun v ->
           match ExpressionParser.of_string path v with
@@ -202,41 +249,83 @@ module Make (S : Decoders.Decode.S) = struct
                   S.fail
                     "A group function cannot contains another group function")
 
-      method source =
-        let* file = S.field "file" S.string
-        and* name = S.field "name" S.string
-        and* tab = S.field_opt_or ~default:1 "tab" S.int in
-        S.succeed { Table.file; name; tab }
-
-      method external_ name =
-        let* intern_key =
-          S.field "intern_key"
-            (self#parse_expression ~eq:Path.equal ExpressionParser.path)
-        and* extern_key =
-          S.field "extern_key"
-            (self#parse_expression ~eq:Int.equal ExpressionParser.column)
-        and* file = S.field "file" S.string
-        and* tab = S.field_opt_or ~default:1 "tab" S.int
-        and* allow_missing =
-          S.field_opt_or ~default:false "allow_missing" S.bool
-        in
+      method look_file :
+          dataset:dataSet option -> name:string -> string S.decoder =
+        fun ~dataset ~name ->
+          self#path_checker
+            (S.one_of
+               [
+                 (* If the file is declared, use it *)
+                 ("file", S.field "file" S.string);
+                 ( "dataset",
+                   (* Otherwise search in the data set *)
+
+                   (* We work inside an option monad : 
+                       - Do we have a dataset
+                       - Do we have a valid root name to look for
+                       - De we have a match in the dataset
+                    *)
+                   let ( let* ) = Option.bind in
+                   let element =
+                     let* _, resources = dataset in
+                     let* root =
+                       match String.split_on_char ~sep:'-' name with
+                       | hd :: _ -> Some hd
+                       | _ -> None
+                     in
+                     let* elem = List.assoc_opt root resources in
+                     Some elem
+                   in
+                   match (element, dataset) with
+                   | Some value, _ -> S.succeed value
+                   | None, Some (t, _) ->
+                       let message = "Looking for \"" ^ name ^ "\"" in
+                       S.fail_with (Decoders.Error.make ~context:t message)
+                   | None, None -> S.fail "No dataset declared" );
+               ])
+
+      method source : dataSet option -> Table.t S.decoder =
+        fun dataset ->
+          (* The file shall either be present in the external, or be declared
+           in the dataset *)
+          let* name = S.field "name" S.string in
+          let* file = self#look_file ~dataset ~name
+          and* tab = S.field_opt_or ~default:1 "tab" S.int in
+          S.succeed { Table.file; Table.name; Table.tab }
 
-        S.succeed
-          ImporterSyntax.Extern.
-            {
-              intern_key;
-              extern_key;
-              target = { name; file; tab };
-              allow_missing;
-              match_rule = None;
-            }
+      method external' :
+          dataSet option -> string -> ImporterSyntax.Extern.t S.decoder =
+        fun dataset name ->
+          let* intern_key =
+            S.field "intern_key"
+              (self#parse_expression ~eq:Path.equal ExpressionParser.path)
+          and* extern_key =
+            S.field "extern_key"
+              (self#parse_expression ~eq:Int.equal ExpressionParser.column)
+          and* file = self#look_file ~dataset ~name
+          and* tab = S.field_opt_or ~default:1 "tab" S.int
+          and* allow_missing =
+            S.field_opt_or ~default:false "allow_missing" S.bool
+          in
+
+          S.succeed
+            ImporterSyntax.Extern.
+              {
+                intern_key;
+                extern_key;
+                target = { name; file; tab };
+                allow_missing;
+                match_rule = None;
+              }
+      (** Load the configuration for an external file to link *)
 
       method sheet =
         (* Check the uniq property first, beecause the group functions need
              to include the same expression (at least) *)
         let* uniq =
           S.field_opt_or ~default:[] "uniq"
-          @@ S.list (self#parse_expression ~eq:Path.equal ExpressionParser.path)
+          @@ S.list
+          @@ self#parse_expression ~eq:Path.equal ExpressionParser.path
         in
 
         let* columns =
@@ -247,23 +336,27 @@ module Make (S : Decoders.Decode.S) = struct
         and* filters =
           S.field_opt_or ~default:[] "filters"
           @@ S.list
-               (self#parse_expression ~eq:Path.equal ~groups:uniq
-                  ExpressionParser.path)
+          @@ self#parse_expression ~eq:Path.equal ~groups:uniq
+               ExpressionParser.path
         and* sort =
           S.field_opt_or ~default:[] "sort"
           @@ S.list
-               (self#parse_expression ~eq:Path.equal ~groups:uniq
-                  ExpressionParser.path)
+          @@ self#parse_expression ~eq:Path.equal ~groups:uniq
+               ExpressionParser.path
         in
         S.succeed @@ fun version source externals locale ->
         ImporterSyntax.
           { version; source; externals; columns; filters; sort; uniq; locale }
 
       method conf =
-        let* source = S.field "source" self#source
+        let* dataset : dataSet option =
+          S.field_opt_or ~default:None "dataset" self#load_resources
+        in
+
+        let* source = S.field "source" (self#source dataset)
         and* externals =
           S.field_opt_or ~default:[] "externals"
-            (S.key_value_pairs_seq self#external_)
+          @@ S.key_value_pairs_seq (self#external' dataset)
         and* locale = S.field_opt "locale" S.string in
         let* sheet =
           S.field "sheet" self#sheet >|= fun v -> v 1 source externals locale
@@ -272,15 +365,8 @@ module Make (S : Decoders.Decode.S) = struct
         S.succeed sheet
     end
 
-  let read_file file =
-    S.decode_file (new loader)#conf file
-    |> Result.map_error (fun v ->
-           let formatter = Format.str_formatter in
-           Format.fprintf formatter "%a@." S.pp_error v;
-           Format.flush_str_formatter ())
-
-  let read toml =
-    S.decode_value (new loader)#conf toml
+  let read context toml =
+    S.decode_value (new loader context)#conf toml
     |> Result.map_error (fun v ->
            let formatter = Format.str_formatter in
            Format.fprintf formatter "%a@." S.pp_error v;
diff --git a/lib/helpers/toml.ml b/lib/helpers/toml.ml
index 1b7fb15..5f441dc 100644
--- a/lib/helpers/toml.ml
+++ b/lib/helpers/toml.ml
@@ -1,9 +1,118 @@
+open StdLabels
+
+let rec pp :
+    ?topLevel:bool -> ?path:string list -> Format.formatter -> Otoml.t -> unit =
+ fun ?(topLevel = true) ?(path = []) format -> function
+  | Otoml.TomlString v -> begin
+      match String.contains v '\n' with
+      | false ->
+          Format.pp_print_string format "\"";
+          Format.pp_print_string format v;
+          Format.pp_print_string format "\""
+      | true ->
+          Format.pp_print_string format {|"""|};
+          Format.pp_print_text format v;
+          Format.pp_print_string format {|"""|}
+    end
+  | Otoml.TomlInteger i -> Format.pp_print_int format i
+  | Otoml.TomlFloat f -> Format.pp_print_float format f
+  | Otoml.TomlBoolean b -> Format.pp_print_bool format b
+  | Otoml.TomlArray l -> begin
+      match (topLevel, l) with
+      | _, [] -> Format.pp_print_string format "[]"
+      | false, _ -> Format.pp_print_string format "..."
+      | true, l ->
+          Format.pp_print_string format "[";
+          Format.pp_print_break format 0 4;
+          Format.pp_open_vbox format 0;
+          Format.pp_print_list
+            ~pp_sep:(fun f () ->
+              Format.pp_print_string f ",";
+              Format.pp_print_cut f ())
+            pp format l;
+          Format.pp_close_box format ();
+          Format.pp_print_cut format ();
+          Format.pp_print_string format "]"
+    end
+  | Otoml.TomlTable elements | Otoml.TomlInlineTable elements ->
+      pp_table ~path format elements
+  | Otoml.TomlTableArray t -> Format.pp_print_list pp format t
+  | Otoml.TomlOffsetDateTime _
+  | Otoml.TomlLocalDate _
+  | Otoml.TomlLocalDateTime _
+  | Otoml.TomlLocalTime _ -> ()
+
+and pp_key_values : Format.formatter -> string * Otoml.t -> unit =
+ fun format (name, value) ->
+  Format.fprintf format "%s = %a" name (pp ~topLevel:false ~path:[]) value
+
+and pp_table :
+    ?path:string list -> Format.formatter -> (string * Otoml.t) list -> unit =
+ fun ?(path = []) format elements ->
+  (* Create two lists, one for the subtables, and one for the values. 
+
+    As a table is valid until the next table, we then start to print the
+    preoperties before printintg the subtables inside the element.
+   *)
+  let subtables, properties =
+    List.partition elements ~f:(fun (_, v) ->
+        match v with
+        | Otoml.TomlTable _ -> true
+        | _ -> false)
+  in
+
+  let () =
+    match properties with
+    | [] -> ()
+    | _ ->
+        let isTopLevel =
+          match path with
+          | [] -> true
+          | _ -> false
+        in
+        if not isTopLevel then begin
+          let path = List.rev path in
+          Format.pp_print_cut format ();
+          Format.fprintf format "[%a]"
+            (Format.pp_print_list
+               ~pp_sep:(fun f () -> Format.pp_print_string f ".")
+               Format.pp_print_string)
+            path;
+          Format.pp_print_break format 0 4;
+          Format.pp_open_vbox format 0
+        end;
+        if isTopLevel then begin
+          Format.pp_print_list ~pp_sep:Format.pp_print_cut
+            (fun format v ->
+              match v with
+              | key, Otoml.TomlTable elements ->
+                  pp_table ~path:(key :: path) format elements
+              | other -> pp_key_values format other)
+            format properties
+        end
+        else begin
+          Format.pp_print_string format "..."
+        end;
+
+        if not isTopLevel then begin
+          Format.pp_close_box format ()
+        end;
+        Format.pp_print_cut format ()
+  in
+  (* Then go deeper inside each subtable *)
+  List.iter subtables ~f:(function
+    | name, Otoml.TomlTable v -> pp_table ~path:(name :: path) format v
+    | _ -> (* Because of the partition, this should not happen *) ())
+
 module Decode = struct
   module S = struct
     type value = Otoml.t
 
     let pp : Format.formatter -> value -> unit =
-     fun format t -> Format.pp_print_string format (Otoml.Printer.to_string t)
+     fun formatter v ->
+      Format.pp_open_vbox formatter 0;
+      pp formatter v;
+      Format.pp_close_box formatter ()
 
     let of_string : string -> (value, string) result =
       Otoml.Parser.from_string_result
@@ -22,7 +131,7 @@ module Decode = struct
 
     let get_key_value_pairs : value -> (value * value) list option =
       Otoml.get_opt (fun key ->
-          Otoml.get_table key |> List.map (fun (k, v) -> (Otoml.string k, v)))
+          Otoml.get_table key |> List.map ~f:(fun (k, v) -> (Otoml.string k, v)))
 
     let to_list : value list -> value = Otoml.array
   end
diff --git a/lib/syntax/importerSyntax.ml b/lib/syntax/importerSyntax.ml
index 7788613..cfbba81 100644
--- a/lib/syntax/importerSyntax.ml
+++ b/lib/syntax/importerSyntax.ml
@@ -128,7 +128,7 @@ let dummy_conf =
   {
     source = { file = ""; tab = 0; name = "" };
     version = latest_version;
-    locale = Some "C";
+    locale = None;
     externals = [];
     columns = [];
     filters = [];
diff --git a/readme.rst b/readme.rst
index 0e1c956..5cefd08 100644
--- a/readme.rst
+++ b/readme.rst
@@ -145,16 +145,22 @@ Fichier de configuration
 Les informations générales
 --------------------------
 
-version
+dataset
+  Il s’agit d’un chemin vers un fichier listant tous les fichiers à utiliser.
+  Quand cet clef est définie, l’application ira chercher les fichier aux
+  emplacements définis ici, et il n’est plus nécessaire de définir les clef
+  `file` dans le reste de la configuration.
 
-  Il s’agit de la version de la syntaxe du fichier de configuration. Valeur par
-  défaut : `1`
+  Son utilité prend son sens quand un nouveau jeu de données doit être traité,
+  et plusieurs règles doivent être exécutées : il suffit alors de changer les
+  chemins dans le dataset et uniquement dans ce fichier.
 
 source
   La clef `source` indique quel est le fichier source : pour chaque ligne
   présente dans ce fichier, une ligne sera générée en sortie.
 
-    :file:  le fichier à charger
+    :file:  le fichier à charger. Ce champ peut être ignoré si le dataset est
+            renseigné
     :tab:   optionnellement l’onglet concerné
     :name:  le nom sous lequel le fichier sera associé.
 
@@ -186,7 +192,8 @@ fichier :
 intern_key
     Il s’agit de la colonne servant à faire la liaison dans la source.
 file
-    Le fichier à charger
+    Le fichier à charger. Ce champ peut être ignoré si le dataset est
+    renseigné.
 tab
     optionnellement l’onglet concerné
 extern_key
@@ -208,16 +215,15 @@ afin de construire des chemins sur plusieurs niveaux :
 
 .. code:: toml
 
-  [externals.acheteur_annuaire]
+  [externals.annuaire]
     intern_key = ":I"
-    extern_key = ":A"
     file = "ANNUAIRE.xlsx"
-
-  [externals.acheteur_societe]
-    intern_key = ":acheteur_annuaire.BJ"
     extern_key = ":A"
-    file = "SOCIETES.xlsx"
 
+  [externals.country]
+    intern_key = ":annuaire.BJ"
+    file = "referentials.xlsx"
+    extern_key = ":A"
 
 Les valeurs présentes dans ces colonnes sont pré-traitées pour éviter les
 erreurs générales lors des imports : les espaces en fin de texte sont
diff --git a/tests/confLoader.ml b/tests/confLoader.ml
index 13f9840..e6187c3 100644
--- a/tests/confLoader.ml
+++ b/tests/confLoader.ml
@@ -1,5 +1,23 @@
-let load' : string -> (ImporterSyntax.t, string) Result.t =
- fun content -> Otoml.Parser.from_string content |> ImportConf.t_of_toml
+(** During the test, we don’t care with the file existence *)
+let context =
+  ImportConf.
+    { loadFile = (fun _ -> Otoml.array []); checkFile = (fun _ -> true) }
+
+let load' :
+    ?dataset:(string -> Otoml.t) ->
+    string ->
+    (ImporterSyntax.t, string) Result.t =
+ fun ?(dataset = fun _ -> Otoml.array []) content ->
+  let toml = Otoml.Parser.from_string content in
+  ImportConf.t_of_toml toml ~context:{ context with loadFile = dataset }
+
+let load_from_file :
+    ?dataset:(string -> Otoml.t) ->
+    string ->
+    (ImporterSyntax.t, string) Result.t =
+ fun ?(dataset = fun _ -> Otoml.array []) content ->
+  let toml = Otoml.Parser.from_file content in
+  ImportConf.t_of_toml toml ~context:{ context with loadFile = dataset }
 
 (** Read the configuration in toml and return the internal representation *)
 let load : string -> ImporterSyntax.t =
diff --git a/tests/configuration_expression.ml b/tests/configuration_expression.ml
index cd28589..6478903 100644
--- a/tests/configuration_expression.ml
+++ b/tests/configuration_expression.ml
@@ -4,104 +4,72 @@ open Test_migration
 let result_testable =
   Alcotest.result Test_migration.expression_testable Alcotest.string
 
+(** Helper used to test the equality between the litteral expression and it’s
+    AST *)
+let test : string -> Path.t ImportExpression.T.t -> unit =
+ fun expr result ->
+  let expression = ImportConf.expression_from_string expr in
+  Alcotest.check result_testable "" (Ok result) expression
+
+let path_column =
+  "column as path" >:: fun () ->
+  test ":A" (Path { Path.alias = None; column = 1 })
+
+let path_table =
+  "path with table" >:: fun () ->
+  test ":table.A" (Path { Path.alias = Some "table"; column = 1 })
+
+let path_subtable =
+  "path with table" >:: fun () ->
+  test ":table.Name.A" (Path { Path.alias = Some "table.Name"; column = 1 })
+
 let parse_dquoted =
   "parse_dquoted" >:: fun _ ->
-  let expr = "match(\"\\(..\\)\", :B)" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable ""
-    (Ok
-       (Function
-          ("match", [ Literal "\\(..\\)"; Path { alias = None; column = 2 } ])))
-    result
+  test "match(\"\\(..\\)\", :B)"
+    (Function
+       ("match", [ Literal "\\(..\\)"; Path { alias = None; column = 2 } ]))
 
 let parse_quoted =
   "parse_quoted" >:: fun _ ->
-  let expr = "match('\\(..\\)', :B)" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable ""
-    (Ok
-       (Function
-          ("match", [ Literal "\\(..\\)"; Path { alias = None; column = 2 } ])))
-    result
+  test "match('\\(..\\)', :B)"
+    (Function
+       ("match", [ Literal "\\(..\\)"; Path { alias = None; column = 2 } ]))
 
 let concat =
   "concat" >:: fun _ ->
-  let expr = ":A ^ :B" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable ""
-    (Ok
-       (Concat
-          [
-            Path { alias = None; column = 1 }; Path { alias = None; column = 2 };
-          ]))
-    result
+  test ":A ^ :B"
+    (Concat
+       [ Path { alias = None; column = 1 }; Path { alias = None; column = 2 } ])
 
 let concat2 =
   "concat2" >:: fun _ ->
-  let expr = "'A' ^ '_' ^ 'B'" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable ""
-    (Ok (Concat [ Literal "A"; Literal "_"; Literal "B" ]))
-    result
+  test "'A' ^ '_' ^ 'B'" (Concat [ Literal "A"; Literal "_"; Literal "B" ])
 
 let litteral =
   "litteral" >:: fun _ ->
   (* The text is quoted in shall not be considered as a path *)
-  let expr = "':A'" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable "" (Ok (Literal ":A")) result
+  test "':A'" (Literal ":A")
 
-let empty =
-  "empty" >:: fun _ ->
-  let expr = "" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable "" (Ok Empty) result
+let empty = "empty" >:: fun _ -> test "" Empty
 
 let upper_nvl =
-  "upper_nvl" >:: fun _ ->
-  let expr = "NVL('','')" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable "" (Ok (Nvl [ Empty; Empty ])) result
+  "upper_nvl" >:: fun _ -> test "NVL('','')" (Nvl [ Empty; Empty ])
 
 let lower_nvl =
-  "lower_nvl" >:: fun _ ->
-  let expr = "nvl('','')" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable "" (Ok (Nvl [ Empty; Empty ])) result
-
-let numeric =
-  "numeric" >:: fun _ ->
-  let expr = "123" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable "" (Ok (Integer "123")) result
+  "lower_nvl" >:: fun _ -> test "nvl('','')" (Nvl [ Empty; Empty ])
 
-let numeric_neg =
-  "numeric_neg" >:: fun _ ->
-  let expr = "-123" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable "" (Ok (Integer "-123")) result
+let numeric = "numeric" >:: fun _ -> test "123" (Integer "123")
+let numeric_neg = "numeric_neg" >:: fun _ -> test "-123" (Integer "-123")
 
 let op_priority =
   "operator_priority" >:: fun _ ->
-  let expr = "1 + 2 > 2" in
-  let result = ImportConf.expression_from_string expr
-  and expected =
-    ImportExpression.T.(
-      BOperator (GT, BOperator (Add, Integer "1", Integer "2"), Integer "2"))
-  in
-
-  Alcotest.check result_testable "" (Ok expected) result
+  test "1 + 2 > 2"
+    (BOperator (GT, BOperator (Add, Integer "1", Integer "2"), Integer "2"))
 
 let op_priority2 =
   "operator_priority" >:: fun _ ->
-  let expr = "1 ^ 2 = 2" in
-  let result = ImportConf.expression_from_string expr
-  and expected =
-    ImportExpression.T.(
-      BOperator (Equal, Concat [ Integer "1"; Integer "2" ], Integer "2"))
-  in
-
-  Alcotest.check result_testable "" (Ok expected) result
+  test "1 ^ 2 = 2"
+    (BOperator (Equal, Concat [ Integer "1"; Integer "2" ], Integer "2"))
 
 let join =
   "join" >:: fun _ ->
@@ -119,29 +87,15 @@ let join =
 
 let join_empty =
   "join" >:: fun _ ->
-  let expr = "join('', :A, :B)" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable ""
-    (Ok
-       (Join
-          ( "",
-            [
-              Path { alias = None; column = 1 };
-              Path { alias = None; column = 2 };
-            ] )))
-    result
+  test "join('', :A, :B)"
+    (Join
+       ( "",
+         [
+           Path { alias = None; column = 1 }; Path { alias = None; column = 2 };
+         ] ))
 
-let upper =
-  "upper" >:: fun _ ->
-  let expr = "upper('')" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable "" (Ok (Function' (Upper, [ Empty ]))) result
-
-let trim =
-  "trim" >:: fun _ ->
-  let expr = "trim('')" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable "" (Ok (Function' (Trim, [ Empty ]))) result
+let upper = "upper" >:: fun _ -> test "upper('')" (Function' (Upper, [ Empty ]))
+let trim = "trim" >:: fun _ -> test "trim('')" (Function' (Trim, [ Empty ]))
 
 (** Extract the columns from a window function *)
 let fold_values =
@@ -182,44 +136,24 @@ let bad_quote =
 
 let nested_expression =
   "nested_expression" >:: fun _ ->
-  let expr = "1 = (1 = 0)" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable ""
-    (Ok
-       (BOperator
-          ( Equal,
-            Integer "1",
-            Expr (BOperator (Equal, Integer "1", Integer "0")) )))
-    result
+  test "1 = (1 = 0)"
+    (BOperator
+       (Equal, Integer "1", Expr (BOperator (Equal, Integer "1", Integer "0"))))
 
 let priority_equality =
   "priority_equality" >:: fun _ ->
-  let expr = "1 = 1 = 0" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable ""
-    (Ok
-       (BOperator
-          (Equal, Integer "1", BOperator (Equal, Integer "1", Integer "0"))))
-    result
+  test "1 = 1 = 0"
+    (BOperator (Equal, Integer "1", BOperator (Equal, Integer "1", Integer "0")))
 
 let priority_operator_and =
   "priority_equality" >:: fun _ ->
-  let expr = "1 and 1 = 0" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable ""
-    (Ok
-       (BOperator (And, Integer "1", BOperator (Equal, Integer "1", Integer "0"))))
-    result
+  test "1 and 1 = 0"
+    (BOperator (And, Integer "1", BOperator (Equal, Integer "1", Integer "0")))
 
 let priority_operator_or =
   "priority_equality" >:: fun _ ->
-  let expr = "1 <> 1 or 0" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable ""
-    (Ok
-       (BOperator
-          (Or, BOperator (Different, Integer "1", Integer "1"), Integer "0")))
-    result
+  test "1 <> 1 or 0"
+    (BOperator (Or, BOperator (Different, Integer "1", Integer "1"), Integer "0"))
 
 let unknown_function =
   "unknown function" >:: fun _ ->
@@ -240,6 +174,9 @@ let wrong_arguments =
 
 let test_suit =
   [
+    path_column;
+    path_table;
+    path_subtable;
     parse_dquoted;
     parse_quoted;
     concat;
diff --git a/tests/configuration_toml.ml b/tests/configuration_toml.ml
index 0a36faf..470af4a 100644
--- a/tests/configuration_toml.ml
+++ b/tests/configuration_toml.ml
@@ -5,12 +5,12 @@ open Test_migration
 let nested_group () =
   let expected =
     Error
-      "in field \"sheet\":\n\
-      \  in field \"columns\":\n\
-      \    while decoding a list:\n\
-      \      element 0:\n\
-      \        A group function cannot contains another group function, but got\n\
-      \        \"max(:A, [counter([:A], [:A])], [])\" \n"
+      {|in field "sheet":
+  in field "columns":
+    while decoding a list:
+      element 0:
+        A group function cannot contains another group function, but got
+        "max(:A, [counter([:A], [:A])], [])"|}
   and result =
     ConfLoader.load'
       {|[source]
@@ -22,14 +22,291 @@ columns = [
     "max(:A, [counter([:A], [:A])], [])", 
 ]|}
   in
-  Alcotest.(check (result Test_migration.syntax string))
+  Alcotest.(check (result Test_migration.syntax Test_migration.trimed_string))
     "duplicate" expected result
 
+(** Load a simple configuration *)
+let load_configuration () =
+  let configuration =
+    ConfLoader.load'
+      {|[source]
+name = ""
+file = ""
+tab = 0
+
+[sheet]
+columns = []|}
+  and expected = Ok ImporterSyntax.dummy_conf in
+  Alcotest.(check (result Test_migration.syntax string))
+    "Simple configuration" expected configuration
+
+let externals () =
+  let configuration =
+    ConfLoader.load'
+      {|[source]
+name = ""
+file = ""
+tab = 0
+
+[externals.other]
+  intern_key = ":A"
+  file = "other.xlsx"
+  extern_key = ":C"
+  allow_missing = false
+
+[sheet]
+columns = []|}
+  and expected =
+    Ok
+      {
+        ImporterSyntax.dummy_conf with
+        externals = [ ConfLoader.external_other ];
+      }
+  in
+  Alcotest.(check (result Test_migration.syntax Test_migration.trimed_string))
+    "Simple external" expected configuration
+
+(** There is an error in this configuration the key [intern_key] is missing in
+    the external *)
+let external_with_missing_key () =
+  let configuration =
+    ConfLoader.load'
+      {|[source]
+name = ""
+file = ""
+
+[externals.other]
+  file = ""
+  extern_key = ""
+
+[sheet]
+columns = []|}
+  and expected =
+    Error
+      {|in field "externals":
+  Failed while decoding key-value pairs:
+    Expected an object with an attribute "intern_key", but got
+    file = ""
+    extern_key = ""|}
+  in
+  Alcotest.(check (result Test_migration.syntax Test_migration.trimed_string))
+    "Missing key" expected configuration
+
+let sub_external () =
+  let configuration =
+    ConfLoader.load'
+      {|[source]
+name = ""
+file = ""
+tab = 0
+
+
+[externals.other-1]
+  intern_key = ":A"
+  file = "other.xlsx"
+  extern_key = ":C"
+  allow_missing = false
+
+[sheet]
+columns = []|}
+  and expected =
+    Ok
+      {
+        ImporterSyntax.dummy_conf with
+        externals =
+          ConfLoader.
+            [
+              {
+                external_other with
+                target = { external_table_other with name = "other-1" };
+              };
+            ];
+      }
+  in
+  Alcotest.(check (result Test_migration.syntax string))
+    "external with path" expected configuration
+
+let sub_external_with_missing_key () =
+  let configuration =
+    ConfLoader.load'
+      {|[source]
+name = ""
+file = ""
+
+[externals.other-1]
+  file = ""
+  extern_key = ""
+
+[sheet]
+columns = []|}
+  and expected =
+    Error
+      {|in field "externals":
+  Failed while decoding key-value pairs:
+    Expected an object with an attribute "intern_key", but got
+    file = ""
+    extern_key = ""|}
+  in
+  Alcotest.(check (result Test_migration.syntax Test_migration.trimed_string))
+    "Missing intern_key" expected configuration
+
+(** The same configuration has external, and sub-element external *)
+let sub_external_mixed () =
+  let configuration =
+    ConfLoader.load'
+      {|[source]
+name = ""
+file = ""
+tab = 0
+
+[externals.other]
+  intern_key = ":A"
+  file = "other.xlsx"
+  extern_key = ":C"
+  allow_missing = false
+
+[externals.other-1]
+  intern_key = ":A"
+  file = "other.xlsx"
+  extern_key = ":C"
+  allow_missing = false
+
+[sheet]
+columns = []|}
+  and expected =
+    Ok
+      {
+        ImporterSyntax.dummy_conf with
+        externals =
+          ConfLoader.
+            [
+              external_other;
+              {
+                external_other with
+                target = { external_table_other with name = "other-1" };
+              };
+            ];
+      }
+  in
+  Alcotest.(check (result Test_migration.syntax string))
+    "external with path" expected configuration
+
+let missing_dataset () =
+  let configuration =
+    ConfLoader.load' {|[source]
+name = ""
+tab = 0
+
+[sheet]
+columns = []|}
+  and expected =
+    Error
+      {|in field "source":
+  I tried the following decoders but they all failed:
+    "file" decoder:
+      Expected an object with an attribute "file", but got name = ""
+                                                           tab = 0
+                                                            
+    "dataset" decoder: No dataset declared, but got name = ""
+                                                    tab = 0|}
+  in
+  Alcotest.(check (result Test_migration.syntax Test_migration.trimed_string))
+    "No dataset provided" expected configuration
+
+let empty_dataset () =
+  let configuration =
+    ConfLoader.load'
+      ~dataset:(fun _ -> Otoml.TomlArray [])
+      {|
+
+dataset = "…"
+
+[source]
+name = ""
+
+[sheet]
+columns = []|}
+  and expected =
+    Error
+      {|in field "dataset": Expected an object with an attribute "files", but got []|}
+  in
+  Alcotest.(check (result Test_migration.syntax Test_migration.trimed_string))
+    "Invalid Dataset" expected configuration
+
+let dataset_with_invalid_key () =
+  let configuration =
+    ConfLoader.load'
+      ~dataset:(fun _ ->
+        Otoml.(
+          TomlTable
+            [ ("files", TomlTable [ ("other-1", TomlString "other.xlsx") ]) ]))
+      {|
+
+dataset = "…"
+
+[source]
+name = ""
+
+[sheet]
+columns = []|}
+  and expected =
+    Error
+      {|in field "dataset":
+  in field "files":
+    Failed while decoding key-value pairs:
+      Expected a key without '-', but got "other-1"|}
+  in
+  Alcotest.(check (result Test_migration.syntax Test_migration.trimed_string))
+    "Invalid Dataset: invalid key" expected configuration
+
+let external_dataset () =
+  let configuration =
+    ConfLoader.load'
+      ~dataset:(fun _ ->
+        Otoml.(
+          TomlTable
+            [ ("files", TomlTable [ ("other", TomlString "other.xlsx") ]) ]))
+      {|
+
+dataset = "…"
+
+[source]
+name = ""
+file = ""
+tab = 0
+
+
+[externals.other-1]
+  # The file is not defined here
+  # And in the dataset, there is no "other-1", just "other": the application
+  # should be able to infer the information from "other" and apply it here.
+  intern_key = ":A"
+  extern_key = ":C"
+  allow_missing = false
+
+[sheet]
+columns = []|}
+  and expected =
+    Ok
+      {
+        ImporterSyntax.dummy_conf with
+        externals =
+          ConfLoader.
+            [
+              {
+                external_other with
+                target = { external_table_other with name = "other-1" };
+              };
+            ];
+      }
+  in
+  Alcotest.(check (result Test_migration.syntax string))
+    "Dataset with alias" expected configuration
+
 let test_suit =
   [
     ( "parse_extern" >:: fun _ ->
-      let toml = Otoml.Parser.from_file "configuration/simple.toml" in
-      let toml = ImportConf.t_of_toml toml in
+      let toml = ConfLoader.load_from_file "configuration/simple.toml" in
       match toml with
       | Error s -> raise (Failure s)
       | Ok result ->
@@ -55,8 +332,7 @@ let test_suit =
                (Alcotest.list Test_migration.extern_testable)
                "" [ expected ] result.externals );
     ( "parse_columns" >:: fun _ ->
-      let toml = Otoml.Parser.from_file "configuration/simple.toml" in
-      let toml = ImportConf.t_of_toml toml in
+      let toml = ConfLoader.load_from_file "configuration/simple.toml" in
 
       match toml with
       | Error s -> raise (Failure s)
@@ -83,10 +359,19 @@ let test_suit =
             (Alcotest.list Test_migration.expression_testable)
             "" expected result.columns );
     ( "parse_csv" >:: fun _ ->
-      let toml = Otoml.Parser.from_file "configuration/example_csv.toml" in
-      let toml = ImportConf.t_of_toml toml in
+      let toml = ConfLoader.load_from_file "configuration/example_csv.toml" in
       ignore toml );
     ("nested group", `Quick, nested_group);
+    ("Basic configuration", `Quick, load_configuration);
+    ("Configuration with external", `Quick, externals);
+    ("Faulty configuration", `Quick, external_with_missing_key);
+    ("Sub external", `Quick, sub_external);
+    ("Faulty configuration", `Quick, sub_external_with_missing_key);
+    ("Mix in external and sub external", `Quick, sub_external_mixed);
+    ("Missing dataset", `Quick, missing_dataset);
+    ("Empty dataset", `Quick, empty_dataset);
+    ("Dataset with invalid key", `Quick, dataset_with_invalid_key);
+    ("External dataset", `Quick, external_dataset);
   ]
 
 let tests = "configuration_toml" >::: test_suit
diff --git a/tests/test_migration.ml b/tests/test_migration.ml
index 17e48cc..acf782d 100644
--- a/tests/test_migration.ml
+++ b/tests/test_migration.ml
@@ -42,6 +42,15 @@ let extern_testable = make_test (module ImporterSyntax.Extern)
 let table_testable = make_test (module ImportDataTypes.Table)
 let int_container_testable = make_test (module ImportContainers.IntSet)
 
+let trimed_string =
+  make_test
+    (module struct
+      type t = string
+
+      let equal s1 s2 = String.equal (String.trim s1) (String.trim s2)
+      let pp format t = Format.fprintf format "%s" (String.trim t)
+    end)
+
 let expression_testable =
   make_test
     (module struct