Declare the files to load from an external configuration file

4 files changed, 386 insertions, 137 deletions

diff --git a/tests/confLoader.ml b/tests/confLoader.ml
index 13f9840..e6187c3 100644
--- a/tests/confLoader.ml
+++ b/tests/confLoader.ml
@@ -1,5 +1,23 @@
-let load' : string -> (ImporterSyntax.t, string) Result.t =
- fun content -> Otoml.Parser.from_string content |> ImportConf.t_of_toml
+(** During the test, we don’t care with the file existence *)
+let context =
+  ImportConf.
+    { loadFile = (fun _ -> Otoml.array []); checkFile = (fun _ -> true) }
+
+let load' :
+    ?dataset:(string -> Otoml.t) ->
+    string ->
+    (ImporterSyntax.t, string) Result.t =
+ fun ?(dataset = fun _ -> Otoml.array []) content ->
+  let toml = Otoml.Parser.from_string content in
+  ImportConf.t_of_toml toml ~context:{ context with loadFile = dataset }
+
+let load_from_file :
+    ?dataset:(string -> Otoml.t) ->
+    string ->
+    (ImporterSyntax.t, string) Result.t =
+ fun ?(dataset = fun _ -> Otoml.array []) content ->
+  let toml = Otoml.Parser.from_file content in
+  ImportConf.t_of_toml toml ~context:{ context with loadFile = dataset }
 
 (** Read the configuration in toml and return the internal representation *)
 let load : string -> ImporterSyntax.t =
diff --git a/tests/configuration_expression.ml b/tests/configuration_expression.ml
index cd28589..6478903 100644
--- a/tests/configuration_expression.ml
+++ b/tests/configuration_expression.ml
@@ -4,104 +4,72 @@ open Test_migration
 let result_testable =
   Alcotest.result Test_migration.expression_testable Alcotest.string
 
+(** Helper used to test the equality between the litteral expression and it’s
+    AST *)
+let test : string -> Path.t ImportExpression.T.t -> unit =
+ fun expr result ->
+  let expression = ImportConf.expression_from_string expr in
+  Alcotest.check result_testable "" (Ok result) expression
+
+let path_column =
+  "column as path" >:: fun () ->
+  test ":A" (Path { Path.alias = None; column = 1 })
+
+let path_table =
+  "path with table" >:: fun () ->
+  test ":table.A" (Path { Path.alias = Some "table"; column = 1 })
+
+let path_subtable =
+  "path with table" >:: fun () ->
+  test ":table.Name.A" (Path { Path.alias = Some "table.Name"; column = 1 })
+
 let parse_dquoted =
   "parse_dquoted" >:: fun _ ->
-  let expr = "match(\"\\(..\\)\", :B)" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable ""
-    (Ok
-       (Function
-          ("match", [ Literal "\\(..\\)"; Path { alias = None; column = 2 } ])))
-    result
+  test "match(\"\\(..\\)\", :B)"
+    (Function
+       ("match", [ Literal "\\(..\\)"; Path { alias = None; column = 2 } ]))
 
 let parse_quoted =
   "parse_quoted" >:: fun _ ->
-  let expr = "match('\\(..\\)', :B)" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable ""
-    (Ok
-       (Function
-          ("match", [ Literal "\\(..\\)"; Path { alias = None; column = 2 } ])))
-    result
+  test "match('\\(..\\)', :B)"
+    (Function
+       ("match", [ Literal "\\(..\\)"; Path { alias = None; column = 2 } ]))
 
 let concat =
   "concat" >:: fun _ ->
-  let expr = ":A ^ :B" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable ""
-    (Ok
-       (Concat
-          [
-            Path { alias = None; column = 1 }; Path { alias = None; column = 2 };
-          ]))
-    result
+  test ":A ^ :B"
+    (Concat
+       [ Path { alias = None; column = 1 }; Path { alias = None; column = 2 } ])
 
 let concat2 =
   "concat2" >:: fun _ ->
-  let expr = "'A' ^ '_' ^ 'B'" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable ""
-    (Ok (Concat [ Literal "A"; Literal "_"; Literal "B" ]))
-    result
+  test "'A' ^ '_' ^ 'B'" (Concat [ Literal "A"; Literal "_"; Literal "B" ])
 
 let litteral =
   "litteral" >:: fun _ ->
   (* The text is quoted in shall not be considered as a path *)
-  let expr = "':A'" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable "" (Ok (Literal ":A")) result
+  test "':A'" (Literal ":A")
 
-let empty =
-  "empty" >:: fun _ ->
-  let expr = "" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable "" (Ok Empty) result
+let empty = "empty" >:: fun _ -> test "" Empty
 
 let upper_nvl =
-  "upper_nvl" >:: fun _ ->
-  let expr = "NVL('','')" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable "" (Ok (Nvl [ Empty; Empty ])) result
+  "upper_nvl" >:: fun _ -> test "NVL('','')" (Nvl [ Empty; Empty ])
 
 let lower_nvl =
-  "lower_nvl" >:: fun _ ->
-  let expr = "nvl('','')" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable "" (Ok (Nvl [ Empty; Empty ])) result
-
-let numeric =
-  "numeric" >:: fun _ ->
-  let expr = "123" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable "" (Ok (Integer "123")) result
+  "lower_nvl" >:: fun _ -> test "nvl('','')" (Nvl [ Empty; Empty ])
 
-let numeric_neg =
-  "numeric_neg" >:: fun _ ->
-  let expr = "-123" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable "" (Ok (Integer "-123")) result
+let numeric = "numeric" >:: fun _ -> test "123" (Integer "123")
+let numeric_neg = "numeric_neg" >:: fun _ -> test "-123" (Integer "-123")
 
 let op_priority =
   "operator_priority" >:: fun _ ->
-  let expr = "1 + 2 > 2" in
-  let result = ImportConf.expression_from_string expr
-  and expected =
-    ImportExpression.T.(
-      BOperator (GT, BOperator (Add, Integer "1", Integer "2"), Integer "2"))
-  in
-
-  Alcotest.check result_testable "" (Ok expected) result
+  test "1 + 2 > 2"
+    (BOperator (GT, BOperator (Add, Integer "1", Integer "2"), Integer "2"))
 
 let op_priority2 =
   "operator_priority" >:: fun _ ->
-  let expr = "1 ^ 2 = 2" in
-  let result = ImportConf.expression_from_string expr
-  and expected =
-    ImportExpression.T.(
-      BOperator (Equal, Concat [ Integer "1"; Integer "2" ], Integer "2"))
-  in
-
-  Alcotest.check result_testable "" (Ok expected) result
+  test "1 ^ 2 = 2"
+    (BOperator (Equal, Concat [ Integer "1"; Integer "2" ], Integer "2"))
 
 let join =
   "join" >:: fun _ ->
@@ -119,29 +87,15 @@ let join =
 
 let join_empty =
   "join" >:: fun _ ->
-  let expr = "join('', :A, :B)" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable ""
-    (Ok
-       (Join
-          ( "",
-            [
-              Path { alias = None; column = 1 };
-              Path { alias = None; column = 2 };
-            ] )))
-    result
+  test "join('', :A, :B)"
+    (Join
+       ( "",
+         [
+           Path { alias = None; column = 1 }; Path { alias = None; column = 2 };
+         ] ))
 
-let upper =
-  "upper" >:: fun _ ->
-  let expr = "upper('')" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable "" (Ok (Function' (Upper, [ Empty ]))) result
-
-let trim =
-  "trim" >:: fun _ ->
-  let expr = "trim('')" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable "" (Ok (Function' (Trim, [ Empty ]))) result
+let upper = "upper" >:: fun _ -> test "upper('')" (Function' (Upper, [ Empty ]))
+let trim = "trim" >:: fun _ -> test "trim('')" (Function' (Trim, [ Empty ]))
 
 (** Extract the columns from a window function *)
 let fold_values =
@@ -182,44 +136,24 @@ let bad_quote =
 
 let nested_expression =
   "nested_expression" >:: fun _ ->
-  let expr = "1 = (1 = 0)" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable ""
-    (Ok
-       (BOperator
-          ( Equal,
-            Integer "1",
-            Expr (BOperator (Equal, Integer "1", Integer "0")) )))
-    result
+  test "1 = (1 = 0)"
+    (BOperator
+       (Equal, Integer "1", Expr (BOperator (Equal, Integer "1", Integer "0"))))
 
 let priority_equality =
   "priority_equality" >:: fun _ ->
-  let expr = "1 = 1 = 0" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable ""
-    (Ok
-       (BOperator
-          (Equal, Integer "1", BOperator (Equal, Integer "1", Integer "0"))))
-    result
+  test "1 = 1 = 0"
+    (BOperator (Equal, Integer "1", BOperator (Equal, Integer "1", Integer "0")))
 
 let priority_operator_and =
   "priority_equality" >:: fun _ ->
-  let expr = "1 and 1 = 0" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable ""
-    (Ok
-       (BOperator (And, Integer "1", BOperator (Equal, Integer "1", Integer "0"))))
-    result
+  test "1 and 1 = 0"
+    (BOperator (And, Integer "1", BOperator (Equal, Integer "1", Integer "0")))
 
 let priority_operator_or =
   "priority_equality" >:: fun _ ->
-  let expr = "1 <> 1 or 0" in
-  let result = ImportConf.expression_from_string expr in
-  Alcotest.check result_testable ""
-    (Ok
-       (BOperator
-          (Or, BOperator (Different, Integer "1", Integer "1"), Integer "0")))
-    result
+  test "1 <> 1 or 0"
+    (BOperator (Or, BOperator (Different, Integer "1", Integer "1"), Integer "0"))
 
 let unknown_function =
   "unknown function" >:: fun _ ->
@@ -240,6 +174,9 @@ let wrong_arguments =
 
 let test_suit =
   [
+    path_column;
+    path_table;
+    path_subtable;
     parse_dquoted;
     parse_quoted;
     concat;
diff --git a/tests/configuration_toml.ml b/tests/configuration_toml.ml
index 0a36faf..470af4a 100644
--- a/tests/configuration_toml.ml
+++ b/tests/configuration_toml.ml
@@ -5,12 +5,12 @@ open Test_migration
 let nested_group () =
   let expected =
     Error
-      "in field \"sheet\":\n\
-      \  in field \"columns\":\n\
-      \    while decoding a list:\n\
-      \      element 0:\n\
-      \        A group function cannot contains another group function, but got\n\
-      \        \"max(:A, [counter([:A], [:A])], [])\" \n"
+      {|in field "sheet":
+  in field "columns":
+    while decoding a list:
+      element 0:
+        A group function cannot contains another group function, but got
+        "max(:A, [counter([:A], [:A])], [])"|}
   and result =
     ConfLoader.load'
       {|[source]
@@ -22,14 +22,291 @@ columns = [
     "max(:A, [counter([:A], [:A])], [])", 
 ]|}
   in
-  Alcotest.(check (result Test_migration.syntax string))
+  Alcotest.(check (result Test_migration.syntax Test_migration.trimed_string))
     "duplicate" expected result
 
+(** Load a simple configuration *)
+let load_configuration () =
+  let configuration =
+    ConfLoader.load'
+      {|[source]
+name = ""
+file = ""
+tab = 0
+
+[sheet]
+columns = []|}
+  and expected = Ok ImporterSyntax.dummy_conf in
+  Alcotest.(check (result Test_migration.syntax string))
+    "Simple configuration" expected configuration
+
+let externals () =
+  let configuration =
+    ConfLoader.load'
+      {|[source]
+name = ""
+file = ""
+tab = 0
+
+[externals.other]
+  intern_key = ":A"
+  file = "other.xlsx"
+  extern_key = ":C"
+  allow_missing = false
+
+[sheet]
+columns = []|}
+  and expected =
+    Ok
+      {
+        ImporterSyntax.dummy_conf with
+        externals = [ ConfLoader.external_other ];
+      }
+  in
+  Alcotest.(check (result Test_migration.syntax Test_migration.trimed_string))
+    "Simple external" expected configuration
+
+(** There is an error in this configuration the key [intern_key] is missing in
+    the external *)
+let external_with_missing_key () =
+  let configuration =
+    ConfLoader.load'
+      {|[source]
+name = ""
+file = ""
+
+[externals.other]
+  file = ""
+  extern_key = ""
+
+[sheet]
+columns = []|}
+  and expected =
+    Error
+      {|in field "externals":
+  Failed while decoding key-value pairs:
+    Expected an object with an attribute "intern_key", but got
+    file = ""
+    extern_key = ""|}
+  in
+  Alcotest.(check (result Test_migration.syntax Test_migration.trimed_string))
+    "Missing key" expected configuration
+
+let sub_external () =
+  let configuration =
+    ConfLoader.load'
+      {|[source]
+name = ""
+file = ""
+tab = 0
+
+
+[externals.other-1]
+  intern_key = ":A"
+  file = "other.xlsx"
+  extern_key = ":C"
+  allow_missing = false
+
+[sheet]
+columns = []|}
+  and expected =
+    Ok
+      {
+        ImporterSyntax.dummy_conf with
+        externals =
+          ConfLoader.
+            [
+              {
+                external_other with
+                target = { external_table_other with name = "other-1" };
+              };
+            ];
+      }
+  in
+  Alcotest.(check (result Test_migration.syntax string))
+    "external with path" expected configuration
+
+let sub_external_with_missing_key () =
+  let configuration =
+    ConfLoader.load'
+      {|[source]
+name = ""
+file = ""
+
+[externals.other-1]
+  file = ""
+  extern_key = ""
+
+[sheet]
+columns = []|}
+  and expected =
+    Error
+      {|in field "externals":
+  Failed while decoding key-value pairs:
+    Expected an object with an attribute "intern_key", but got
+    file = ""
+    extern_key = ""|}
+  in
+  Alcotest.(check (result Test_migration.syntax Test_migration.trimed_string))
+    "Missing intern_key" expected configuration
+
+(** The same configuration has external, and sub-element external *)
+let sub_external_mixed () =
+  let configuration =
+    ConfLoader.load'
+      {|[source]
+name = ""
+file = ""
+tab = 0
+
+[externals.other]
+  intern_key = ":A"
+  file = "other.xlsx"
+  extern_key = ":C"
+  allow_missing = false
+
+[externals.other-1]
+  intern_key = ":A"
+  file = "other.xlsx"
+  extern_key = ":C"
+  allow_missing = false
+
+[sheet]
+columns = []|}
+  and expected =
+    Ok
+      {
+        ImporterSyntax.dummy_conf with
+        externals =
+          ConfLoader.
+            [
+              external_other;
+              {
+                external_other with
+                target = { external_table_other with name = "other-1" };
+              };
+            ];
+      }
+  in
+  Alcotest.(check (result Test_migration.syntax string))
+    "external with path" expected configuration
+
+let missing_dataset () =
+  let configuration =
+    ConfLoader.load' {|[source]
+name = ""
+tab = 0
+
+[sheet]
+columns = []|}
+  and expected =
+    Error
+      {|in field "source":
+  I tried the following decoders but they all failed:
+    "file" decoder:
+      Expected an object with an attribute "file", but got name = ""
+                                                           tab = 0
+                                                            
+    "dataset" decoder: No dataset declared, but got name = ""
+                                                    tab = 0|}
+  in
+  Alcotest.(check (result Test_migration.syntax Test_migration.trimed_string))
+    "No dataset provided" expected configuration
+
+let empty_dataset () =
+  let configuration =
+    ConfLoader.load'
+      ~dataset:(fun _ -> Otoml.TomlArray [])
+      {|
+
+dataset = "…"
+
+[source]
+name = ""
+
+[sheet]
+columns = []|}
+  and expected =
+    Error
+      {|in field "dataset": Expected an object with an attribute "files", but got []|}
+  in
+  Alcotest.(check (result Test_migration.syntax Test_migration.trimed_string))
+    "Invalid Dataset" expected configuration
+
+let dataset_with_invalid_key () =
+  let configuration =
+    ConfLoader.load'
+      ~dataset:(fun _ ->
+        Otoml.(
+          TomlTable
+            [ ("files", TomlTable [ ("other-1", TomlString "other.xlsx") ]) ]))
+      {|
+
+dataset = "…"
+
+[source]
+name = ""
+
+[sheet]
+columns = []|}
+  and expected =
+    Error
+      {|in field "dataset":
+  in field "files":
+    Failed while decoding key-value pairs:
+      Expected a key without '-', but got "other-1"|}
+  in
+  Alcotest.(check (result Test_migration.syntax Test_migration.trimed_string))
+    "Invalid Dataset: invalid key" expected configuration
+
+let external_dataset () =
+  let configuration =
+    ConfLoader.load'
+      ~dataset:(fun _ ->
+        Otoml.(
+          TomlTable
+            [ ("files", TomlTable [ ("other", TomlString "other.xlsx") ]) ]))
+      {|
+
+dataset = "…"
+
+[source]
+name = ""
+file = ""
+tab = 0
+
+
+[externals.other-1]
+  # The file is not defined here
+  # And in the dataset, there is no "other-1", just "other": the application
+  # should be able to infer the information from "other" and apply it here.
+  intern_key = ":A"
+  extern_key = ":C"
+  allow_missing = false
+
+[sheet]
+columns = []|}
+  and expected =
+    Ok
+      {
+        ImporterSyntax.dummy_conf with
+        externals =
+          ConfLoader.
+            [
+              {
+                external_other with
+                target = { external_table_other with name = "other-1" };
+              };
+            ];
+      }
+  in
+  Alcotest.(check (result Test_migration.syntax string))
+    "Dataset with alias" expected configuration
+
 let test_suit =
   [
     ( "parse_extern" >:: fun _ ->
-      let toml = Otoml.Parser.from_file "configuration/simple.toml" in
-      let toml = ImportConf.t_of_toml toml in
+      let toml = ConfLoader.load_from_file "configuration/simple.toml" in
       match toml with
       | Error s -> raise (Failure s)
       | Ok result ->
@@ -55,8 +332,7 @@ let test_suit =
                (Alcotest.list Test_migration.extern_testable)
                "" [ expected ] result.externals );
     ( "parse_columns" >:: fun _ ->
-      let toml = Otoml.Parser.from_file "configuration/simple.toml" in
-      let toml = ImportConf.t_of_toml toml in
+      let toml = ConfLoader.load_from_file "configuration/simple.toml" in
 
       match toml with
       | Error s -> raise (Failure s)
@@ -83,10 +359,19 @@ let test_suit =
             (Alcotest.list Test_migration.expression_testable)
             "" expected result.columns );
     ( "parse_csv" >:: fun _ ->
-      let toml = Otoml.Parser.from_file "configuration/example_csv.toml" in
-      let toml = ImportConf.t_of_toml toml in
+      let toml = ConfLoader.load_from_file "configuration/example_csv.toml" in
       ignore toml );
     ("nested group", `Quick, nested_group);
+    ("Basic configuration", `Quick, load_configuration);
+    ("Configuration with external", `Quick, externals);
+    ("Faulty configuration", `Quick, external_with_missing_key);
+    ("Sub external", `Quick, sub_external);
+    ("Faulty configuration", `Quick, sub_external_with_missing_key);
+    ("Mix in external and sub external", `Quick, sub_external_mixed);
+    ("Missing dataset", `Quick, missing_dataset);
+    ("Empty dataset", `Quick, empty_dataset);
+    ("Dataset with invalid key", `Quick, dataset_with_invalid_key);
+    ("External dataset", `Quick, external_dataset);
   ]
 
 let tests = "configuration_toml" >::: test_suit
diff --git a/tests/test_migration.ml b/tests/test_migration.ml
index 17e48cc..acf782d 100644
--- a/tests/test_migration.ml
+++ b/tests/test_migration.ml
@@ -42,6 +42,15 @@ let extern_testable = make_test (module ImporterSyntax.Extern)
 let table_testable = make_test (module ImportDataTypes.Table)
 let int_container_testable = make_test (module ImportContainers.IntSet)
 
+let trimed_string =
+  make_test
+    (module struct
+      type t = string
+
+      let equal s1 s2 = String.equal (String.trim s1) (String.trim s2)
+      let pp format t = Format.fprintf format "%s" (String.trim t)
+    end)
+
 let expression_testable =
   make_test
     (module struct