diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/analyser_dependency.ml | 8 | ||||
| -rw-r--r-- | tests/analyser_filters.ml | 28 | ||||
| -rw-r--r-- | tests/analyser_query_test.ml | 97 | ||||
| -rw-r--r-- | tests/confLoader.ml | 26 | ||||
| -rw-r--r-- | tests/configuration_expression.ml | 181 | ||||
| -rw-r--r-- | tests/configuration_toml.ml | 348 | ||||
| -rw-r--r-- | tests/test_migration.ml | 9 |
7 files changed, 535 insertions, 162 deletions
diff --git a/tests/analyser_dependency.ml b/tests/analyser_dependency.ml index 511b706..00f21d7 100644 --- a/tests/analyser_dependency.ml +++ b/tests/analyser_dependency.ml @@ -40,6 +40,7 @@ let test_keys = name = "other"; expression = Expression.Path 3; columns = lazy (Cont.IntSet.singleton 3); + filters = []; }; ] in @@ -61,6 +62,7 @@ let test_keys_missing = name = "last_file"; expression = Expression.Path 3; columns = lazy (Cont.IntSet.singleton 3); + filters = []; }; ] in @@ -145,14 +147,14 @@ let test_unlinked = target = { file = "other.xlsx"; tab = 1; name = "circular" }; extern_key = Path 3; allow_missing = true; - match_rule = None; + filters = []; }; { intern_key = Path { alias = Some "circular"; column = 1 }; target = { file = "other2.xlsx"; tab = 1; name = "circular2" }; extern_key = Path 3; allow_missing = true; - match_rule = None; + filters = []; }; ]; columns = []; @@ -177,7 +179,7 @@ let conf_with_unlinked = target = { file = "other.xlsx"; tab = 1; name = "other" }; extern_key = Path 3; allow_missing = false; - match_rule = None; + filters = []; }; ]; columns = diff --git a/tests/analyser_filters.ml b/tests/analyser_filters.ml index 864cab7..9a54bde 100644 --- a/tests/analyser_filters.ml +++ b/tests/analyser_filters.ml @@ -35,7 +35,7 @@ let simple_filter () = let chunk_predicates = Filters.generate_sql ~conf filter chunk_links in let expected_predicates = Chunk.create () in - Chunk.add_string expected_predicates " WHERE 1=COALESCE('source'.'col_1',0)"; + Chunk.add_string expected_predicates " WHERE (1=COALESCE('source'.'col_1',0))"; Alcotest.(check @@ pair Test_migration.chunk Test_migration.chunk) "Simple predicate" @@ -58,7 +58,7 @@ let multiple_filters () = (* The predicates can be executed in reverse order, but it’s not an issue because they all are applied at the same time in the projection *) Chunk.add_string expected_predicates - " WHERE COALESCE('source'.'col_1','')=?\nAND 1"; + " WHERE (COALESCE('source'.'col_1','')=?)\nAND (1)"; Alcotest.(check @@ pair Test_migration.chunk Test_migration.chunk) "Combined predicate" @@ -80,9 +80,9 @@ let group_filter () = let expected_links = Chunk.create () in Chunk.add_string expected_links - "WITH filter0 AS (SELECT source.id, LAST_VALUE('source'.'col_1') OVER \ + "WITH filter0 AS (SELECT source.id, (LAST_VALUE('source'.'col_1') OVER \ (PARTITION BY 'source'.'col_1' ORDER BY 'source'.'col_1' RANGE BETWEEN \ - UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)=1 AS group_function\n\ + UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)=1) AS group_function\n\ FROM 'source' AS 'source')\n"; Alcotest.(check @@ pair Test_migration.chunk Test_migration.chunk) @@ -106,10 +106,10 @@ let expression_with_group () = let expected_links = Chunk.create () in Chunk.add_string expected_links - "WITH filter0 AS (SELECT source.id, LAST_VALUE('source'.'col_1') OVER \ + "WITH filter0 AS (SELECT source.id, (LAST_VALUE('source'.'col_1') OVER \ (PARTITION BY 'source'.'col_1' ORDER BY 'source'.'col_1' RANGE BETWEEN \ - UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)=1 AS group_function\n\ - FROM 'source' AS 'source' WHERE 1=COALESCE('source'.'col_1',0))\n"; + UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)=1) AS group_function\n\ + FROM 'source' AS 'source' WHERE (1=COALESCE('source'.'col_1',0)))\n"; Alcotest.(check @@ pair Test_migration.chunk Test_migration.chunk) "The predicate expression is inside of the CTE" @@ -130,13 +130,13 @@ let group_with_expression () = Chunk.add_string expected_predicates "\n\ INNER JOIN 'filter0' ON filter0.id = source.id\n\ - WHERE 1=COALESCE('source'.'col_1',0) AND filter0.group_function"; + WHERE (1=COALESCE('source'.'col_1',0)) AND filter0.group_function"; let expected_links = Chunk.create () in Chunk.add_string expected_links - "WITH filter0 AS (SELECT source.id, LAST_VALUE('source'.'col_1') OVER \ + "WITH filter0 AS (SELECT source.id, (LAST_VALUE('source'.'col_1') OVER \ (PARTITION BY 'source'.'col_1' ORDER BY 'source'.'col_1' RANGE BETWEEN \ - UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)=1 AS group_function\n\ + UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)=1) AS group_function\n\ FROM 'source' AS 'source')\n"; Alcotest.(check @@ pair Test_migration.chunk Test_migration.chunk) @@ -161,13 +161,13 @@ let group_with_group () = let expected_links = Chunk.create () in Chunk.add_string expected_links - "WITH filter0 AS (SELECT source.id, LAST_VALUE('source'.'col_1') OVER \ + "WITH filter0 AS (SELECT source.id, (LAST_VALUE('source'.'col_1') OVER \ (PARTITION BY 'source'.'col_1' ORDER BY 'source'.'col_1' RANGE BETWEEN \ - UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)=1 AS group_function\n\ + UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)=1) AS group_function\n\ FROM 'source' AS 'source')\n\ - , filter1 AS (SELECT source.id, LAST_VALUE('source'.'col_1') OVER \ + , filter1 AS (SELECT source.id, (LAST_VALUE('source'.'col_1') OVER \ (PARTITION BY 'source'.'col_1' ORDER BY 'source'.'col_1' RANGE BETWEEN \ - UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)=1 AS group_function\n\ + UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)=1) AS group_function\n\ FROM 'source' AS 'source'\n\ INNER JOIN 'filter0' ON filter0.id = source.id\n\ WHERE filter0.group_function)\n"; diff --git a/tests/analyser_query_test.ml b/tests/analyser_query_test.ml index fd8914b..37a748b 100644 --- a/tests/analyser_query_test.ml +++ b/tests/analyser_query_test.ml @@ -46,11 +46,11 @@ let check_externals = let query = Q.check_external conf (List.hd conf.externals) in let expected_query = - "SELECT 'source'.'id', 'source'.'col_1'\n\ + "SELECT 'source'.'id', ('source'.'col_1')\n\ FROM 'source' AS 'source'\n\ LEFT JOIN 'other' AS 'other' ON rtrim(upper('source'.'col_1')) = \ 'other'.'key_other' WHERE 'other'.'key_other' IS NULL AND \ - 'source'.'col_1' IS NOT NULL AND 'source'.'col_1' <> ''" + ('source'.'col_1') IS NOT NULL AND ('source'.'col_1') <> ''" in Alcotest.check Alcotest.string "" expected_query query.q @@ -179,6 +179,7 @@ let prepare_insert = name = "key_test"; expression = Concat [ Path 1; Literal "_"; Empty ]; columns = lazy (ImportContainers.IntSet.singleton 1); + filters = []; } in @@ -207,7 +208,7 @@ let filter_group = let contents, _ = ImportAnalyser.Query.select conf in let expected = - {|WITH filter0 AS (SELECT source.id, LAST_VALUE('source'.'col_3') OVER (PARTITION BY 'source'.'col_1' ORDER BY 'source'.'col_1' RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS group_function + {|WITH filter0 AS (SELECT source.id, (LAST_VALUE('source'.'col_3') OVER (PARTITION BY 'source'.'col_1' ORDER BY 'source'.'col_1' RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) AS group_function FROM 'source' AS 'source' LEFT JOIN 'other' AS 'other' ON rtrim(upper('source'.'col_1')) = 'other'.'key_other' LEFT JOIN 'last' AS 'last_file' ON rtrim(upper('other'.'col_1')) = 'last_file'.'key_last_file') @@ -239,7 +240,7 @@ let filter_group2 = let contents, _ = ImportAnalyser.Query.select conf in let expected = - {|WITH filter0 AS (SELECT source.id, LAST_VALUE('source'.'col_3') OVER (PARTITION BY 'source'.'col_1' ORDER BY 'source'.'col_1' RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS group_function + {|WITH filter0 AS (SELECT source.id, (LAST_VALUE('source'.'col_3') OVER (PARTITION BY 'source'.'col_1' ORDER BY 'source'.'col_1' RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) AS group_function FROM 'source' AS 'source' LEFT JOIN 'other' AS 'other' ON rtrim(upper('source'.'col_1')) = 'other'.'key_other' LEFT JOIN 'last' AS 'last_file' ON rtrim(upper('other'.'col_1')) = 'last_file'.'key_last_file') @@ -248,11 +249,94 @@ FROM 'source' AS 'source' LEFT JOIN 'other' AS 'other' ON rtrim(upper('source'.'col_1')) = 'other'.'key_other' LEFT JOIN 'last' AS 'last_file' ON rtrim(upper('other'.'col_1')) = 'last_file'.'key_last_file' INNER JOIN 'filter0' ON filter0.id = source.id -WHERE COALESCE('source'.'col_3',0)=0 AND filter0.group_function|} +WHERE (COALESCE('source'.'col_3',0)=0) AND filter0.group_function|} in Alcotest.check Alcotest.string "" expected contents.q +(** Add an external with a filter. Ensure the predicate is reported in the + query. *) +let external_filter = + "external_filter" >:: fun _ -> + let conf = + Syntax. + { + ConfLoader.conf with + externals = + [ + { + ConfLoader.external_other with + filters = [ Expression_builder.(equal (path 1) integer_one) ]; + }; + ]; + columns = [ Expression_builder.empty ]; + } + in + + let query, _ = ImportAnalyser.Query.select conf in + let expected_query = + {|SELECT '' AS result_0 +FROM 'source' AS 'source' +LEFT JOIN 'other' AS 'other' ON rtrim(upper('source'.'col_1')) = 'other'.'key_other' AND (COALESCE('other'.'col_1',0)=1)|} + in + Alcotest.check Test_migration.trimed_string "" expected_query query.q + +let order_by = + "order_by" >:: fun () -> + let conf = + Syntax. + { + ConfLoader.conf with + externals = []; + columns = + [ + Expression_builder.path + ImportDataTypes.Path.{ alias = None; column = 1 }; + ]; + sort = + [ + Expression_builder.path + ImportDataTypes.Path.{ alias = None; column = 1 }; + Expression_builder.integer_one; + ]; + } + in + let query, _ = ImportAnalyser.Query.select conf in + let expected_query = + {|SELECT 'source'.'col_1' AS result_0 +FROM 'source' AS 'source' +ORDER BY ('source'.'col_1'), (1)|} + in + Alcotest.check Test_migration.trimed_string "" expected_query query.q + +let group_by = + "order_by" >:: fun () -> + let conf = + Syntax. + { + ConfLoader.conf with + externals = []; + columns = + [ + Expression_builder.path + ImportDataTypes.Path.{ alias = None; column = 1 }; + ]; + uniq = + [ + Expression_builder.path + ImportDataTypes.Path.{ alias = None; column = 1 }; + Expression_builder.integer_one; + ]; + } + in + let query, _ = ImportAnalyser.Query.select conf in + let expected_query = + {|SELECT 'source'.'col_1' AS result_0 +FROM 'source' AS 'source' +GROUP BY ('source'.'col_1'), (1)|} + in + Alcotest.check Test_migration.trimed_string "" expected_query query.q + let test_suit = [ create_table; @@ -265,6 +349,9 @@ let test_suit = prepare_insert; filter_group; filter_group2; + external_filter; + order_by; + group_by; ] let tests = "analyser_query_test" >::: test_suit diff --git a/tests/confLoader.ml b/tests/confLoader.ml index 13f9840..b0be690 100644 --- a/tests/confLoader.ml +++ b/tests/confLoader.ml @@ -1,5 +1,23 @@ -let load' : string -> (ImporterSyntax.t, string) Result.t = - fun content -> Otoml.Parser.from_string content |> ImportConf.t_of_toml +(** During the test, we don’t care with the file existence *) +let context = + ImportConf. + { loadFile = (fun _ -> Otoml.array []); checkFile = (fun _ -> true) } + +let load' : + ?dataset:(string -> Otoml.t) -> + string -> + (ImporterSyntax.t, string) Result.t = + fun ?(dataset = fun _ -> Otoml.array []) content -> + let toml = Otoml.Parser.from_string content in + ImportConf.t_of_toml toml ~context:{ context with loadFile = dataset } + +let load_from_file : + ?dataset:(string -> Otoml.t) -> + string -> + (ImporterSyntax.t, string) Result.t = + fun ?(dataset = fun _ -> Otoml.array []) content -> + let toml = Otoml.Parser.from_file content in + ImportConf.t_of_toml toml ~context:{ context with loadFile = dataset } (** Read the configuration in toml and return the internal representation *) let load : string -> ImporterSyntax.t = @@ -43,7 +61,7 @@ let external_other = target = external_table_other; extern_key = Path 3; allow_missing = false; - match_rule = None; + filters = []; } let external_table_last = @@ -56,5 +74,5 @@ let external_last = target = external_table_last; extern_key = Path 3; allow_missing = true; - match_rule = None; + filters = []; } diff --git a/tests/configuration_expression.ml b/tests/configuration_expression.ml index cd28589..6478903 100644 --- a/tests/configuration_expression.ml +++ b/tests/configuration_expression.ml @@ -4,104 +4,72 @@ open Test_migration let result_testable = Alcotest.result Test_migration.expression_testable Alcotest.string +(** Helper used to test the equality between the litteral expression and it’s + AST *) +let test : string -> Path.t ImportExpression.T.t -> unit = + fun expr result -> + let expression = ImportConf.expression_from_string expr in + Alcotest.check result_testable "" (Ok result) expression + +let path_column = + "column as path" >:: fun () -> + test ":A" (Path { Path.alias = None; column = 1 }) + +let path_table = + "path with table" >:: fun () -> + test ":table.A" (Path { Path.alias = Some "table"; column = 1 }) + +let path_subtable = + "path with table" >:: fun () -> + test ":table.Name.A" (Path { Path.alias = Some "table.Name"; column = 1 }) + let parse_dquoted = "parse_dquoted" >:: fun _ -> - let expr = "match(\"\\(..\\)\", :B)" in - let result = ImportConf.expression_from_string expr in - Alcotest.check result_testable "" - (Ok - (Function - ("match", [ Literal "\\(..\\)"; Path { alias = None; column = 2 } ]))) - result + test "match(\"\\(..\\)\", :B)" + (Function + ("match", [ Literal "\\(..\\)"; Path { alias = None; column = 2 } ])) let parse_quoted = "parse_quoted" >:: fun _ -> - let expr = "match('\\(..\\)', :B)" in - let result = ImportConf.expression_from_string expr in - Alcotest.check result_testable "" - (Ok - (Function - ("match", [ Literal "\\(..\\)"; Path { alias = None; column = 2 } ]))) - result + test "match('\\(..\\)', :B)" + (Function + ("match", [ Literal "\\(..\\)"; Path { alias = None; column = 2 } ])) let concat = "concat" >:: fun _ -> - let expr = ":A ^ :B" in - let result = ImportConf.expression_from_string expr in - Alcotest.check result_testable "" - (Ok - (Concat - [ - Path { alias = None; column = 1 }; Path { alias = None; column = 2 }; - ])) - result + test ":A ^ :B" + (Concat + [ Path { alias = None; column = 1 }; Path { alias = None; column = 2 } ]) let concat2 = "concat2" >:: fun _ -> - let expr = "'A' ^ '_' ^ 'B'" in - let result = ImportConf.expression_from_string expr in - Alcotest.check result_testable "" - (Ok (Concat [ Literal "A"; Literal "_"; Literal "B" ])) - result + test "'A' ^ '_' ^ 'B'" (Concat [ Literal "A"; Literal "_"; Literal "B" ]) let litteral = "litteral" >:: fun _ -> (* The text is quoted in shall not be considered as a path *) - let expr = "':A'" in - let result = ImportConf.expression_from_string expr in - Alcotest.check result_testable "" (Ok (Literal ":A")) result + test "':A'" (Literal ":A") -let empty = - "empty" >:: fun _ -> - let expr = "" in - let result = ImportConf.expression_from_string expr in - Alcotest.check result_testable "" (Ok Empty) result +let empty = "empty" >:: fun _ -> test "" Empty let upper_nvl = - "upper_nvl" >:: fun _ -> - let expr = "NVL('','')" in - let result = ImportConf.expression_from_string expr in - Alcotest.check result_testable "" (Ok (Nvl [ Empty; Empty ])) result + "upper_nvl" >:: fun _ -> test "NVL('','')" (Nvl [ Empty; Empty ]) let lower_nvl = - "lower_nvl" >:: fun _ -> - let expr = "nvl('','')" in - let result = ImportConf.expression_from_string expr in - Alcotest.check result_testable "" (Ok (Nvl [ Empty; Empty ])) result - -let numeric = - "numeric" >:: fun _ -> - let expr = "123" in - let result = ImportConf.expression_from_string expr in - Alcotest.check result_testable "" (Ok (Integer "123")) result + "lower_nvl" >:: fun _ -> test "nvl('','')" (Nvl [ Empty; Empty ]) -let numeric_neg = - "numeric_neg" >:: fun _ -> - let expr = "-123" in - let result = ImportConf.expression_from_string expr in - Alcotest.check result_testable "" (Ok (Integer "-123")) result +let numeric = "numeric" >:: fun _ -> test "123" (Integer "123") +let numeric_neg = "numeric_neg" >:: fun _ -> test "-123" (Integer "-123") let op_priority = "operator_priority" >:: fun _ -> - let expr = "1 + 2 > 2" in - let result = ImportConf.expression_from_string expr - and expected = - ImportExpression.T.( - BOperator (GT, BOperator (Add, Integer "1", Integer "2"), Integer "2")) - in - - Alcotest.check result_testable "" (Ok expected) result + test "1 + 2 > 2" + (BOperator (GT, BOperator (Add, Integer "1", Integer "2"), Integer "2")) let op_priority2 = "operator_priority" >:: fun _ -> - let expr = "1 ^ 2 = 2" in - let result = ImportConf.expression_from_string expr - and expected = - ImportExpression.T.( - BOperator (Equal, Concat [ Integer "1"; Integer "2" ], Integer "2")) - in - - Alcotest.check result_testable "" (Ok expected) result + test "1 ^ 2 = 2" + (BOperator (Equal, Concat [ Integer "1"; Integer "2" ], Integer "2")) let join = "join" >:: fun _ -> @@ -119,29 +87,15 @@ let join = let join_empty = "join" >:: fun _ -> - let expr = "join('', :A, :B)" in - let result = ImportConf.expression_from_string expr in - Alcotest.check result_testable "" - (Ok - (Join - ( "", - [ - Path { alias = None; column = 1 }; - Path { alias = None; column = 2 }; - ] ))) - result + test "join('', :A, :B)" + (Join + ( "", + [ + Path { alias = None; column = 1 }; Path { alias = None; column = 2 }; + ] )) -let upper = - "upper" >:: fun _ -> - let expr = "upper('')" in - let result = ImportConf.expression_from_string expr in - Alcotest.check result_testable "" (Ok (Function' (Upper, [ Empty ]))) result - -let trim = - "trim" >:: fun _ -> - let expr = "trim('')" in - let result = ImportConf.expression_from_string expr in - Alcotest.check result_testable "" (Ok (Function' (Trim, [ Empty ]))) result +let upper = "upper" >:: fun _ -> test "upper('')" (Function' (Upper, [ Empty ])) +let trim = "trim" >:: fun _ -> test "trim('')" (Function' (Trim, [ Empty ])) (** Extract the columns from a window function *) let fold_values = @@ -182,44 +136,24 @@ let bad_quote = let nested_expression = "nested_expression" >:: fun _ -> - let expr = "1 = (1 = 0)" in - let result = ImportConf.expression_from_string expr in - Alcotest.check result_testable "" - (Ok - (BOperator - ( Equal, - Integer "1", - Expr (BOperator (Equal, Integer "1", Integer "0")) ))) - result + test "1 = (1 = 0)" + (BOperator + (Equal, Integer "1", Expr (BOperator (Equal, Integer "1", Integer "0")))) let priority_equality = "priority_equality" >:: fun _ -> - let expr = "1 = 1 = 0" in - let result = ImportConf.expression_from_string expr in - Alcotest.check result_testable "" - (Ok - (BOperator - (Equal, Integer "1", BOperator (Equal, Integer "1", Integer "0")))) - result + test "1 = 1 = 0" + (BOperator (Equal, Integer "1", BOperator (Equal, Integer "1", Integer "0"))) let priority_operator_and = "priority_equality" >:: fun _ -> - let expr = "1 and 1 = 0" in - let result = ImportConf.expression_from_string expr in - Alcotest.check result_testable "" - (Ok - (BOperator (And, Integer "1", BOperator (Equal, Integer "1", Integer "0")))) - result + test "1 and 1 = 0" + (BOperator (And, Integer "1", BOperator (Equal, Integer "1", Integer "0"))) let priority_operator_or = "priority_equality" >:: fun _ -> - let expr = "1 <> 1 or 0" in - let result = ImportConf.expression_from_string expr in - Alcotest.check result_testable "" - (Ok - (BOperator - (Or, BOperator (Different, Integer "1", Integer "1"), Integer "0"))) - result + test "1 <> 1 or 0" + (BOperator (Or, BOperator (Different, Integer "1", Integer "1"), Integer "0")) let unknown_function = "unknown function" >:: fun _ -> @@ -240,6 +174,9 @@ let wrong_arguments = let test_suit = [ + path_column; + path_table; + path_subtable; parse_dquoted; parse_quoted; concat; diff --git a/tests/configuration_toml.ml b/tests/configuration_toml.ml index 0a36faf..620a106 100644 --- a/tests/configuration_toml.ml +++ b/tests/configuration_toml.ml @@ -5,12 +5,12 @@ open Test_migration let nested_group () = let expected = Error - "in field \"sheet\":\n\ - \ in field \"columns\":\n\ - \ while decoding a list:\n\ - \ element 0:\n\ - \ A group function cannot contains another group function, but got\n\ - \ \"max(:A, [counter([:A], [:A])], [])\" \n" + {|in field "sheet": + in field "columns": + while decoding a list: + element 0: + A group function cannot contains another group function, but got + "max(:A, [counter([:A], [:A])], [])"|} and result = ConfLoader.load' {|[source] @@ -22,14 +22,325 @@ columns = [ "max(:A, [counter([:A], [:A])], [])", ]|} in - Alcotest.(check (result Test_migration.syntax string)) + Alcotest.(check (result Test_migration.syntax Test_migration.trimed_string)) "duplicate" expected result +(** Load a simple configuration *) +let load_configuration () = + let configuration = + ConfLoader.load' + {|[source] +name = "" +file = "" +tab = 0 + +[sheet] +columns = []|} + and expected = Ok ImporterSyntax.dummy_conf in + Alcotest.(check (result Test_migration.syntax string)) + "Simple configuration" expected configuration + +let externals () = + let configuration = + ConfLoader.load' + {|[source] +name = "" +file = "" +tab = 0 + +[externals.other] + intern_key = ":A" + file = "other.xlsx" + extern_key = ":C" + allow_missing = false + +[sheet] +columns = []|} + and expected = + Ok + { + ImporterSyntax.dummy_conf with + externals = [ ConfLoader.external_other ]; + } + in + Alcotest.(check (result Test_migration.syntax Test_migration.trimed_string)) + "Simple external" expected configuration + +(** There is an error in this configuration the key [intern_key] is missing in + the external *) +let external_with_missing_key () = + let configuration = + ConfLoader.load' + {|[source] +name = "" +file = "" + +[externals.other] + file = "" + extern_key = "" + +[sheet] +columns = []|} + and expected = + Error + {|in field "externals": + Failed while decoding key-value pairs: + Expected an object with an attribute "intern_key", but got + file = "" + extern_key = ""|} + in + Alcotest.(check (result Test_migration.syntax Test_migration.trimed_string)) + "Missing key" expected configuration + +let sub_external () = + let configuration = + ConfLoader.load' + {|[source] +name = "" +file = "" +tab = 0 + + +[externals.other-1] + intern_key = ":A" + file = "other.xlsx" + extern_key = ":C" + allow_missing = false + +[sheet] +columns = []|} + and expected = + Ok + { + ImporterSyntax.dummy_conf with + externals = + ConfLoader. + [ + { + external_other with + target = { external_table_other with name = "other-1" }; + }; + ]; + } + in + Alcotest.(check (result Test_migration.syntax string)) + "external with path" expected configuration + +let sub_external_with_missing_key () = + let configuration = + ConfLoader.load' + {|[source] +name = "" +file = "" + +[externals.other-1] + file = "" + extern_key = "" + +[sheet] +columns = []|} + and expected = + Error + {|in field "externals": + Failed while decoding key-value pairs: + Expected an object with an attribute "intern_key", but got + file = "" + extern_key = ""|} + in + Alcotest.(check (result Test_migration.syntax Test_migration.trimed_string)) + "Missing intern_key" expected configuration + +(** The same configuration has external, and sub-element external *) +let sub_external_mixed () = + let configuration = + ConfLoader.load' + {|[source] +name = "" +file = "" +tab = 0 + +[externals.other] + intern_key = ":A" + file = "other.xlsx" + extern_key = ":C" + allow_missing = false + +[externals.other-1] + intern_key = ":A" + file = "other.xlsx" + extern_key = ":C" + allow_missing = false + +[sheet] +columns = []|} + and expected = + Ok + { + ImporterSyntax.dummy_conf with + externals = + ConfLoader. + [ + external_other; + { + external_other with + target = { external_table_other with name = "other-1" }; + }; + ]; + } + in + Alcotest.(check (result Test_migration.syntax string)) + "external with path" expected configuration + +let missing_dataset () = + let configuration = + ConfLoader.load' {|[source] +name = "" +tab = 0 + +[sheet] +columns = []|} + and expected = + Error + {|in field "source": + I tried the following decoders but they all failed: + "file" decoder: + Expected an object with an attribute "file", but got name = "" + tab = 0 + + "dataset" decoder: No dataset declared, but got name = "" + tab = 0|} + in + Alcotest.(check (result Test_migration.syntax Test_migration.trimed_string)) + "No dataset provided" expected configuration + +let empty_dataset () = + let configuration = + ConfLoader.load' + ~dataset:(fun _ -> Otoml.TomlArray []) + {| + +dataset = "…" + +[source] +name = "" + +[sheet] +columns = []|} + and expected = + Error + {|in field "dataset": Expected an object with an attribute "files", but got []|} + in + Alcotest.(check (result Test_migration.syntax Test_migration.trimed_string)) + "Invalid Dataset" expected configuration + +let dataset_with_invalid_key () = + let configuration = + ConfLoader.load' + ~dataset:(fun _ -> + Otoml.( + TomlTable + [ ("files", TomlTable [ ("other-1", TomlString "other.xlsx") ]) ])) + {| + +dataset = "…" + +[source] +name = "" + +[sheet] +columns = []|} + and expected = + Error + {|in field "dataset": + in field "files": + Failed while decoding key-value pairs: + Expected a key without '-', but got "other-1"|} + in + Alcotest.(check (result Test_migration.syntax Test_migration.trimed_string)) + "Invalid Dataset: invalid key" expected configuration + +let external_dataset () = + let configuration = + ConfLoader.load' + ~dataset:(fun _ -> + Otoml.( + TomlTable + [ ("files", TomlTable [ ("other", TomlString "other.xlsx") ]) ])) + {| + +dataset = "…" + +[source] +name = "" +file = "" +tab = 0 + + +[externals.other-1] + # The file is not defined here + # And in the dataset, there is no "other-1", just "other": the application + # should be able to infer the information from "other" and apply it here. + intern_key = ":A" + extern_key = ":C" + allow_missing = false + +[sheet] +columns = []|} + and expected = + Ok + { + ImporterSyntax.dummy_conf with + externals = + ConfLoader. + [ + { + external_other with + target = { external_table_other with name = "other-1" }; + }; + ]; + } + in + Alcotest.(check (result Test_migration.syntax string)) + "Dataset with alias" expected configuration + +let external_filters () = + let configuration = + ConfLoader.load' + {|[source] +name = "" +file = "" +tab = 0 + +[externals.other] + intern_key = ":A" + file = "other.xlsx" + extern_key = ":C" + filters = [":B = 1"] + + +[sheet] +columns = []|} + and expected = + Ok + { + ImporterSyntax.dummy_conf with + externals = + ConfLoader. + [ + { + external_other with + filters = [ Expression_builder.(equal (path 2) integer_one) ]; + }; + ]; + } + in + Alcotest.(check (result Test_migration.syntax string)) + "Filters in external" expected configuration + let test_suit = [ ( "parse_extern" >:: fun _ -> - let toml = Otoml.Parser.from_file "configuration/simple.toml" in - let toml = ImportConf.t_of_toml toml in + let toml = ConfLoader.load_from_file "configuration/simple.toml" in match toml with | Error s -> raise (Failure s) | Ok result -> @@ -45,7 +356,7 @@ let test_suit = Path { alias = None; column = 1 }; Path { alias = None; column = 2 }; ] ); - match_rule = None; + filters = []; allow_missing = true; } in @@ -55,8 +366,7 @@ let test_suit = (Alcotest.list Test_migration.extern_testable) "" [ expected ] result.externals ); ( "parse_columns" >:: fun _ -> - let toml = Otoml.Parser.from_file "configuration/simple.toml" in - let toml = ImportConf.t_of_toml toml in + let toml = ConfLoader.load_from_file "configuration/simple.toml" in match toml with | Error s -> raise (Failure s) @@ -83,10 +393,20 @@ let test_suit = (Alcotest.list Test_migration.expression_testable) "" expected result.columns ); ( "parse_csv" >:: fun _ -> - let toml = Otoml.Parser.from_file "configuration/example_csv.toml" in - let toml = ImportConf.t_of_toml toml in + let toml = ConfLoader.load_from_file "configuration/example_csv.toml" in ignore toml ); ("nested group", `Quick, nested_group); + ("Basic configuration", `Quick, load_configuration); + ("Configuration with external", `Quick, externals); + ("Faulty configuration", `Quick, external_with_missing_key); + ("Sub external", `Quick, sub_external); + ("Faulty configuration", `Quick, sub_external_with_missing_key); + ("Mix in external and sub external", `Quick, sub_external_mixed); + ("Missing dataset", `Quick, missing_dataset); + ("Empty dataset", `Quick, empty_dataset); + ("Dataset with invalid key", `Quick, dataset_with_invalid_key); + ("External dataset", `Quick, external_dataset); + ("External with filter", `Quick, external_filters); ] let tests = "configuration_toml" >::: test_suit diff --git a/tests/test_migration.ml b/tests/test_migration.ml index 17e48cc..acf782d 100644 --- a/tests/test_migration.ml +++ b/tests/test_migration.ml @@ -42,6 +42,15 @@ let extern_testable = make_test (module ImporterSyntax.Extern) let table_testable = make_test (module ImportDataTypes.Table) let int_container_testable = make_test (module ImportContainers.IntSet) +let trimed_string = + make_test + (module struct + type t = string + + let equal s1 s2 = String.equal (String.trim s1) (String.trim s2) + let pp format t = Format.fprintf format "%s" (String.trim t) + end) + let expression_testable = make_test (module struct |