From 6b377719c10d5ab3343fd5221f99a4a21008e25a Mon Sep 17 00:00:00 2001 From: Sébastien Dailly Date: Thu, 14 Mar 2024 08:26:58 +0100 Subject: Initial commit --- tests/analyser_dependency.ml | 247 ++++++++++++++++++++++++++++ tests/analyser_query_test.ml | 304 +++++++++++++++++++++++++++++++++++ tests/confLoader.ml | 128 +++++++++++++++ tests/configuration/example_csv.toml | 31 ++++ tests/configuration/simple.toml | 20 +++ tests/configuration_expression.ml | 253 +++++++++++++++++++++++++++++ tests/configuration_toml.ml | 71 ++++++++ tests/dune | 17 ++ tests/expression_builder.ml | 47 ++++++ tests/expression_query.ml | 208 ++++++++++++++++++++++++ tests/expression_repr.ml | 37 +++++ tests/expression_type_of.ml | 65 ++++++++ tests/importCSV_test.ml | 29 ++++ tests/importConf_test.ml | 23 +++ tests/importer_test.ml | 21 +++ tests/sql_date.ml | 18 +++ tests/sql_db.ml | 198 +++++++++++++++++++++++ tests/sql_int.ml | 26 +++ tests/sql_match.ml | 12 ++ tests/sql_trim.ml | 11 ++ 20 files changed, 1766 insertions(+) create mode 100644 tests/analyser_dependency.ml create mode 100644 tests/analyser_query_test.ml create mode 100644 tests/confLoader.ml create mode 100644 tests/configuration/example_csv.toml create mode 100644 tests/configuration/simple.toml create mode 100644 tests/configuration_expression.ml create mode 100644 tests/configuration_toml.ml create mode 100644 tests/dune create mode 100644 tests/expression_builder.ml create mode 100644 tests/expression_query.ml create mode 100644 tests/expression_repr.ml create mode 100644 tests/expression_type_of.ml create mode 100644 tests/importCSV_test.ml create mode 100644 tests/importConf_test.ml create mode 100644 tests/importer_test.ml create mode 100644 tests/sql_date.ml create mode 100644 tests/sql_db.ml create mode 100644 tests/sql_int.ml create mode 100644 tests/sql_match.ml create mode 100644 tests/sql_trim.ml (limited to 'tests') diff --git a/tests/analyser_dependency.ml b/tests/analyser_dependency.ml new file mode 100644 index 0000000..dea7727 --- /dev/null +++ b/tests/analyser_dependency.ml @@ -0,0 +1,247 @@ +open OUnit2 +open StdLabels +module A = ImportAnalyser.Dependency +module Cont = ImportContainers +module Syntax = ImportConf.Syntax +module Expression = ImportExpression.T +module Table = ImportDataTypes.Table +open ConfLoader + +let test_order = + "Order" >:: fun _ -> + let order = A.get_process_order conf |> List.map ~f:A.table in + let expected_order = + [ external_table_last; external_table_other; external_table_source ] + in + assert_equal ~cmp:(cmp_list cmp_source) ~printer:show_sources expected_order + order + +let test_columns = + "Columns" >:: fun _ -> + let expected_colums = Cont.IntSet.of_list [ 1; 2 ] in + let columns = + A.get_process_order conf + |> List.find ~f:(fun v -> A.table v = external_table_source) + |> A.columns + in + + assert_equal + ~cmp:(fun a b -> 0 = Cont.IntSet.compare a b) + ~printer:Cont.show_intSet expected_colums columns + +let test_keys = + "Keys" >:: fun _ -> + (* We should have one key in the table other, because it is referenced as a + source in last file *) + let expected_keys = + A. + [ + { + name = "other"; + expression = Expression.Path 3; + columns = lazy (Cont.IntSet.singleton 3); + }; + ] + in + + let keys = + A.get_process_order conf + |> List.find ~f:(fun v -> A.table v = external_table_other) + |> A.keys + in + + assert_equal ~cmp:(cmp_list key_cmp) ~printer:keys_printer expected_keys keys + +let test_keys_missing = + "Keys missing" >:: fun _ -> + (* We have no key in last_file because the key is declared as missing *) + let expected_keys = + A. + [ + { + name = "last_file"; + expression = Expression.Path 3; + columns = lazy (Cont.IntSet.singleton 3); + }; + ] + in + + let keys = + A.get_process_order conf + |> List.find ~f:(fun v -> A.table v = external_table_last) + |> A.keys + in + + assert_equal ~cmp:(cmp_list key_cmp) ~printer:keys_printer expected_keys keys + +let test_unknow_source = + "Unknown source" >:: fun _ -> + let conf = { conf with externals = [] } in + assert_raises (ImportErrors.Unknown_source "last_file") (fun () -> + A.get_process_order conf) + +let test_unordered = + "Unorderd references" >:: fun _ -> + (* Externals not described in the right order shall not raise any + error. *) + let conf = + load + {|version = 1 +[source] + file = "source.xlsx" + name = "source" + +[externals.last_file] + intern_key = ":other.A" + file = "last.xlsx" + extern_key = ":C" + allow_missing = true + +[externals.other] + intern_key = ":A" + file = "other.xlsx" + extern_key = ":C" + allow_missing = false + +[sheet] + columns = []|} + in + assert_raises (ImportErrors.Unknown_source "other") (fun () -> + A.get_process_order conf) + +let test_circular = + "Unlinked reference" >:: fun _ -> + (* A reference to itself should be understood *) + let conf = + load + {|version = 1 +[source] + file = "source.xlsx" + name = "source" + +[externals.circular] + intern_key = ":circular.A" + file = "last.xlsx" + extern_key = ":A" + allow_missing = true + +[sheet] + columns = []|} + in + + let elements = A.get_process_order conf in + assert_equal ~printer:string_of_int 1 (List.length elements) + +let test_unlinked = + "Circular reference" >:: fun _ -> + (* An element linked to anything (except itself) should be ignored *) + let conf = + Syntax. + { + version = 1; + source = external_table_source; + externals = + [ + { + intern_key = Path { alias = Some "circular2"; column = 1 }; + target = { file = "other.xlsx"; tab = 1; name = "circular" }; + extern_key = Path 3; + allow_missing = true; + match_rule = None; + }; + { + intern_key = Path { alias = Some "circular"; column = 1 }; + target = { file = "other2.xlsx"; tab = 1; name = "circular2" }; + extern_key = Path 3; + allow_missing = true; + match_rule = None; + }; + ]; + columns = []; + filters = []; + sort = []; + uniq = []; + } + in + assert_raises (ImportErrors.Unknown_source "circular2") (fun () -> + A.get_process_order conf |> List.map ~f:A.table) + +let conf_with_unlinked = + Syntax. + { + version = 1; + source = external_table_source; + externals = + [ + { + intern_key = Path { alias = None; column = 1 }; + target = { file = "other.xlsx"; tab = 1; name = "other" }; + extern_key = Path 3; + allow_missing = false; + match_rule = None; + }; + ]; + columns = + [ + Concat [ Path { alias = None; column = 1 }; Literal "_"; Empty ]; + Path { alias = None; column = 2 }; + ]; + filters = []; + sort = []; + uniq = []; + } + +(** A table referenced only in a filter list shall be loaded correctly *) +let test_order_filter = + "Order filter" >:: fun _ -> + let order = + { + conf_with_unlinked with + filters = [ Path { alias = Some "other"; column = 5 } ]; + } + |> A.get_process_order |> List.map ~f:A.table + in + let expected_order = [ external_table_other; external_table_source ] in + assert_equal ~printer:show_sources expected_order order + +(** A table referenced only in the order list shall be loaded correctly *) +let test_order_sort = + "Order sort" >:: fun _ -> + let order = + { + conf_with_unlinked with + sort = [ Path { alias = Some "other"; column = 5 } ]; + } + |> A.get_process_order |> List.map ~f:A.table + in + let expected_order = [ external_table_other; external_table_source ] in + assert_equal ~printer:show_sources expected_order order + +(** A table referenced only in the uniq list shall be loaded correctly *) +let test_order_uniq = + "Order uniq" >:: fun _ -> + let order = + { + conf_with_unlinked with + uniq = [ Path { alias = Some "other"; column = 5 } ]; + } + |> A.get_process_order |> List.map ~f:A.table + in + let expected_order = [ external_table_other; external_table_source ] in + assert_equal ~printer:show_sources expected_order order + +let tests = + "analyser_dependency" + >::: [ + test_order; + test_columns; + test_keys; + test_keys_missing; + test_unknow_source; + test_unordered; + test_circular; + test_unlinked; + test_order_filter; + test_order_sort; + test_order_uniq; + ] diff --git a/tests/analyser_query_test.ml b/tests/analyser_query_test.ml new file mode 100644 index 0000000..3559de4 --- /dev/null +++ b/tests/analyser_query_test.ml @@ -0,0 +1,304 @@ +open OUnit2 +open StdLabels +module A = ImportAnalyser.Dependency +module Q = ImportAnalyser.Query +module C = ImportConf +module Syntax = ImportConf.Syntax +module Expr = Expression_builder + +let show_source (source : ImportDataTypes.Table.t) = + Printf.sprintf "%s:%d" source.ImportDataTypes.Table.file source.tab + +let show_sources sources = + let b = Buffer.create 16 in + Buffer.add_string b "["; + List.iter sources ~f:(fun source -> + Buffer.add_string b (show_source source); + Buffer.add_string b ","); + + let len = Buffer.length b in + if len > 1 then Buffer.truncate b (len - 1); + Buffer.add_string b "]"; + + Buffer.contents b + +(** This is sample configuration used in the tests *) +let conf = + Syntax. + { + version = 1; + source = { file = "source.xlsx"; tab = 1; name = "source" }; + externals = + [ + { + intern_key = Path { alias = None; column = 1 }; + target = { file = "other.xlsx"; tab = 1; name = "other" }; + extern_key = Path 3; + allow_missing = false; + match_rule = None; + }; + { + intern_key = Path { alias = Some "other"; column = 1 }; + target = { file = "last.xlsx"; tab = 1; name = "last_file" }; + extern_key = Path 3; + allow_missing = true; + match_rule = None; + }; + ]; + columns = + [ + Concat [ Path { alias = None; column = 1 }; Literal "_"; Empty ]; + Path { alias = None; column = 2 }; + Path { alias = Some "last_file"; column = 5 }; + ]; + filters = []; + sort = []; + uniq = []; + } + +let create_table = + "Create table" >:: fun _ -> + let out = A.get_process_order conf in + + let query = Q.create_table (List.hd out) in + + assert_equal ~printer:Fun.id + "CREATE TABLE 'last' (id INTEGER PRIMARY KEY,'key_last_file','col_5')" query + +let select = + "Select" >:: fun _ -> + let query, _ = Q.select conf in + let expected_query = + {|SELECT COALESCE('source'.col_1,'') || ? || '' AS result_0, +'source'.col_2 AS result_1, +'last_file'.col_5 AS result_2 +FROM 'source' AS 'source' +LEFT JOIN 'other' AS 'other' ON rtrim(upper('source'.col_1)) = 'other'.'key_other' +LEFT JOIN 'last' AS 'last_file' ON rtrim(upper('other'.col_1)) = 'last_file'.'key_last_file'|} + in + + assert_equal ~printer:Fun.id expected_query query.q + +let check_externals = + "Check external" >:: fun _ -> + let query = Q.check_external conf (List.hd conf.externals) in + + let expected_query = + "SELECT 'source'.'id', 'source'.col_1 FROM\n\ + 'source' AS 'source' LEFT JOIN 'other' AS 'other' ON \ + rtrim(upper('source'.col_1)) = 'other'.'key_other' WHERE \ + 'other'.'key_other' IS NULL AND 'source'.col_1 IS NOT NULL AND \ + 'source'.col_1 <> ''" + in + + assert_equal ~printer:Fun.id expected_query query.q + +let previous = + "Test window previous" >:: fun _ -> + (* This is sample configuration used in the tests *) + let conf = + Syntax. + { + version = 1; + source = { file = "source.xlsx"; tab = 1; name = "previous" }; + externals = []; + columns = + [ + Window + ( Previous (Path { alias = None; column = 5 }), + [ Path { alias = None; column = 1 } ], + [ Path { alias = None; column = 3 } ] ); + ]; + filters = []; + sort = []; + uniq = []; + } + in + + let res, _ = ImportAnalyser.Query.select conf in + let query = + "SELECT LAG('previous'.col_5) OVER (PARTITION BY 'previous'.col_1 ORDER BY \ + 'previous'.col_3) AS result_0\n\ + FROM 'source' AS 'previous'" + in + assert_equal ~printer:Fun.id query res.q + +let sum = + "Test window sum" >:: fun _ -> + (* This is sample configuration used in the tests *) + let conf = + Syntax. + { + version = 1; + source = { file = "source.xlsx"; tab = 1; name = "previous" }; + externals = []; + columns = + [ + Window + ( Sum (Path { alias = None; column = 5 }), + [ Path { alias = None; column = 1 } ], + [] ); + ]; + filters = []; + sort = []; + uniq = []; + } + in + + let res, _ = ImportAnalyser.Query.select conf in + let query = + "SELECT SUM('previous'.col_5) OVER (PARTITION BY 'previous'.col_1) AS \ + result_0\n\ + FROM 'source' AS 'previous'" + in + assert_equal ~printer:Fun.id query res.q + +let sum_total = + "Test sum over the whole range" >:: fun _ -> + (* This is sample configuration used in the tests *) + let conf = + Syntax. + { + version = 1; + source = { file = "source.xlsx"; tab = 1; name = "previous" }; + externals = []; + columns = [ Window (Sum (Path { alias = None; column = 5 }), [], []) ]; + filters = []; + sort = []; + uniq = []; + } + in + + let res, _ = ImportAnalyser.Query.select conf in + let query = + "SELECT SUM('previous'.col_5) AS result_0\nFROM 'source' AS 'previous'" + in + assert_equal ~printer:Fun.id query res.q + +let sum_unfiltered = + "Test sum over the whole range" >:: fun _ -> + (* This is sample configuration used in the tests *) + let conf = + Syntax. + { + version = 1; + source = { file = "source.xlsx"; tab = 1; name = "previous" }; + externals = []; + columns = + [ + Window + ( Sum (Path { alias = None; column = 5 }), + [], + [ Path { alias = None; column = 1 } ] ); + ]; + filters = []; + sort = []; + uniq = []; + } + in + + let res, _ = ImportAnalyser.Query.select conf in + let query = + "SELECT SUM('previous'.col_5) AS result_0\nFROM 'source' AS 'previous'" + in + assert_equal ~printer:Fun.id query res.q + +let prepare_insert = + "Test prepare_insert" >:: fun _ -> + let key = + ImportAnalyser.Dependency. + { + name = "key_test"; + expression = Concat [ Path 1; Literal "_"; Empty ]; + columns = lazy (ImportContainers.IntSet.singleton 1); + } + in + + let buffer = Buffer.create 16 in + let () = ImportAnalyser.Query.build_key_insert buffer key in + let contents = Buffer.contents buffer in + + let expected = "rtrim(upper(COALESCE(:col_1,'') || '_' || ''))" in + + assert_equal ~printer:Fun.id expected contents + +(** Test a request with a group in a filter. + +This generate a CTE expression in order to evaluate the group before loading +the results from the query. *) +let filter_group = + "Test filter_group" >:: fun _ -> + let c col = Expr.path ImportDataTypes.Path.{ alias = None; column = col } in + let conf = + { + conf with + columns = [ c 1 ]; + filters = [ Expr.(max (c 3) [ c 1 ] [ c 1 ]) ]; + } + in + let contents, _ = ImportAnalyser.Query.select conf in + + let expected = + {|WITH cte AS (SELECT source.id, LAST_VALUE('source'.col_3) OVER (PARTITION BY 'source'.col_1 ORDER BY 'source'.col_1 RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS group0 +FROM 'source' AS 'source' +LEFT JOIN 'other' AS 'other' ON rtrim(upper('source'.col_1)) = 'other'.'key_other' +LEFT JOIN 'last' AS 'last_file' ON rtrim(upper('other'.col_1)) = 'last_file'.'key_last_file') +SELECT 'source'.col_1 AS result_0 +FROM 'source' AS 'source' +LEFT JOIN 'other' AS 'other' ON rtrim(upper('source'.col_1)) = 'other'.'key_other' +LEFT JOIN 'last' AS 'last_file' ON rtrim(upper('other'.col_1)) = 'last_file'.'key_last_file' +INNER JOIN 'cte' ON cte.id = source.id +WHERE (cte.group0)|} + in + + assert_equal ~printer:(fun s -> Printf.sprintf "\n%s" s) expected contents.q + +(** Test a request with a group in a filter. + +This generate a CTE expression in order to evaluate the group before loading +the results from the query. *) +let filter_group2 = + "Test filter_group" >:: fun _ -> + let c col = Expr.path ImportDataTypes.Path.{ alias = None; column = col } in + let conf = + { + conf with + columns = [ c 1 ]; + filters = + [ Expr.(max (c 3) [ c 1 ] [ c 1 ]); Expr.equal (c 3) Expr.integer_zero ]; + } + in + let contents, _ = ImportAnalyser.Query.select conf in + + let expected = + {|WITH cte AS (SELECT source.id, LAST_VALUE('source'.col_3) OVER (PARTITION BY 'source'.col_1 ORDER BY 'source'.col_1 RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS group0 +FROM 'source' AS 'source' +LEFT JOIN 'other' AS 'other' ON rtrim(upper('source'.col_1)) = 'other'.'key_other' +LEFT JOIN 'last' AS 'last_file' ON rtrim(upper('other'.col_1)) = 'last_file'.'key_last_file' +WHERE COALESCE('source'.col_3,0)=0) +SELECT 'source'.col_1 AS result_0 +FROM 'source' AS 'source' +LEFT JOIN 'other' AS 'other' ON rtrim(upper('source'.col_1)) = 'other'.'key_other' +LEFT JOIN 'last' AS 'last_file' ON rtrim(upper('other'.col_1)) = 'last_file'.'key_last_file' +INNER JOIN 'cte' ON cte.id = source.id +WHERE COALESCE('source'.col_3,0)=0 +AND (cte.group0)|} + in + + assert_equal ~printer:(fun s -> Printf.sprintf "\n%s" s) expected contents.q + +let test_suit = + [ + create_table; + select; + check_externals; + previous; + sum; + sum_total; + sum_unfiltered; + prepare_insert; + filter_group; + filter_group2; + ] + +let tests = "analyser_query_test" >::: test_suit diff --git a/tests/confLoader.ml b/tests/confLoader.ml new file mode 100644 index 0000000..266ff33 --- /dev/null +++ b/tests/confLoader.ml @@ -0,0 +1,128 @@ +open StdLabels + +(** Read the configuration in toml and return the internal representation *) +let load : string -> ImportConf.Syntax.t = + fun content -> + Otoml.Parser.from_string content |> ImportConf.t_of_toml |> Result.get_ok + +let conf = + load + {|version = 1 + +[source] + file = "source.xlsx" + name = "source" + +[externals.other] + intern_key = ":A" + file = "other.xlsx" + extern_key = ":C" + allow_missing = false + +[externals.last_file] + intern_key = ":other.A" + file = "last.xlsx" + extern_key = ":C" + allow_missing = true + +[sheet] + columns = [ + ":A ^ '_'", + ":B", + ":last_file.E", + ]|} + +let external_table_source = + ImportDataTypes.Table.{ file = "source.xlsx"; tab = 1; name = "source" } + +let external_table_other = + ImportDataTypes.Table.{ file = "other.xlsx"; tab = 1; name = "other" } + +let external_other = + ImportConf.Syntax. + { + intern_key = Path { alias = None; column = 1 }; + target = external_table_other; + extern_key = Path 3; + allow_missing = false; + match_rule = None; + } + +let external_table_last = + ImportDataTypes.Table.{ file = "last.xlsx"; tab = 1; name = "last_file" } + +let external_last = + ImportConf.Syntax. + { + intern_key = Path { alias = Some "other"; column = 1 }; + target = external_table_last; + extern_key = Path 3; + allow_missing = true; + match_rule = None; + } + +let show_source (source : ImportDataTypes.Table.t) = + Printf.sprintf "%s:%d" source.ImportDataTypes.Table.file + source.ImportDataTypes.Table.tab + +(* + * Compare two external sources + *) + +let show_sources sources = + let b = Buffer.create 16 in + Buffer.add_string b "["; + List.iter sources ~f:(fun source -> + Buffer.add_string b (show_source source); + Buffer.add_string b ","); + + let len = Buffer.length b in + if len > 1 then Buffer.truncate b (len - 1); + Buffer.add_string b "]"; + + Buffer.contents b + +and cmp_source : ImportDataTypes.Table.t -> ImportDataTypes.Table.t -> bool = + fun s1 s2 -> + String.equal s1.ImportDataTypes.Table.name s2.ImportDataTypes.Table.name + && String.equal s1.ImportDataTypes.Table.file s2.ImportDataTypes.Table.file + && s1.ImportDataTypes.Table.tab = s2.ImportDataTypes.Table.tab + +let cmp_list : ('a -> 'a -> bool) -> 'a list -> 'a list -> bool = + fun cmp elems1 elems2 -> List.for_all2 ~f:cmp elems1 elems2 + +(* + * Compare keys in the dependencies + *) + +let key_printer : ImportAnalyser.Dependency.key -> string = + fun { name; expression; _ } -> + let path_name = + let buffer = Buffer.create 16 in + ImportExpression.Headers.headers_of_expression buffer + (fun col buffer -> + Buffer.add_string buffer (ImportCSV.Csv.column_to_string col)) + expression; + Buffer.contents buffer + in + Printf.sprintf "%s, %s" name path_name + +and key_cmp a b = + 0 + = ImportExpression.T.cmp + (fun a b -> a - b) + a.ImportAnalyser.Dependency.expression + b.ImportAnalyser.Dependency.expression + +let keys_printer : ImportAnalyser.Dependency.key list -> string = + fun contents -> + let b = Buffer.create 16 in + List.iter contents ~f:(fun v -> Buffer.add_string b (key_printer v)); + Buffer.contents b + +(* + * Represents externals + *) + +let pp_externals : ImportConf.Syntax.extern list -> string = + fun ext -> ImportConf.Syntax.toml_of_externs ext |> Otoml.Printer.to_string diff --git a/tests/configuration/example_csv.toml b/tests/configuration/example_csv.toml new file mode 100644 index 0000000..24cee9b --- /dev/null +++ b/tests/configuration/example_csv.toml @@ -0,0 +1,31 @@ +version = 1 + +[source] + file = "importer.csv" + name = "source" + +[externals.target] + intern_key = ":source.A" + extern_key = ":A" + file = "financial.xlsx" + allow_missing = false + +[externals.a_financial] + intern_key = ":target.A" + extern_key = ":O" + file = "financial.xlsx" + allow_missing = false + +[sheet] + columns = [":A", + "concat(\"-\", :A, :target.E, :B)", + ":C", + "counter([:C],[:A])", + ":E", + "match(\"\\(..\\)\", :B)", + ":D", + "counter([:D],[:A])" + ] + filters = [] + sort = [] + uniq = [] diff --git a/tests/configuration/simple.toml b/tests/configuration/simple.toml new file mode 100644 index 0000000..d41383a --- /dev/null +++ b/tests/configuration/simple.toml @@ -0,0 +1,20 @@ +version = 1 + +[source] +name = "source_name" +file = "source_file" +tab = 1 + +[externals.target] +extern_key = "\"_B\"" +# Here, the values A & B are considered as column, and not litteral +intern_key = "function(:A, :B)" +allow_missing = true +file = "" +tab = 1 + +[sheet] +columns = [ + "function(:target.A, :B, 'free\\' text')", + "counter([:target.A],[:target.A])" +] diff --git a/tests/configuration_expression.ml b/tests/configuration_expression.ml new file mode 100644 index 0000000..a5c4755 --- /dev/null +++ b/tests/configuration_expression.ml @@ -0,0 +1,253 @@ +open StdLabels +open OUnit2 +module Expression = ImportExpression.T +module Path = ImportDataTypes.Path +open Path + +let printer = function + | Ok e -> ImportExpression.Repr.repr ImportConf.Path.repr e + | Error msg -> msg + +let parse_dquoted = + "parse_dquoted" >:: fun _ -> + let expr = "match(\"\\(..\\)\", :B)" in + let result = ImportConf.expression_from_string expr in + assert_equal ~printer + (Ok + (Function + ("match", [ Literal "\\(..\\)"; Path { alias = None; column = 2 } ]))) + result + +let parse_quoted = + "parse_quoted" >:: fun _ -> + let expr = "match('\\(..\\)', :B)" in + let result = ImportConf.expression_from_string expr in + assert_equal ~printer + (Ok + (Function + ("match", [ Literal "\\(..\\)"; Path { alias = None; column = 2 } ]))) + result + +let concat = + "concat" >:: fun _ -> + let expr = ":A ^ :B" in + let result = ImportConf.expression_from_string expr in + assert_equal ~printer + (Ok + (Concat + [ + Path { alias = None; column = 1 }; Path { alias = None; column = 2 }; + ])) + result + +let concat2 = + "concat2" >:: fun _ -> + let expr = "'A' ^ '_' ^ 'B'" in + let result = ImportConf.expression_from_string expr in + assert_equal ~printer + (Ok (Concat [ Literal "A"; Literal "_"; Literal "B" ])) + result + +let litteral = + "litteral" >:: fun _ -> + (* The text is quoted in shall not be considered as a path *) + let expr = "':A'" in + let result = ImportConf.expression_from_string expr in + assert_equal ~printer (Ok (Literal ":A")) result + +let empty = + "empty" >:: fun _ -> + let expr = "" in + let result = ImportConf.expression_from_string expr in + assert_equal ~printer (Ok Empty) result + +let upper_nvl = + "upper_nvl" >:: fun _ -> + let expr = "NVL('','')" in + let result = ImportConf.expression_from_string expr in + assert_equal ~printer (Ok (Nvl [ Empty; Empty ])) result + +let lower_nvl = + "lower_nvl" >:: fun _ -> + let expr = "nvl('','')" in + let result = ImportConf.expression_from_string expr in + assert_equal ~printer (Ok (Nvl [ Empty; Empty ])) result + +let numeric = + "numeric" >:: fun _ -> + let expr = "123" in + let result = ImportConf.expression_from_string expr in + assert_equal ~printer (Ok (Integer "123")) result + +let numeric_neg = + "numeric_neg" >:: fun _ -> + let expr = "-123" in + let result = ImportConf.expression_from_string expr in + assert_equal ~printer (Ok (Integer "-123")) result + +let op_priority = + "operator_priority" >:: fun _ -> + let expr = "1 + 2 > 2" in + let result = ImportConf.expression_from_string expr + and expected = + ImportExpression.T.( + BOperator (GT, BOperator (Add, Integer "1", Integer "2"), Integer "2")) + in + + assert_equal ~printer (Ok expected) result + +let op_priority2 = + "operator_priority" >:: fun _ -> + let expr = "1 ^ 2 = 2" in + let result = ImportConf.expression_from_string expr + and expected = + ImportExpression.T.( + BOperator (Equal, Concat [ Integer "1"; Integer "2" ], Integer "2")) + in + + assert_equal ~printer (Ok expected) result + +let join = + "join" >:: fun _ -> + let expr = "join('sep', :A, :B)" in + let result = ImportConf.expression_from_string expr in + assert_equal ~printer + (Ok + (Join + ( "sep", + [ + Path { alias = None; column = 1 }; + Path { alias = None; column = 2 }; + ] ))) + result + +let join_empty = + "join" >:: fun _ -> + let expr = "join('', :A, :B)" in + let result = ImportConf.expression_from_string expr in + assert_equal ~printer + (Ok + (Join + ( "", + [ + Path { alias = None; column = 1 }; + Path { alias = None; column = 2 }; + ] ))) + result + +let upper = + "upper" >:: fun _ -> + let expr = "upper('')" in + let result = ImportConf.expression_from_string expr in + assert_equal ~printer (Ok (Function' (Upper, [ Empty ]))) result + +let trim = + "trim" >:: fun _ -> + let expr = "trim('')" in + let result = ImportConf.expression_from_string expr in + assert_equal ~printer (Ok (Function' (Trim, [ Empty ]))) result + +(** Extract the columns from a window function *) +let fold_values = + "fold_values" >:: fun _ -> + (* The expression we want to test *) + let expr = + Expression.Window + ( Previous (Path { alias = None; column = 1 }), + [ Path { alias = None; column = 2 } ], + [ Path { alias = None; column = 3 } ] ) + in + + (* Extract the columns from the expression. The result is sorted because + the order is not preserved during the extraction. *) + let result = + Expression.fold_values ~init:[] ~f:(fun acc v -> v :: acc) expr + |> List.sort ~cmp:Path.compare + in + + let expected = + [ + { alias = None; column = 1 }; + { alias = None; column = 2 }; + { alias = None; column = 3 }; + ] + in + + assert_equal expected result + +let bad_quote = + "bad_quote" >:: fun _ -> + let expr = "':source.A" in + let result = ImportConf.expression_from_string expr in + + assert_equal ~printer (Error "Unclosed quote at line 1 : \"':source.A\"") + result + +let nested_expression = + "nested_expression" >:: fun _ -> + let expr = "1 = (1 = 0)" in + let result = ImportConf.expression_from_string expr in + assert_equal ~printer + (Ok + (BOperator + ( Equal, + Integer "1", + Expr (BOperator (Equal, Integer "1", Integer "0")) ))) + result + +let priority_equality = + "priority_equality" >:: fun _ -> + let expr = "1 = 1 = 0" in + let result = ImportConf.expression_from_string expr in + assert_equal ~printer + (Ok + (BOperator + (Equal, Integer "1", BOperator (Equal, Integer "1", Integer "0")))) + result + +let priority_operator_and = + "priority_equality" >:: fun _ -> + let expr = "1 and 1 = 0" in + let result = ImportConf.expression_from_string expr in + assert_equal ~printer + (Ok + (BOperator (And, Integer "1", BOperator (Equal, Integer "1", Integer "0")))) + result + +let priority_operator_or = + "priority_equality" >:: fun _ -> + let expr = "1 <> 1 or 0" in + let result = ImportConf.expression_from_string expr in + assert_equal ~printer + (Ok + (BOperator + (Or, BOperator (Different, Integer "1", Integer "1"), Integer "0"))) + result + +let test_suit = + [ + parse_dquoted; + parse_quoted; + concat; + concat2; + litteral; + empty; + upper_nvl; + lower_nvl; + numeric; + numeric_neg; + op_priority; + op_priority2; + join; + upper; + trim; + join_empty; + fold_values; + bad_quote; + nested_expression; + priority_equality; + priority_operator_and; + priority_operator_or; + ] + +let tests = "configuration_expression" >::: test_suit diff --git a/tests/configuration_toml.ml b/tests/configuration_toml.ml new file mode 100644 index 0000000..3c8bfc2 --- /dev/null +++ b/tests/configuration_toml.ml @@ -0,0 +1,71 @@ +open OUnit2 +module Expression = ImportExpression.T +module Path = ImportDataTypes.Path + +let test_suit = + [ + ( "parse_extern" >:: fun _ -> + let toml = Otoml.Parser.from_file "configuration/simple.toml" in + let toml = ImportConf.t_of_toml toml in + match toml with + | Error s -> raise (Failure s) + | Ok result -> + let open ImportConf.Syntax in + let expected = + { + target = { file = ""; tab = 1; name = "target" }; + extern_key = Literal "_B"; + intern_key = + Function + ( "function", + [ + Path { alias = None; column = 1 }; + Path { alias = None; column = 2 }; + ] ); + match_rule = None; + allow_missing = true; + } + in + + let printer s = + String.concat "," (List.map ImportConf.print_extern s) + in + + assert_equal ~printer [ expected ] result.externals ); + ( "parse_columns" >:: fun _ -> + let toml = Otoml.Parser.from_file "configuration/simple.toml" in + let toml = ImportConf.t_of_toml toml in + + match toml with + | Error s -> raise (Failure s) + | Ok result -> + let open Path in + let open Expression in + let expected = + [ + Function + ( "function", + [ + Path { alias = Some "target"; column = 1 }; + Path { alias = None; column = 2 }; + Literal "free' text"; + ] ); + Window + ( Counter, + [ Path { alias = Some "target"; column = 1 } ], + [ Path { alias = Some "target"; column = 1 } ] ); + ] + in + + List.iter2 + (fun expected result -> + assert_equal ~printer:ImportConf.print_path_expression expected + result) + expected result.columns ); + ( "parse_csv" >:: fun _ -> + let toml = Otoml.Parser.from_file "configuration/example_csv.toml" in + let toml = ImportConf.t_of_toml toml in + ignore toml ); + ] + +let tests = "configuration_toml" >::: test_suit diff --git a/tests/dune b/tests/dune new file mode 100644 index 0000000..c777aef --- /dev/null +++ b/tests/dune @@ -0,0 +1,17 @@ +(test + (name importer_test) + (deps (source_tree configuration)) + (libraries + ounit2 + otoml + ppx_deriving.runtime + sqlite3 + importConf + importAnalyser + importContainers + importCSV + importDataTypes + importErrors + importExpression + importSQL +)) diff --git a/tests/expression_builder.ml b/tests/expression_builder.ml new file mode 100644 index 0000000..fd9a17f --- /dev/null +++ b/tests/expression_builder.ml @@ -0,0 +1,47 @@ +module T = ImportExpression.T + +let empty : 'a T.t = T.Empty +let path : 'a -> 'a T.t = fun v -> T.Path v +let literal_test : 'a T.t = T.Literal "test" +let literal_quoted : 'a T.t = T.Literal "'" +let literal_zero : 'a T.t = T.Literal "0" +let integer_zero : 'a T.t = T.Integer "0" +let integer_one : 'a T.t = T.Integer "1" +let concat : 'a T.t = T.Concat [ T.Empty; T.Literal "test" ] +let expr : 'a T.t = T.Function ("expr", [ literal_test; T.Literal "NOT NULL" ]) + +let equal : 'a T.t -> 'a T.t -> 'a T.t = + fun e1 e2 -> T.BOperator (T.Equal, e1, e2) + +let different : 'a T.t -> 'a T.t -> 'a T.t = + fun e1 e2 -> T.BOperator (T.Different, e1, e2) + +let divide : 'a T.t -> 'a T.t -> 'a T.t = + fun e1 e2 -> T.BOperator (T.Division, e1, e2) + +let nvl : 'a T.t -> 'a T.t -> 'a T.t = fun e1 e2 -> T.Nvl [ e1; e2 ] + +let if_ : 'a T.t -> 'a T.t -> 'a T.t -> 'a T.t = + fun pred e1 e2 -> T.Function ("if", [ pred; e1; e2 ]) + +let in_ : 'a T.t -> 'a T.t list -> 'a T.t = + fun e1 group -> T.GEquality (T.Equal, e1, group) + +let not_in : 'a T.t -> 'a T.t list -> 'a T.t = + fun e1 group -> T.GEquality (T.Different, e1, group) + +let max : 'a T.t -> 'a T.t list -> 'a T.t list -> 'a T.t = + fun e group order -> T.Window (T.Max e, group, order) + +let counter : 'a T.t list -> 'a T.t list -> 'a T.t = + fun group order -> T.Window (T.Counter, group, order) + +let function' : T.funct -> 'a T.t list -> 'a T.t = + fun name param -> T.Function' (name, param) + +module Make (Sym : ImportExpression.Sym.SYM_EXPR) = struct + module M = ImportExpression.Sym.M (Sym) + + let eval : 'a T.t -> path_repr:'b Sym.path_repr -> 'a Sym.repr = + fun v ~path_repr -> M.eval ~path_repr v +end diff --git a/tests/expression_query.ml b/tests/expression_query.ml new file mode 100644 index 0000000..d260a76 --- /dev/null +++ b/tests/expression_query.ml @@ -0,0 +1,208 @@ +open OUnit2 +module T = ImportExpression.T +module Expr = Expression_builder +module M = Expr.Make (ImportExpression.Query.Query) + +let eval = + M.eval ~path_repr:(fun formatter n -> Format.fprintf formatter "%s" n) + +let printer = Fun.id + +let test_expr ?(nested = ImportExpression.Query.QueryParameter.Literal) expr = + let buffer = Buffer.create 16 in + let formatter = Format.formatter_of_buffer buffer in + let () = ImportExpression.Query.Query.observe expr formatter ~nested in + Format.pp_print_flush formatter (); + Buffer.contents buffer + +let empty = + "empty" >:: fun _ -> + let expr = eval Expr.empty in + let content = test_expr expr and expected = "''" in + + assert_equal ~printer expected content + +let litteral = + "literal" >:: fun _ -> + let expr = eval Expr.literal_test in + let content = test_expr expr and expected = "'test'" in + + assert_equal ~printer expected content + +let litteral_quoted = + "literal_quoted" >:: fun _ -> + let expr = eval Expr.literal_quoted in + let content = test_expr expr and expected = "'\''" in + + assert_equal ~printer expected content + +let litteral_raw = + "literal_raw" >:: fun _ -> + let expr = eval Expr.literal_test in + let nested = ImportExpression.Query.QueryParameter.(Raw Literal) in + let content = test_expr expr ~nested and expected = "test" in + + assert_equal ~printer expected content + +let path = + "path" >:: fun _ -> + (* In the path, the given function do all the job *) + let expr = eval @@ Expr.path "test" in + let content = test_expr expr and expected = "test" in + + assert_equal ~printer expected content + +let concat = + "concat" >:: fun _ -> + let expr = eval Expr.concat in + let content = test_expr expr and expected = "'' || 'test'" in + + assert_equal ~printer expected content + +let nvl = + "nvl" >:: fun _ -> + let expr = eval @@ Expr.nvl Expr.empty Expr.literal_test in + let content = test_expr expr and expected = "COALESCE('', 'test')" in + + assert_equal ~printer expected content + +let upper = + "upper" >:: fun _ -> + let expr = eval @@ Expr.function' T.Upper [ Expr.literal_test ] in + let content = test_expr expr and expected = "UPPER('test')" in + + assert_equal ~printer expected content + +let join = + "join" >:: fun _ -> + let expr = + ImportExpression.Query.Query.( + join "," [ eval Expr.empty; eval Expr.literal_test ]) + in + let content = test_expr expr and expected = "CONCAT(',', '', 'test')" in + + assert_equal ~printer expected content + +let boperator_eq = + "boperator_eq" >:: fun _ -> + let expr = eval @@ Expr.equal Expr.empty Expr.literal_test in + let content = test_expr expr and expected = "''='test'" in + + assert_equal ~printer expected content + +let boperator_div = + "boperator_div" >:: fun _ -> + let expr = eval @@ Expr.divide Expr.integer_one Expr.integer_zero in + let content = test_expr expr and expected = "CAST(1 AS REAL)/0" in + + assert_equal ~printer expected content + +let boperator_neq = + "boperator_neq" >:: fun _ -> + let expr = eval @@ Expr.different Expr.empty Expr.literal_test in + let content = test_expr expr and expected = "''<>'test'" in + + assert_equal ~printer expected content + +let expr = + "expr" >:: fun _ -> + let expr = eval Expr.expr in + let content = test_expr expr and expected = "(test NOT NULL)" in + + assert_equal ~printer expected content + +let unify_int = + "unify_int" >:: fun _ -> + let expr = eval @@ Expr.equal (Expr.path "external") Expr.integer_zero in + let content = test_expr expr and expected = "COALESCE(external,0)=0" in + + assert_equal ~printer expected content + +let unify_string = + "unify_string" >:: fun _ -> + let expr = eval @@ Expr.equal (Expr.path "external") Expr.literal_zero in + let content = test_expr expr and expected = "COALESCE(external,'')='0'" in + + assert_equal ~printer expected content + +let in_string = + "in_string" >:: fun _ -> + let expr = eval @@ Expr.in_ (Expr.path "external") [ Expr.literal_zero ] in + let content = test_expr expr and expected = "COALESCE(external,'') IN('0')" in + assert_equal ~printer expected content + +let not_in_string = + "in_string" >:: fun _ -> + let expr = eval @@ Expr.not_in (Expr.path "external") [ Expr.literal_zero ] in + let content = test_expr expr + and expected = "COALESCE(external,'') NOT IN('0')" in + assert_equal ~printer expected content + +(* Evaluate the max function *) +let max = + "max" >:: fun _ -> + let expr = + eval @@ Expr.(max (path ":C") [ path ":A" ] [ path ":A"; path ":B" ]) + in + + let content = test_expr expr + and expected = + "LAST_VALUE(:C) OVER (PARTITION BY :A ORDER BY :A, :B RANGE BETWEEN \ + UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)" + in + + assert_equal ~printer expected content + +let in_int = + "in_int" >:: fun _ -> + let expr = + eval + @@ Expr.in_ (Expr.path "external") [ Expr.integer_zero; Expr.integer_one ] + in + let content = test_expr expr and expected = "COALESCE(external,0) IN(0, 1)" in + assert_equal ~printer expected content + +let counter_no_order = + "counter_no_order" >:: fun _ -> + let expr = eval @@ Expr.(counter [ path ":A" ] []) in + + let content = test_expr expr + and expected = "COUNT() OVER (PARTITION BY :A)" in + + assert_equal ~printer expected content + +let counter_order = + "counter_no_order" >:: fun _ -> + let expr = eval @@ Expr.(counter [ path ":A" ] [ path ":B" ]) in + + let content = test_expr expr + and expected = "ROW_NUMBER() OVER (PARTITION BY :A ORDER BY :B)" in + + assert_equal ~printer expected content + +let test_suit = + [ + empty; + litteral; + litteral_quoted; + litteral_raw; + path; + concat; + nvl; + upper; + join; + boperator_eq; + boperator_neq; + boperator_div; + expr; + unify_int; + unify_string; + in_string; + not_in_string; + in_int; + max; + counter_no_order; + counter_order; + ] + +let tests = "expression_query" >::: test_suit diff --git a/tests/expression_repr.ml b/tests/expression_repr.ml new file mode 100644 index 0000000..9ad321a --- /dev/null +++ b/tests/expression_repr.ml @@ -0,0 +1,37 @@ +open OUnit2 +module Expression = ImportExpression.T +module Path = ImportDataTypes.Path +module Expr = Expression_builder +module M = Expr.Make (ImportExpression.Repr.E) + +let eval = M.eval ~path_repr:ImportCSV.Csv.column_to_string +let test_expr expr = ImportExpression.Repr.E.observe ~top:true expr + +let printer = function + | Ok e -> ImportConf.print_path_expression e + | Error msg -> msg + +let print_literal = + "print_litteral" >:: fun _ -> + let result = test_expr @@ eval (Literal "Content") in + assert_equal ~printer:Fun.id "'Content'" result + +let print_quoted_literal = + "print_quoted_literal" >:: fun _ -> + let result = test_expr @@ eval Expression_builder.literal_quoted in + assert_equal ~printer:Fun.id "'\\''" result + +let print_dquoted_literal = + "print_dquoted_literal" >:: fun _ -> + let result = test_expr @@ eval (Literal "\"") in + assert_equal ~printer:Fun.id "'\"'" result + +let print_numeric = + "print_numeric" >:: fun _ -> + let result = test_expr @@ eval (Literal "123") in + assert_equal ~printer:Fun.id "123" result + +let test_suit = + [ print_literal; print_quoted_literal; print_dquoted_literal; print_numeric ] + +let tests = "expression_repr" >::: test_suit diff --git a/tests/expression_type_of.ml b/tests/expression_type_of.ml new file mode 100644 index 0000000..706b3e7 --- /dev/null +++ b/tests/expression_type_of.ml @@ -0,0 +1,65 @@ +open OUnit2 +module T = ImportExpression.T +module Types = ImportDataTypes.Types +module Expr = Expression_builder +module M = Expr.Make (ImportExpression.Type_of) + +let eval = M.eval ~path_repr:(fun _ -> ()) +let printer = Types.string_of_t +let test_expr expr = ImportExpression.Type_of.observe expr + +let empty = + "empty" >:: fun _ -> + let expr = eval Expr.empty in + let content = test_expr expr and expected = Types.None in + + assert_equal ~printer expected content + +(** Control an if statement with a predicate which is not a boolean *) +let invalid_if = + "invalid_if" >:: fun _ -> + (* The expression we want to test *) + let raw_expr = + Expr.if_ Expr.literal_test Expr.integer_zero Expr.integer_one + in + + let exn = + ImportErrors.TypeError + { + expected = Types.Bool; + actual = Types.String; + expression = ImportExpression.Repr.repr Fun.id raw_expr; + subset = "the predicate"; + } + in + assert_raises exn (fun () -> + let expr = eval raw_expr in + test_expr expr) + +(** The application should infer that the expression is a string *) +let valid_if = + "valid_if" >:: fun _ -> + let expr = + eval + @@ Expr.if_ + (Expr.equal Expr.integer_one Expr.integer_zero) + Expr.literal_test Expr.literal_test + in + let content = test_expr expr and expected = Types.String in + assert_equal ~printer expected content + +let upper = + "upper" >:: fun _ -> + let expr = eval @@ Expr.function' T.Upper [Expr.literal_test] in + + let content = test_expr expr and expected = Types.String in + assert_equal ~printer expected content + +let in_int = + "in_int" >:: fun _ -> + let expr = eval @@ Expr.in_ (Expr.path "external") [ Expr.integer_one ] in + let content = test_expr expr and expected = Types.Bool in + assert_equal ~printer expected content + +let tests = "expression_type_of" >::: [ empty; invalid_if; valid_if; in_int ; +upper ] diff --git a/tests/importCSV_test.ml b/tests/importCSV_test.ml new file mode 100644 index 0000000..bc21992 --- /dev/null +++ b/tests/importCSV_test.ml @@ -0,0 +1,29 @@ +open OUnit2 +open ImportCSV + +let test_suit = + [ + ( "Column A" >:: fun _ -> + assert_equal + ~printer:(fun i -> Printf.sprintf "%d (%s)" i (Csv.column_to_string i)) + 1 (Csv.column_of_string "A") ); + ( "Column a" >:: fun _ -> + assert_equal + ~printer:(fun i -> Printf.sprintf "%d (%s)" i (Csv.column_to_string i)) + 1 (Csv.column_of_string "a") ); + ( "Column name" >:: fun _ -> + let () = + for i = 1 to 1_000 do + let column_name = Csv.column_to_string i in + let column_index = Csv.column_of_string column_name in + + assert_equal + ~printer:(fun i -> + Printf.sprintf "%d (%s)" i (Csv.column_to_string i)) + i column_index + done + in + () ); + ] + +let tests = "importCSV_test" >::: test_suit diff --git a/tests/importConf_test.ml b/tests/importConf_test.ml new file mode 100644 index 0000000..c94eb91 --- /dev/null +++ b/tests/importConf_test.ml @@ -0,0 +1,23 @@ +open OUnit2 +open ConfLoader + +(** Test the dependencies extracted from the external named "source". + + Refer to the default configuration used in [ConfLoader] to see the + configuration. + *) +let test_get_dependencies_for_source = + "get_dependancies_for_table" >:: fun _ -> + let result = ImportConf.get_dependancies_for_table conf conf.source + and expected = [ external_other ] in + assert_equal ~printer:pp_externals expected result + +let test_get_dependencies_for_other = + "get_dependancies_for_table" >:: fun _ -> + let result = ImportConf.get_dependancies_for_table conf external_table_other + and expected = [ external_last ] in + assert_equal ~printer:pp_externals expected result + +let tests = + "importConf_test" + >::: [ test_get_dependencies_for_source; test_get_dependencies_for_other ] diff --git a/tests/importer_test.ml b/tests/importer_test.ml new file mode 100644 index 0000000..16ea663 --- /dev/null +++ b/tests/importer_test.ml @@ -0,0 +1,21 @@ +open OUnit2 + +let _ = + run_test_tt_main + ("importer_tests" + >::: [ + ImportCSV_test.tests; + Sql_date.tests; + Sql_match.tests; + Sql_int.tests; + Sql_trim.tests; + ImportConf_test.tests; + Configuration_toml.tests; + Configuration_expression.tests; + Expression_repr.tests; + Expression_type_of.tests; + Expression_query.tests; + Analyser_dependency.tests; + Analyser_query_test.tests; + Sql_db.tests; + ]) diff --git a/tests/sql_date.ml b/tests/sql_date.ml new file mode 100644 index 0000000..4becdf5 --- /dev/null +++ b/tests/sql_date.ml @@ -0,0 +1,18 @@ +open OUnit2 + +let test_suit = + [ + ( "Parse date" >:: fun _ -> + let text_date = Sqlite3.Data.TEXT "2002-04-08 15:59:41.000" + and format_date = Sqlite3.Data.TEXT "%Y-%m-%d %H:%M:%S.000" in + + assert_equal (Sqlite3.Data.INT 37354L) + (ImportSQL.Date.f format_date text_date) ); + ( "Parse date as int" >:: fun _ -> + let int_date = Sqlite3.Data.INT 37354L + and format_date = Sqlite3.Data.TEXT "%Y-%m-%d %H:%M:%S.000" in + + assert_equal int_date (ImportSQL.Date.f format_date int_date) ); + ] + +let tests = "sql_date" >::: test_suit diff --git a/tests/sql_db.ml b/tests/sql_db.ml new file mode 100644 index 0000000..c966f4e --- /dev/null +++ b/tests/sql_db.ml @@ -0,0 +1,198 @@ +(** Test the behavior of the sqlite with a in-memory database *) + +open OUnit2 +open StdLabels + +let ( let* ) res cont = + match res with + | Ok value -> cont value + | Error e -> raise e + +(** Test a process with a simple configuration in-memory *) +let run_test ~configuration ~input ~expected name = + name >:: fun _ -> + (* We expect a valid configuration *) + let conf = + ImportConf.t_of_toml (Otoml.Parser.from_string configuration) + |> Result.get_ok + in + + let exec db = + let table = List.hd @@ ImportAnalyser.Dependency.get_process_order conf in + let* () = ImportSQL.Db.create_table db table in + + (* Prepare the statement in order to import data *) + let* stmt = ImportSQL.Db.prepare_insert db table in + + (* Inject some data into the table *) + let result, _ = + List.fold_left ~init:(Ok (), 0) input ~f:(fun (_, i) data -> + let result = + let* () = ImportSQL.Db.insert ~id:i db stmt data in + let* () = ImportSQL.Db.reset stmt in + Ok () + in + (result, i + 1)) + in + let* () = result in + + let* () = ImportSQL.Db.finalize stmt in + + let expected = ref expected in + + (* Collect the data *) + let* () = + ImportSQL.Db.query db conf ~f:(fun rows -> + match !expected with + | [] -> () + | hd :: tl -> + expected := tl; + let () = + Array.iter2 rows hd ~f:(fun (_, value) expected -> + assert_equal ~printer:ImportCSV.DataType.to_string value + expected) + in + ()) + in + + Ok () + in + + (* Use a magic keyword for in-memory database *) + ignore @@ ImportSQL.Db.with_db ":memory:" exec + +(** Simple test used to check the process *) +let simple_extraction = + run_test "simple_extraction" + ~configuration: + {|version = 1 + +[source] +name = "source_name" +file = "source_file" + +[sheet] +columns = [ + ":A ^ '_'", + ":B", + ":E"]|} + ~input: + [ + [ + (0, ImportCSV.DataType.Integer 123); + (1, ImportCSV.DataType.Integer 2); + (4, ImportCSV.DataType.Integer 5); + ]; + ] + ~expected: + [ + [| + ImportCSV.DataType.Content "123_"; + ImportCSV.DataType.Integer 2; + ImportCSV.DataType.Integer 5; + |]; + ] + +(** Ensure the behavior of the sum function when a filter is given. It is + expected to accumulate the values over each line *) +let sum_sort = + run_test "sum_sort" + ~configuration: + {|version = 1 + +[source] +name = "source_name" +file = "source_file" + +[sheet] +columns = [ + ":A", + "sum(:C, [:B], [:A])", +]|} + ~input: + [ + [ + (0, ImportCSV.DataType.Integer 1); + (1, ImportCSV.DataType.Content "A"); + (2, ImportCSV.DataType.Integer 100); + ]; + [ + (0, ImportCSV.DataType.Integer 2); + (1, ImportCSV.DataType.Content "A"); + (2, ImportCSV.DataType.Integer 100); + ]; + [ + (0, ImportCSV.DataType.Integer 3); + (1, ImportCSV.DataType.Content "A"); + (2, ImportCSV.DataType.Integer 100); + ]; + ] + ~expected: + [ + [| ImportCSV.DataType.Integer 1; ImportCSV.DataType.Integer 100 |]; + [| ImportCSV.DataType.Integer 2; ImportCSV.DataType.Integer 200 |]; + [| ImportCSV.DataType.Integer 3; ImportCSV.DataType.Integer 300 |]; + ] + +let sum_total = + run_test "sum_total" + ~configuration: + {|version = 1 + +[source] +name = "source_name" +file = "source_file" + +[sheet] +columns = [ + ":A", + "sum(:C, [], [])", +]|} + ~input: + [ + [ + (0, ImportCSV.DataType.Integer 1); (2, ImportCSV.DataType.Integer 100); + ]; + [ + (0, ImportCSV.DataType.Integer 2); (2, ImportCSV.DataType.Integer 100); + ]; + ] + ~expected: + [ + [| ImportCSV.DataType.Integer 1; ImportCSV.DataType.Integer 200 |]; + [| ImportCSV.DataType.Integer 2; ImportCSV.DataType.Integer 200 |]; + ] + +(** Ensure the behavior of the sum function when no filter is given. It is + expected to get the total sum for each line *) +let sum_unfiltered = + run_test "sum_unfiltered" + ~configuration: + {|version = 1 + +[source] +name = "source_name" +file = "source_file" + +[sheet] +columns = [ + ":A", + "sum(:C, [], [:A])", +]|} + ~input: + [ + [ + (0, ImportCSV.DataType.Integer 1); (2, ImportCSV.DataType.Integer 100); + ]; + [ + (0, ImportCSV.DataType.Integer 2); (2, ImportCSV.DataType.Integer 100); + ]; + ] + ~expected: + [ + [| ImportCSV.DataType.Integer 1; ImportCSV.DataType.Integer 200 |]; + [| ImportCSV.DataType.Integer 2; ImportCSV.DataType.Integer 200 |]; + ] + +let test_suit = [ simple_extraction; sum_sort; sum_total; sum_unfiltered ] +let tests = "sql_db" >::: test_suit diff --git a/tests/sql_int.ml b/tests/sql_int.ml new file mode 100644 index 0000000..87b1086 --- /dev/null +++ b/tests/sql_int.ml @@ -0,0 +1,26 @@ +open OUnit2 + +let printer = function + | Sqlite3.Data.INT t -> Int64.to_string t + | Sqlite3.Data.NONE -> "None" + | Sqlite3.Data.NULL -> "Null" + | Sqlite3.Data.FLOAT f -> Float.to_string f + | Sqlite3.Data.TEXT t | Sqlite3.Data.BLOB t -> t + +let test_suit = + [ + ( "Int_of_int" >:: fun _ -> + assert_equal (Sqlite3.Data.INT 37354L) + (ImportSQL.Math.int (Sqlite3.Data.INT 37354L)) ); + ( "Int_of_string" >:: fun _ -> + assert_equal (Sqlite3.Data.INT 37354L) + (ImportSQL.Math.int (Sqlite3.Data.TEXT "37354")) ); + ( "Int_of_string2" >:: fun _ -> + assert_equal ~printer (Sqlite3.Data.INT 37354L) + (ImportSQL.Math.int (Sqlite3.Data.TEXT "37354.0")) ); + ( "Int_of_float" >:: fun _ -> + assert_equal (Sqlite3.Data.INT 37354L) + (ImportSQL.Math.int (Sqlite3.Data.FLOAT 37354.0)) ); + ] + +let tests = "sql_int" >::: test_suit diff --git a/tests/sql_match.ml b/tests/sql_match.ml new file mode 100644 index 0000000..0314bb3 --- /dev/null +++ b/tests/sql_match.ml @@ -0,0 +1,12 @@ +open OUnit2 + +let test_suit = + [ + ( "Parse regex" >:: fun _ -> + let text = Sqlite3.Data.TEXT "hello world" + and regex = Sqlite3.Data.TEXT "hello ([A-Za-z]+)" in + + assert_equal (Sqlite3.Data.TEXT "world") (ImportSQL.Match.f regex text) ); + ] + +let tests = "sql_match" >::: test_suit diff --git a/tests/sql_trim.ml b/tests/sql_trim.ml new file mode 100644 index 0000000..77e755e --- /dev/null +++ b/tests/sql_trim.ml @@ -0,0 +1,11 @@ +open OUnit2 + +let test_suit = + [ + ( "Trim" >:: fun _ -> + let text = Sqlite3.Data.TEXT " \nABC \n" in + + assert_equal (Sqlite3.Data.TEXT "ABC") (ImportSQL.Trim.f text) ); + ] + +let tests = "sql_trim" >::: test_suit -- cgit v1.2.3