Added filters expressions in the externals

5 files changed, 144 insertions, 20 deletions

diff --git a/tests/analyser_dependency.ml b/tests/analyser_dependency.ml
index 511b706..00f21d7 100644
--- a/tests/analyser_dependency.ml
+++ b/tests/analyser_dependency.ml
@@ -40,6 +40,7 @@ let test_keys =
           name = "other";
           expression = Expression.Path 3;
           columns = lazy (Cont.IntSet.singleton 3);
+          filters = [];
         };
       ]
   in
@@ -61,6 +62,7 @@ let test_keys_missing =
           name = "last_file";
           expression = Expression.Path 3;
           columns = lazy (Cont.IntSet.singleton 3);
+          filters = [];
         };
       ]
   in
@@ -145,14 +147,14 @@ let test_unlinked =
               target = { file = "other.xlsx"; tab = 1; name = "circular" };
               extern_key = Path 3;
               allow_missing = true;
-              match_rule = None;
+              filters = [];
             };
             {
               intern_key = Path { alias = Some "circular"; column = 1 };
               target = { file = "other2.xlsx"; tab = 1; name = "circular2" };
               extern_key = Path 3;
               allow_missing = true;
-              match_rule = None;
+              filters = [];
             };
           ];
         columns = [];
@@ -177,7 +179,7 @@ let conf_with_unlinked =
             target = { file = "other.xlsx"; tab = 1; name = "other" };
             extern_key = Path 3;
             allow_missing = false;
-            match_rule = None;
+            filters = [];
           };
         ];
       columns =
diff --git a/tests/analyser_filters.ml b/tests/analyser_filters.ml
index ef51e0c..9a54bde 100644
--- a/tests/analyser_filters.ml
+++ b/tests/analyser_filters.ml
@@ -80,9 +80,9 @@ let group_filter () =
 
   let expected_links = Chunk.create () in
   Chunk.add_string expected_links
-    "WITH filter0 AS (SELECT source.id, LAST_VALUE('source'.'col_1') OVER \
+    "WITH filter0 AS (SELECT source.id, (LAST_VALUE('source'.'col_1') OVER \
      (PARTITION BY 'source'.'col_1' ORDER BY 'source'.'col_1' RANGE BETWEEN \
-     UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)=1 AS group_function\n\
+     UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)=1) AS group_function\n\
      FROM 'source' AS 'source')\n";
 
   Alcotest.(check @@ pair Test_migration.chunk Test_migration.chunk)
@@ -106,9 +106,9 @@ let expression_with_group () =
 
   let expected_links = Chunk.create () in
   Chunk.add_string expected_links
-    "WITH filter0 AS (SELECT source.id, LAST_VALUE('source'.'col_1') OVER \
+    "WITH filter0 AS (SELECT source.id, (LAST_VALUE('source'.'col_1') OVER \
      (PARTITION BY 'source'.'col_1' ORDER BY 'source'.'col_1' RANGE BETWEEN \
-     UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)=1 AS group_function\n\
+     UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)=1) AS group_function\n\
      FROM 'source' AS 'source' WHERE (1=COALESCE('source'.'col_1',0)))\n";
 
   Alcotest.(check @@ pair Test_migration.chunk Test_migration.chunk)
@@ -134,9 +134,9 @@ let group_with_expression () =
 
   let expected_links = Chunk.create () in
   Chunk.add_string expected_links
-    "WITH filter0 AS (SELECT source.id, LAST_VALUE('source'.'col_1') OVER \
+    "WITH filter0 AS (SELECT source.id, (LAST_VALUE('source'.'col_1') OVER \
      (PARTITION BY 'source'.'col_1' ORDER BY 'source'.'col_1' RANGE BETWEEN \
-     UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)=1 AS group_function\n\
+     UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)=1) AS group_function\n\
      FROM 'source' AS 'source')\n";
 
   Alcotest.(check @@ pair Test_migration.chunk Test_migration.chunk)
@@ -161,13 +161,13 @@ let group_with_group () =
 
   let expected_links = Chunk.create () in
   Chunk.add_string expected_links
-    "WITH filter0 AS (SELECT source.id, LAST_VALUE('source'.'col_1') OVER \
+    "WITH filter0 AS (SELECT source.id, (LAST_VALUE('source'.'col_1') OVER \
      (PARTITION BY 'source'.'col_1' ORDER BY 'source'.'col_1' RANGE BETWEEN \
-     UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)=1 AS group_function\n\
+     UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)=1) AS group_function\n\
      FROM 'source' AS 'source')\n\
-     , filter1 AS (SELECT source.id, LAST_VALUE('source'.'col_1') OVER \
+     , filter1 AS (SELECT source.id, (LAST_VALUE('source'.'col_1') OVER \
      (PARTITION BY 'source'.'col_1' ORDER BY 'source'.'col_1' RANGE BETWEEN \
-     UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)=1 AS group_function\n\
+     UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)=1) AS group_function\n\
      FROM 'source' AS 'source'\n\
      INNER JOIN 'filter0' ON filter0.id = source.id\n\
      WHERE filter0.group_function)\n";
diff --git a/tests/analyser_query_test.ml b/tests/analyser_query_test.ml
index ed89623..37a748b 100644
--- a/tests/analyser_query_test.ml
+++ b/tests/analyser_query_test.ml
@@ -46,11 +46,11 @@ let check_externals =
   let query = Q.check_external conf (List.hd conf.externals) in
 
   let expected_query =
-    "SELECT 'source'.'id', 'source'.'col_1'\n\
+    "SELECT 'source'.'id', ('source'.'col_1')\n\
      FROM 'source' AS 'source'\n\
      LEFT JOIN 'other' AS 'other' ON rtrim(upper('source'.'col_1')) = \
      'other'.'key_other' WHERE 'other'.'key_other' IS NULL AND \
-     'source'.'col_1' IS NOT NULL AND 'source'.'col_1' <> ''"
+     ('source'.'col_1') IS NOT NULL AND ('source'.'col_1') <> ''"
   in
 
   Alcotest.check Alcotest.string "" expected_query query.q
@@ -179,6 +179,7 @@ let prepare_insert =
         name = "key_test";
         expression = Concat [ Path 1; Literal "_"; Empty ];
         columns = lazy (ImportContainers.IntSet.singleton 1);
+        filters = [];
       }
   in
 
@@ -207,7 +208,7 @@ let filter_group =
   let contents, _ = ImportAnalyser.Query.select conf in
 
   let expected =
-    {|WITH filter0 AS (SELECT source.id, LAST_VALUE('source'.'col_3') OVER (PARTITION BY 'source'.'col_1' ORDER BY 'source'.'col_1' RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS group_function
+    {|WITH filter0 AS (SELECT source.id, (LAST_VALUE('source'.'col_3') OVER (PARTITION BY 'source'.'col_1' ORDER BY 'source'.'col_1' RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) AS group_function
 FROM 'source' AS 'source'
 LEFT JOIN 'other' AS 'other' ON rtrim(upper('source'.'col_1')) = 'other'.'key_other'
 LEFT JOIN 'last' AS 'last_file' ON rtrim(upper('other'.'col_1')) = 'last_file'.'key_last_file')
@@ -239,7 +240,7 @@ let filter_group2 =
   let contents, _ = ImportAnalyser.Query.select conf in
 
   let expected =
-    {|WITH filter0 AS (SELECT source.id, LAST_VALUE('source'.'col_3') OVER (PARTITION BY 'source'.'col_1' ORDER BY 'source'.'col_1' RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS group_function
+    {|WITH filter0 AS (SELECT source.id, (LAST_VALUE('source'.'col_3') OVER (PARTITION BY 'source'.'col_1' ORDER BY 'source'.'col_1' RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) AS group_function
 FROM 'source' AS 'source'
 LEFT JOIN 'other' AS 'other' ON rtrim(upper('source'.'col_1')) = 'other'.'key_other'
 LEFT JOIN 'last' AS 'last_file' ON rtrim(upper('other'.'col_1')) = 'last_file'.'key_last_file')
@@ -253,6 +254,89 @@ WHERE (COALESCE('source'.'col_3',0)=0) AND filter0.group_function|}
 
   Alcotest.check Alcotest.string "" expected contents.q
 
+(** Add an external with a filter. Ensure the predicate is reported in the
+    query. *)
+let external_filter =
+  "external_filter" >:: fun _ ->
+  let conf =
+    Syntax.
+      {
+        ConfLoader.conf with
+        externals =
+          [
+            {
+              ConfLoader.external_other with
+              filters = [ Expression_builder.(equal (path 1) integer_one) ];
+            };
+          ];
+        columns = [ Expression_builder.empty ];
+      }
+  in
+
+  let query, _ = ImportAnalyser.Query.select conf in
+  let expected_query =
+    {|SELECT '' AS result_0
+FROM 'source' AS 'source'
+LEFT JOIN 'other' AS 'other' ON rtrim(upper('source'.'col_1')) = 'other'.'key_other' AND (COALESCE('other'.'col_1',0)=1)|}
+  in
+  Alcotest.check Test_migration.trimed_string "" expected_query query.q
+
+let order_by =
+  "order_by" >:: fun () ->
+  let conf =
+    Syntax.
+      {
+        ConfLoader.conf with
+        externals = [];
+        columns =
+          [
+            Expression_builder.path
+              ImportDataTypes.Path.{ alias = None; column = 1 };
+          ];
+        sort =
+          [
+            Expression_builder.path
+              ImportDataTypes.Path.{ alias = None; column = 1 };
+            Expression_builder.integer_one;
+          ];
+      }
+  in
+  let query, _ = ImportAnalyser.Query.select conf in
+  let expected_query =
+    {|SELECT 'source'.'col_1' AS result_0
+FROM 'source' AS 'source'
+ORDER BY ('source'.'col_1'), (1)|}
+  in
+  Alcotest.check Test_migration.trimed_string "" expected_query query.q
+
+let group_by =
+  "order_by" >:: fun () ->
+  let conf =
+    Syntax.
+      {
+        ConfLoader.conf with
+        externals = [];
+        columns =
+          [
+            Expression_builder.path
+              ImportDataTypes.Path.{ alias = None; column = 1 };
+          ];
+        uniq =
+          [
+            Expression_builder.path
+              ImportDataTypes.Path.{ alias = None; column = 1 };
+            Expression_builder.integer_one;
+          ];
+      }
+  in
+  let query, _ = ImportAnalyser.Query.select conf in
+  let expected_query =
+    {|SELECT 'source'.'col_1' AS result_0
+FROM 'source' AS 'source'
+GROUP BY ('source'.'col_1'), (1)|}
+  in
+  Alcotest.check Test_migration.trimed_string "" expected_query query.q
+
 let test_suit =
   [
     create_table;
@@ -265,6 +349,9 @@ let test_suit =
     prepare_insert;
     filter_group;
     filter_group2;
+    external_filter;
+    order_by;
+    group_by;
   ]
 
 let tests = "analyser_query_test" >::: test_suit
diff --git a/tests/confLoader.ml b/tests/confLoader.ml
index e6187c3..b0be690 100644
--- a/tests/confLoader.ml
+++ b/tests/confLoader.ml
@@ -61,7 +61,7 @@ let external_other =
       target = external_table_other;
       extern_key = Path 3;
       allow_missing = false;
-      match_rule = None;
+      filters = [];
     }
 
 let external_table_last =
@@ -74,5 +74,5 @@ let external_last =
       target = external_table_last;
       extern_key = Path 3;
       allow_missing = true;
-      match_rule = None;
+      filters = [];
     }
diff --git a/tests/configuration_toml.ml b/tests/configuration_toml.ml
index 470af4a..620a106 100644
--- a/tests/configuration_toml.ml
+++ b/tests/configuration_toml.ml
@@ -303,6 +303,40 @@ columns = []|}
   Alcotest.(check (result Test_migration.syntax string))
     "Dataset with alias" expected configuration
 
+let external_filters () =
+  let configuration =
+    ConfLoader.load'
+      {|[source]
+name = ""
+file = ""
+tab = 0
+
+[externals.other]
+  intern_key = ":A"
+  file = "other.xlsx"
+  extern_key = ":C"
+  filters = [":B = 1"]
+
+
+[sheet]
+columns = []|}
+  and expected =
+    Ok
+      {
+        ImporterSyntax.dummy_conf with
+        externals =
+          ConfLoader.
+            [
+              {
+                external_other with
+                filters = [ Expression_builder.(equal (path 2) integer_one) ];
+              };
+            ];
+      }
+  in
+  Alcotest.(check (result Test_migration.syntax string))
+    "Filters in external" expected configuration
+
 let test_suit =
   [
     ( "parse_extern" >:: fun _ ->
@@ -322,7 +356,7 @@ let test_suit =
                         Path { alias = None; column = 1 };
                         Path { alias = None; column = 2 };
                       ] );
-                match_rule = None;
+                filters = [];
                 allow_missing = true;
               }
           in
@@ -372,6 +406,7 @@ let test_suit =
     ("Empty dataset", `Quick, empty_dataset);
     ("Dataset with invalid key", `Quick, dataset_with_invalid_key);
     ("External dataset", `Quick, external_dataset);
+    ("External with filter", `Quick, external_filters);
   ]
 
 let tests = "configuration_toml" >::: test_suit