aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSébastien Dailly <sebastien@dailly.me>2025-03-17 09:11:25 +0100
committerSébastien Dailly <sebastien@dailly.me>2025-03-17 18:59:32 +0100
commit8b8b730d3ba98d6c9e4e6274844641043b5fefbb (patch)
tree4cb60dafa05b479d0ca287d501a51db88cecaaa4
parent7bfbb67d83011f3e1845dcb9e44c3b6a5e93a9da (diff)
Moved the syntax module in its own library
-rwxr-xr-xbin/dune4
-rw-r--r--bin/importer.ml33
-rw-r--r--lib/analysers/chunk.ml21
-rw-r--r--lib/analysers/chunk.mli8
-rw-r--r--lib/analysers/dependency.ml36
-rw-r--r--lib/analysers/dependency.mli2
-rwxr-xr-xlib/analysers/dune7
-rw-r--r--lib/analysers/filters.ml23
-rw-r--r--lib/analysers/filters.mli2
-rw-r--r--lib/analysers/headers.ml18
-rw-r--r--lib/analysers/headers.mli5
-rw-r--r--lib/analysers/printers.ml5
-rw-r--r--lib/analysers/printers.mli2
-rw-r--r--lib/analysers/query.ml37
-rw-r--r--lib/analysers/query.mli7
-rwxr-xr-xlib/configuration/dune7
-rw-r--r--lib/configuration/expression_parser.mly10
-rw-r--r--lib/configuration/importConf.ml57
-rw-r--r--lib/configuration/importConf.mli16
-rw-r--r--lib/configuration/read_conf.ml4
-rwxr-xr-xlib/containers/dune1
-rw-r--r--lib/containers/importContainers.ml2
-rw-r--r--lib/csv/csv.ml30
-rwxr-xr-xlib/csv/dune7
-rw-r--r--lib/data_types/dune4
-rw-r--r--lib/data_types/format.c (renamed from lib/csv/format.c)0
-rw-r--r--lib/data_types/path.ml33
-rw-r--r--lib/data_types/path.mli12
-rw-r--r--lib/data_types/value.ml (renamed from lib/csv/dataType.ml)0
-rw-r--r--lib/data_types/value.mli (renamed from lib/csv/dataType.mli)0
-rw-r--r--lib/errors/dune1
-rw-r--r--lib/errors/importErrors.ml8
-rw-r--r--lib/errors/importErrors.mli4
-rw-r--r--lib/expression/_readme.rst7
-rwxr-xr-xlib/expression/dune1
-rw-r--r--lib/expression/query.ml6
-rw-r--r--lib/expression/query.mli12
-rw-r--r--lib/file_handler/csv2sql.ml31
-rw-r--r--lib/file_handler/csv2sql.mli8
-rwxr-xr-xlib/file_handler/dune8
-rw-r--r--lib/file_handler/state.ml17
-rw-r--r--lib/file_handler/state.mli7
-rw-r--r--lib/file_handler/xlsx2sql.ml44
-rw-r--r--lib/file_handler/xlsx2sql.mli8
-rw-r--r--lib/sql/db.ml20
-rw-r--r--lib/sql/db.mli25
-rw-r--r--lib/sql/dune7
-rw-r--r--lib/sql/hashs.ml8
-rw-r--r--lib/sql/header.ml5
-rw-r--r--lib/sql/t.ml22
-rw-r--r--lib/syntax/cte.ml (renamed from lib/configuration/cte.ml)0
-rw-r--r--lib/syntax/cte.mli (renamed from lib/configuration/cte.mli)9
-rwxr-xr-xlib/syntax/dune12
-rw-r--r--lib/syntax/importerSyntax.ml (renamed from lib/configuration/syntax.ml)46
-rw-r--r--lib/syntax/importerSyntax.mli37
-rw-r--r--tests/analyser_dependency.ml2
-rw-r--r--tests/analyser_filters.ml18
-rw-r--r--tests/analyser_query_test.ml3
-rw-r--r--tests/confLoader.ml8
-rw-r--r--tests/configuration_toml.ml2
-rw-r--r--tests/dune2
-rw-r--r--tests/expression_repr.ml2
-rw-r--r--tests/importCSV_test.ml15
-rw-r--r--tests/importConf_test.ml5
-rw-r--r--tests/sql_db.ml49
-rw-r--r--tests/test_migration.ml8
66 files changed, 442 insertions, 418 deletions
diff --git a/bin/dune b/bin/dune
index 6496cd3..bd33b01 100755
--- a/bin/dune
+++ b/bin/dune
@@ -17,15 +17,15 @@
sqlite3
tools
helpers
+ importerSyntax
importConf
importAnalyser
importContainers
importDataTypes
- importCSV
importErrors
importExpression
- importFileHandler
importSQL
+ importFileHandler
)
(link_flags (:standard))
)
diff --git a/bin/importer.ml b/bin/importer.ml
index 8549643..0da2ab7 100644
--- a/bin/importer.ml
+++ b/bin/importer.ml
@@ -24,18 +24,18 @@ let creation_date file =
module Args = struct
type arguments = {
- configuration : ImportConf.Syntax.t;
+ configuration : ImporterSyntax.t;
conf_name : string;
bom : bool;
print_conf : bool;
mapping_date : float option;
}
- let load_conf : string -> ImportConf.Syntax.t =
+ let load_conf : string -> ImporterSyntax.t =
fun file ->
match Filename.extension file with
| _ -> (
- let (conf : (Db.Syntax.t, string) result) =
+ let (conf : (ImporterSyntax.t, string) result) =
let* configuration_file =
try Ok (Otoml.Parser.from_file (exists file)) with
| Otoml.Parse_error (position, message) ->
@@ -60,7 +60,7 @@ module Args = struct
| Ok e -> e)
let load () =
- let conf = ref ("", ImportConf.dummy_conf)
+ let conf = ref ("", ImporterSyntax.dummy_conf)
and bom = ref true
and usage = "importer [--conf configuration.toml]"
and print_conf = ref false in
@@ -101,11 +101,11 @@ end
function is expected to convert the result into string in order to include
the content in the output CSV. *)
let printer :
- string -> Path.t ImportExpression.T.t * ImportCSV.DataType.t -> string =
+ string -> Path.t ImportExpression.T.t * ImportDataTypes.Value.t -> string =
fun locale (column, value) ->
ignore column;
(* Set the output locale accoording the configuration file *)
- ImportCSV.DataType.to_string locale value
+ ImportDataTypes.Value.to_string locale value
let bom = "\xEF\xBB\xBF"
@@ -137,7 +137,7 @@ let process_table :
table. *)
match ImportSQL.Db.query_headers db source with
| Ok v ->
- let f = ImportCSV.DataType.to_string locale in
+ let f = ImportDataTypes.Value.to_string locale in
let text_headers = Array.map v ~f in
Headers.SheeetMap.add source text_headers map
| Error _ -> map)
@@ -163,24 +163,23 @@ let process_table :
match headers_opt with
| None -> map
| Some v ->
- let f = ImportCSV.DataType.to_string locale in
+ let f = ImportDataTypes.Value.to_string locale in
let text_headers = Array.map v ~f in
Headers.SheeetMap.add source text_headers map
in
headers
let check_deps :
- 'a Db.t -> Csv.out_channel Lazy.t -> ImportConf.Syntax.t -> Table.t -> unit
- =
+ 'a Db.t -> Csv.out_channel Lazy.t -> ImporterSyntax.t -> Table.t -> unit =
fun db log_error conf source ->
(* For each external check if the values are loaded *)
- let dependancies = ImportConf.get_dependancies_for_table conf source in
+ let dependancies = ImporterSyntax.get_dependancies_for_table conf source in
List.iter dependancies ~f:(fun ext ->
- match ext.ImportConf.Syntax.Extern.allow_missing with
+ match ext.ImporterSyntax.Extern.allow_missing with
| true -> ()
| false -> (
Printf.printf "Checking dependancies for %s\n%!"
- ext.ImportConf.Syntax.Extern.target.ImportDataTypes.Table.name;
+ ext.ImporterSyntax.Extern.target.ImportDataTypes.Table.name;
try
ignore
@@ Db.check_foreign db conf ext ~f:(fun values ->
@@ -188,7 +187,7 @@ let check_deps :
let row =
match snd (Array.get values 0) with
- | ImportCSV.DataType.Integer i -> i
+ | ImportDataTypes.Value.Integer i -> i
| _ -> -1
and value = snd (Array.get values 1) in
let error =
@@ -198,11 +197,11 @@ let check_deps :
sheet = source.Table.tab;
row;
value;
- target = Some ext.ImportConf.Syntax.Extern.target;
+ target = Some ext.ImporterSyntax.Extern.target;
exn =
Failure
(Printf.sprintf "Key '%s' not found"
- (CSV.DataType.to_string "C" value));
+ (ImportDataTypes.Value.to_string "C" value));
}
in
@@ -232,7 +231,7 @@ let () =
(* With the printconf option, we do not need to open any file *)
if conf.print_conf then (
- let toml = ImportConf.Syntax.repr conf.configuration in
+ let toml = ImporterSyntax.repr conf.configuration in
Otoml.Printer.to_channel ~collapse_tables:true stdout toml;
exit 0);
diff --git a/lib/analysers/chunk.ml b/lib/analysers/chunk.ml
index 2fa4808..cefa6d8 100644
--- a/lib/analysers/chunk.ml
+++ b/lib/analysers/chunk.ml
@@ -8,13 +8,13 @@
type t = {
b : Buffer.t;
- parameters : ImportCSV.DataType.t Queue.t;
+ parameters : ImportDataTypes.Value.t Queue.t;
}
let create : unit -> t =
fun () -> { b = Buffer.create 16; parameters = Queue.create () }
-let create' : Buffer.t -> ImportCSV.DataType.t Queue.t -> t =
+let create' : Buffer.t -> ImportDataTypes.Value.t Queue.t -> t =
fun b parameters -> { b; parameters }
(* Append the element from [tail] at the end of [head]
@@ -39,10 +39,9 @@ let copy : t -> t =
Buffer.add_buffer b t.b;
{ b; parameters }
-let add_parameters : t -> ImportCSV.DataType.t Seq.t -> unit =
+let add_parameters : t -> ImportDataTypes.Value.t Seq.t -> unit =
fun t p -> Queue.add_seq t.parameters p
-module Syntax = ImportConf.Syntax
module Table = ImportDataTypes.Table
module Q = ImportExpression.Query
open StdLabels
@@ -53,7 +52,8 @@ open StdLabels
- the join query in order to load the data from the external column
- the column corresponding to the key in order to identify the missing links
later. *)
-let join_external : conf:Syntax.t -> join_buffer:t -> Syntax.Extern.t -> unit =
+let join_external :
+ conf:ImporterSyntax.t -> join_buffer:t -> ImporterSyntax.Extern.t -> unit =
fun ~conf ~join_buffer external_ ->
let extern_table = Table.name external_.target in
@@ -67,8 +67,8 @@ let join_external : conf:Syntax.t -> join_buffer:t -> Syntax.Extern.t -> unit =
in
add_parameters join_buffer (Queue.to_seq q)))
- (Table.print_column external_.Syntax.Extern.target
- ("key_" ^ external_.Syntax.Extern.target.name));
+ (Table.print_column external_.ImporterSyntax.Extern.target
+ ("key_" ^ external_.ImporterSyntax.Extern.target.name));
Format.pp_print_flush formatter ()
@@ -78,7 +78,7 @@ let join_external : conf:Syntax.t -> join_buffer:t -> Syntax.Extern.t -> unit =
SQLite is able to optimize the query and do not load the table not used in
the select clause. *)
let create_from_statement_of_chunck :
- ?externals:Syntax.Extern.t list -> Syntax.t -> t -> unit =
+ ?externals:ImporterSyntax.Extern.t list -> ImporterSyntax.t -> t -> unit =
fun ?externals conf c ->
let externals = Option.value externals ~default:conf.externals in
add_string c "\nFROM '";
@@ -91,7 +91,10 @@ let create_from_statement_of_chunck :
List.iter externals ~f:(join_external ~conf ~join_buffer:c)
let add_expression :
- conf:Syntax.t -> t -> ImportDataTypes.Path.t ImportExpression.T.t -> unit =
+ conf:ImporterSyntax.t ->
+ t ->
+ ImportDataTypes.Path.t ImportExpression.T.t ->
+ unit =
fun ~conf group expression ->
let formatter = Format.formatter_of_buffer group.b in
let queue =
diff --git a/lib/analysers/chunk.mli b/lib/analysers/chunk.mli
index d4f69e7..13a748a 100644
--- a/lib/analysers/chunk.mli
+++ b/lib/analysers/chunk.mli
@@ -2,11 +2,11 @@
type t = {
b : Buffer.t;
- parameters : ImportCSV.DataType.t Queue.t;
+ parameters : ImportDataTypes.Value.t Queue.t;
}
val create : unit -> t
-val create' : Buffer.t -> ImportCSV.DataType.t Queue.t -> t
+val create' : Buffer.t -> ImportDataTypes.Value.t Queue.t -> t
val append : head:t -> tail:t -> unit
(** Append the element from [tail] at the end of [head]
@@ -19,7 +19,7 @@ val add_string : t -> string -> unit
val copy : t -> t
val create_from_statement_of_chunck :
- ?externals:ImportConf.Syntax.Extern.t list -> ImportConf.Syntax.t -> t -> unit
+ ?externals:ImporterSyntax.Extern.t list -> ImporterSyntax.t -> t -> unit
(** Create the from part of the query, adding all the declared externals (even
when not required)
@@ -27,7 +27,7 @@ val create_from_statement_of_chunck :
the select clause. *)
val add_expression :
- conf:ImportConf.Syntax.t ->
+ conf:ImporterSyntax.t ->
t ->
ImportDataTypes.Path.t ImportExpression.T.t ->
unit
diff --git a/lib/analysers/dependency.ml b/lib/analysers/dependency.ml
index d0ea8b3..38bc23c 100644
--- a/lib/analysers/dependency.ml
+++ b/lib/analysers/dependency.ml
@@ -1,6 +1,5 @@
open StdLabels
module IntSet = ImportContainers.IntSet
-module Syntax = ImportConf.Syntax
module Table = ImportDataTypes.Table
module Path = ImportDataTypes.Path
module Expression = ImportExpression.T
@@ -61,11 +60,15 @@ type 'a expression_extractor = {
The function may raise [Unknown_source] if the the path describe an unknown
table. *)
let add_path_in_map :
- f:'a expression_extractor -> conf:Syntax.t -> 'a -> build_map -> build_map =
+ f:'a expression_extractor ->
+ conf:ImporterSyntax.t ->
+ 'a ->
+ build_map ->
+ build_map =
fun ~f ~conf path map ->
let table_source, column = f.of_path path in
let table =
- try ImportConf.get_table_for_name conf table_source with
+ try ImporterSyntax.get_table_for_name conf table_source with
| Not_found -> raise (ImportErrors.Unknown_source (Option.get table_source))
in
@@ -81,7 +84,7 @@ let add_path_in_map :
let add_expression_in_map :
f:'a expression_extractor ->
- conf:Syntax.t ->
+ conf:ImporterSyntax.t ->
'a Expression.t ->
build_map ->
build_map =
@@ -91,7 +94,7 @@ let add_expression_in_map :
let add_columns_in_map :
f:'a expression_extractor ->
- conf:Syntax.t ->
+ conf:ImporterSyntax.t ->
'a Expression.t list ->
build_map ->
build_map =
@@ -109,9 +112,9 @@ let add_columns_in_map :
This function is called for each path declared inside the expression. *)
let add_dependancies :
- conf:Syntax.t -> Syntax.Extern.t -> deps -> Path.t -> deps =
+ conf:ImporterSyntax.t -> ImporterSyntax.Extern.t -> deps -> Path.t -> deps =
fun ~conf extern graph path ->
- let source_table = ImportConf.get_table_for_name conf path.Path.alias in
+ let source_table = ImporterSyntax.get_table_for_name conf path.Path.alias in
let source = ImportContainers.Source.from_table source_table in
let target = ImportContainers.Source.from_table extern.target in
@@ -121,14 +124,17 @@ let add_dependancies :
| _ -> (target, [ source ]) :: graph
let add_external_in_map :
- conf:Syntax.t -> Syntax.Extern.t -> build_map * deps -> build_map * deps =
+ conf:ImporterSyntax.t ->
+ ImporterSyntax.Extern.t ->
+ build_map * deps ->
+ build_map * deps =
fun ~conf extern (map, graph) ->
let dest = ImportContainers.KeyName.from_table extern.target in
(* Pre-check that every source is already declared in the configuration. *)
let _ =
Expression.fold_values extern.intern_key ~init:() ~f:(fun () path ->
try
- let _ = ImportConf.get_table_for_name conf path.Path.alias in
+ let _ = ImporterSyntax.get_table_for_name conf path.Path.alias in
()
with
| Not_found -> (
@@ -177,7 +183,7 @@ let add_external_in_map :
{
of_path =
(fun Path.{ alias; column } ->
- let table = ImportConf.get_table_for_name conf alias in
+ let table = ImporterSyntax.get_table_for_name conf alias in
(Some table.name, column));
to_mapping =
(fun mapping column ->
@@ -194,11 +200,11 @@ let mapper =
of_path = (fun ({ alias; column } : Path.t) -> (alias, column));
}
-let get_mapping : Syntax.t -> build_map * deps =
+let get_mapping : ImporterSyntax.t -> build_map * deps =
fun conf ->
- let root = ImportContainers.Source.from_table (ImportConf.root_table conf)
+ let root = ImportContainers.Source.from_table (ImporterSyntax.root_table conf)
and root' =
- ImportContainers.KeyName.from_table (ImportConf.root_table conf)
+ ImportContainers.KeyName.from_table (ImporterSyntax.root_table conf)
in
let graph = [ (root, []) ] in
@@ -207,7 +213,7 @@ let get_mapping : Syntax.t -> build_map * deps =
let init =
( ImportContainers.Externals.singleton root'
{
- table = ImportConf.root_table conf;
+ table = ImporterSyntax.root_table conf;
columns = IntSet.empty;
keys = [];
},
@@ -229,7 +235,7 @@ let get_mapping : Syntax.t -> build_map * deps =
in
(map, graph)
-let get_process_order : Syntax.t -> t list =
+let get_process_order : ImporterSyntax.t -> t list =
fun map ->
let map, graph = get_mapping map in
diff --git a/lib/analysers/dependency.mli b/lib/analysers/dependency.mli
index bc761ae..522436c 100644
--- a/lib/analysers/dependency.mli
+++ b/lib/analysers/dependency.mli
@@ -1,6 +1,6 @@
type t
-val get_process_order : ImportConf.Syntax.t -> t list
+val get_process_order : ImporterSyntax.t -> t list
(** Extract the file list to process, following the identified dependancies. Try
to load first the document which does not required another spreadsheet, and
keep going in the topological order
diff --git a/lib/analysers/dune b/lib/analysers/dune
index 382dd6b..3ba018c 100755
--- a/lib/analysers/dune
+++ b/lib/analysers/dune
@@ -1,12 +1,11 @@
(library
(name importAnalyser)
(libraries
- importConf
- importContainers
- importCSV
importDataTypes
- importExpression
importErrors
+ importExpression
+ importerSyntax
+ importContainers
tsort
)
(preprocess (pps
diff --git a/lib/analysers/filters.ml b/lib/analysers/filters.ml
index 4e8b175..7044798 100644
--- a/lib/analysers/filters.ml
+++ b/lib/analysers/filters.ml
@@ -1,14 +1,12 @@
(** Build a fragment of the query match a filter *)
-module Syntax = ImportConf.Syntax
module Path = ImportDataTypes.Path
module Expression = ImportExpression
-module CTE = ImportConf.CTE
open StdLabels
(** Add a list of expressions into the group *)
let rec add_filters :
- conf:Syntax.t -> Chunk.t -> Path.t Expression.T.t list -> unit =
+ conf:ImporterSyntax.t -> Chunk.t -> Path.t Expression.T.t list -> unit =
fun ~conf group -> function
| [] -> ()
| hd :: [] -> Chunk.add_expression ~conf group hd
@@ -25,7 +23,7 @@ type 'a cte_acc = {
latest_expression : Path.t Expression.T.t list;
}
-let add_inner : conf:Syntax.t -> int -> Buffer.t -> unit =
+let add_inner : conf:ImporterSyntax.t -> int -> Buffer.t -> unit =
fun ~conf n b ->
let name = "filter" ^ string_of_int n in
(* We use an INNER JOIN here because we want to be sure to get all the rows
@@ -39,15 +37,15 @@ let add_inner : conf:Syntax.t -> int -> Buffer.t -> unit =
Buffer.add_string b ".id\n"
let print :
- conf:Syntax.t ->
+ conf:ImporterSyntax.t ->
(Chunk.t * Chunk.t) cte_acc ->
- CTE.t ->
+ ImporterSyntax.CTE.t ->
(Chunk.t * Chunk.t) cte_acc =
fun ~conf acc cte ->
let predicates, query = acc.acc in
let n = acc.n in
let cte_index =
- match cte.CTE.group with
+ match cte.ImporterSyntax.CTE.group with
| Some expression ->
begin
if acc.has_previous then Chunk.add_string query ", "
@@ -73,11 +71,11 @@ let print :
end;
begin
- match cte.CTE.filters with
+ match cte.ImporterSyntax.CTE.filters with
| [] -> ()
| _ ->
Chunk.add_string query " WHERE ";
- add_filters ~conf query cte.CTE.filters
+ add_filters ~conf query cte.ImporterSyntax.CTE.filters
end;
Chunk.add_string query ")\n";
Some acc.n
@@ -85,7 +83,7 @@ let print :
(* Do not add the filters in the CTE (we don’t have any) but in the main
query *)
Chunk.add_string predicates "WHERE ";
- add_filters ~conf predicates cte.CTE.filters;
+ add_filters ~conf predicates cte.ImporterSyntax.CTE.filters;
acc.cte_index
in
{
@@ -93,10 +91,11 @@ let print :
has_previous = true;
n = acc.n + 1;
cte_index;
- latest_expression = cte.CTE.filters;
+ latest_expression = cte.ImporterSyntax.CTE.filters;
}
-let generate_sql : conf:Syntax.t -> CTE.t list -> Chunk.t -> Chunk.t =
+let generate_sql :
+ conf:ImporterSyntax.t -> ImporterSyntax.CTE.t list -> Chunk.t -> Chunk.t =
fun ~conf filters links' ->
let predicates = Chunk.create () and links = Chunk.create () in
let eval =
diff --git a/lib/analysers/filters.mli b/lib/analysers/filters.mli
index 7783799..3a81202 100644
--- a/lib/analysers/filters.mli
+++ b/lib/analysers/filters.mli
@@ -1,2 +1,2 @@
val generate_sql :
- conf:ImportConf.Syntax.t -> ImportConf.CTE.t list -> Chunk.t -> Chunk.t
+ conf:ImporterSyntax.t -> ImporterSyntax.CTE.t list -> Chunk.t -> Chunk.t
diff --git a/lib/analysers/headers.ml b/lib/analysers/headers.ml
index 916dfee..cbeddfb 100644
--- a/lib/analysers/headers.ml
+++ b/lib/analysers/headers.ml
@@ -1,7 +1,5 @@
open StdLabels
-module I = ImportConf
module E = ImportExpression.T
-module Syntax = ImportConf.Syntax
module Table = ImportDataTypes.Table
module Path = ImportDataTypes.Path
@@ -15,7 +13,7 @@ end)
type content = string array
type t = content SheeetMap.t
-(** The map associate a line of headers for each table.
+(** The map associate a line of headers for each table.
The header are always in string. *)
@@ -23,9 +21,8 @@ type t = content SheeetMap.t
and will reformat the first line with the values from the cell. The
functions will not be evaluated (instead they will be displayed "as is".
- When there is no value for this path, return empty string.
- *)
-let columns : Syntax.t -> t -> string list =
+ When there is no value for this path, return empty string. *)
+let columns : ImporterSyntax.t -> t -> string list =
fun conf t ->
(* We build here a custom printer which search in the array for the column
name.
@@ -33,7 +30,7 @@ let columns : Syntax.t -> t -> string list =
This function will be given as argument in the expression printer. *)
let f : Path.t -> Buffer.t -> unit =
fun path b ->
- let source = I.get_table_for_name conf path.alias in
+ let source = ImporterSyntax.get_table_for_name conf path.alias in
match SheeetMap.find_opt source t with
| None -> ()
@@ -42,12 +39,13 @@ let columns : Syntax.t -> t -> string list =
| _ ->
prerr_endline
@@ Printf.sprintf "No header found for :%s.%s"
- (Option.value ~default:(I.root_table conf).Table.name
+ (Option.value
+ ~default:(ImporterSyntax.root_table conf).Table.name
path.alias)
- (ImportCSV.Csv.column_to_string path.column))
+ (ImportDataTypes.Path.column_to_string path.column))
in
- List.map conf.Syntax.columns ~f:(fun column ->
+ List.map conf.ImporterSyntax.columns ~f:(fun column ->
let b = Buffer.create 4 in
ImportExpression.Headers.headers_of_expression b f column;
diff --git a/lib/analysers/headers.mli b/lib/analysers/headers.mli
index 03e384b..b9149a9 100644
--- a/lib/analysers/headers.mli
+++ b/lib/analysers/headers.mli
@@ -2,10 +2,9 @@ module SheeetMap : Map.S with type key = ImportDataTypes.Table.t
type t = string array SheeetMap.t
-val columns : ImportConf.Syntax.t -> t -> string list
+val columns : ImporterSyntax.t -> t -> string list
(** Get the headers. The function has to be called after reading each document,
and will reformat the first line with the values from the cell. The
functions will not be evaluated (instead they will be displayed "as is".
- When there is no value for this path, return empty string.
- *)
+ When there is no value for this path, return empty string. *)
diff --git a/lib/analysers/printers.ml b/lib/analysers/printers.ml
index 1c73c13..3dc2f02 100644
--- a/lib/analysers/printers.ml
+++ b/lib/analysers/printers.ml
@@ -1,10 +1,9 @@
-module Syntax = ImportConf.Syntax
module Table = ImportDataTypes.Table
module Path = ImportDataTypes.Path
-let path : conf:Syntax.t -> Format.formatter -> Path.t -> unit =
+let path : conf:ImporterSyntax.t -> Format.formatter -> Path.t -> unit =
fun ~conf buffer { alias; column } ->
- let table = ImportConf.get_table_for_name conf alias in
+ let table = ImporterSyntax.get_table_for_name conf alias in
Format.fprintf buffer "%s"
(Table.print_column table ("col_" ^ string_of_int column))
diff --git a/lib/analysers/printers.mli b/lib/analysers/printers.mli
index 102bb91..3916e1c 100644
--- a/lib/analysers/printers.mli
+++ b/lib/analysers/printers.mli
@@ -1,5 +1,5 @@
val path :
- conf:ImportConf.Syntax.t -> Format.formatter -> ImportDataTypes.Path.t -> unit
+ conf:ImporterSyntax.t -> Format.formatter -> ImportDataTypes.Path.t -> unit
(** Represent a path in a SQL query.
This function is given in the Expression.Query module. *)
diff --git a/lib/analysers/query.ml b/lib/analysers/query.ml
index e24da78..f89f5f0 100644
--- a/lib/analysers/query.ml
+++ b/lib/analysers/query.ml
@@ -1,15 +1,15 @@
open StdLabels
module Expression = ImportExpression
module Q = Expression.Query
-module Syntax = ImportConf.Syntax
module Table = ImportDataTypes.Table
module Path = ImportDataTypes.Path
(* Collect all the tables pointed by the expression. *)
-let pointed_tables : Syntax.t -> 'a Expression.T.t -> (Table.t * string) list =
+let pointed_tables :
+ ImporterSyntax.t -> 'a Expression.T.t -> (Table.t * string) list =
fun conf expression ->
Expression.T.fold_values expression ~init:[] ~f:(fun acc path ->
- let table = ImportConf.get_table_for_name conf path.Path.alias in
+ let table = ImporterSyntax.get_table_for_name conf path.Path.alias in
let table_name = Table.name table in
(table, table_name) :: acc)
|> List.sort_uniq ~cmp:Stdlib.compare
@@ -37,7 +37,7 @@ let create_table : Dependency.t -> string =
type query = {
q : string;
- parameters : ImportCSV.DataType.t Seq.t;
+ parameters : ImportDataTypes.Value.t Seq.t;
}
let rec take_elements :
@@ -88,9 +88,9 @@ let clean_window :
The select query will name each column with an alias, and the map allow to
find which source is pointed by this alias. *)
-let select : Syntax.t -> query * Path.t ImportExpression.T.t array =
+let select : ImporterSyntax.t -> query * Path.t ImportExpression.T.t array =
fun conf ->
- let filter = ImportConf.CTE.of_filters conf.filters in
+ let filter = ImporterSyntax.CTE.of_filters conf.filters in
(* For each column in the configuration file, add the corresponding element
in the query.
@@ -136,7 +136,7 @@ let select : Syntax.t -> query * Path.t ImportExpression.T.t array =
Chunk.append ~head:request_header ~tail:filters;
let formatter = Format.formatter_of_buffer b in
- (match conf.Syntax.uniq with
+ (match conf.ImporterSyntax.uniq with
| [] -> ()
| uniq ->
Format.fprintf formatter "\nGROUP BY %a"
@@ -149,7 +149,7 @@ let select : Syntax.t -> query * Path.t ImportExpression.T.t array =
in
Queue.transfer seq parameters))
uniq);
- (match conf.Syntax.sort with
+ (match conf.ImporterSyntax.sort with
| [] -> ()
| sort ->
Format.fprintf formatter "\nORDER BY %a"
@@ -166,15 +166,16 @@ let select : Syntax.t -> query * Path.t ImportExpression.T.t array =
({ q = Buffer.contents b; parameters = Queue.to_seq parameters }, headers)
-let check_external : Syntax.t -> Syntax.Extern.t -> query =
+let check_external : ImporterSyntax.t -> ImporterSyntax.Extern.t -> query =
fun conf external_ ->
let internal_chunk = Chunk.create () in
- Chunk.add_expression ~conf internal_chunk external_.Syntax.Extern.intern_key;
+ Chunk.add_expression ~conf internal_chunk
+ external_.ImporterSyntax.Extern.intern_key;
let external_key_buffer = Buffer.create 16 in
Buffer.add_string external_key_buffer
- (Table.print_column external_.Syntax.Extern.target
- ("key_" ^ external_.Syntax.Extern.target.name));
+ (Table.print_column external_.ImporterSyntax.Extern.target
+ ("key_" ^ external_.ImporterSyntax.Extern.target.name));
let pointed_tables = pointed_tables conf external_.intern_key in
@@ -190,7 +191,9 @@ let check_external : Syntax.t -> Syntax.Extern.t -> query =
This not the usual way to proceed (we start from the source and link the externals)
*)
let rec collect_links :
- Syntax.Extern.t -> Syntax.Extern.t list -> Syntax.Extern.t list =
+ ImporterSyntax.Extern.t ->
+ ImporterSyntax.Extern.t list ->
+ ImporterSyntax.Extern.t list =
fun table init ->
let res =
(* Do not add the same external if the value is already present *)
@@ -200,16 +203,16 @@ let check_external : Syntax.t -> Syntax.Extern.t -> query =
| Some _ -> init
in
- Expression.T.fold_values ~init table.Syntax.Extern.intern_key
+ Expression.T.fold_values ~init table.ImporterSyntax.Extern.intern_key
~f:(fun acc expr ->
match expr.Path.alias with
| None -> acc
| Some _ as path -> (
- let table = ImportConf.get_table_for_name conf path in
+ let table = ImporterSyntax.get_table_for_name conf path in
(* Look for this table in the externals *)
let external_opt =
- List.find_opt conf.Syntax.externals ~f:(fun t ->
- t.Syntax.Extern.target == table)
+ List.find_opt conf.ImporterSyntax.externals ~f:(fun t ->
+ t.ImporterSyntax.Extern.target == table)
in
match external_opt with
| None -> acc
diff --git a/lib/analysers/query.mli b/lib/analysers/query.mli
index d158867..491c891 100644
--- a/lib/analysers/query.mli
+++ b/lib/analysers/query.mli
@@ -2,16 +2,15 @@ val create_table : Dependency.t -> string
type query = {
q : string; (** The query to execute *)
- parameters : ImportCSV.DataType.t Seq.t;
+ parameters : ImportDataTypes.Value.t Seq.t;
}
(** This type represent a query to execute. [q] is the template to run, and
shall be run with all the binded parameters. *)
val select :
- ImportConf.Syntax.t ->
- query * ImportDataTypes.Path.t ImportExpression.T.t array
+ ImporterSyntax.t -> query * ImportDataTypes.Path.t ImportExpression.T.t array
-val check_external : ImportConf.Syntax.t -> ImportConf.Syntax.Extern.t -> query
+val check_external : ImporterSyntax.t -> ImporterSyntax.Extern.t -> query
(** Create a query which select all the missing key in an external *)
val build_key_insert : Buffer.t -> Dependency.key -> unit
diff --git a/lib/configuration/dune b/lib/configuration/dune
index 6a0bc61..7f78cca 100755
--- a/lib/configuration/dune
+++ b/lib/configuration/dune
@@ -4,17 +4,14 @@
decoders
otoml
menhirLib
- importCSV
- re
helpers
importDataTypes
- importExpression
importErrors
+ importExpression
+ importerSyntax
)
(preprocess (pps
ppx_deriving.ord
- ppx_deriving.show
- ppx_deriving.eq
))
)
diff --git a/lib/configuration/expression_parser.mly b/lib/configuration/expression_parser.mly
index 18b79c8..1761cce 100644
--- a/lib/configuration/expression_parser.mly
+++ b/lib/configuration/expression_parser.mly
@@ -36,8 +36,8 @@ path_:
| COLUMN
column = IDENT
{ ImportExpression.T.Path
- Syntax.Path.{ alias = None
- ; column = ImportCSV.Csv.column_of_string column
+ ImportDataTypes.Path.{ alias = None
+ ; column = ImportDataTypes.Path.column_of_string column
}
}
@@ -46,14 +46,14 @@ path_:
DOT
column = IDENT
{ ImportExpression.T.Path
- Syntax.Path.{ alias = Some table
- ; column = ImportCSV.Csv.column_of_string column}
+ ImportDataTypes.Path.{ alias = Some table
+ ; column = ImportDataTypes.Path.column_of_string column}
}
column_:
| COLUMN
column = IDENT
- { try ImportExpression.T.Path (ImportCSV.Csv.column_of_string column)
+ { try ImportExpression.T.Path (ImportDataTypes.Path.column_of_string column)
with _ -> ImportExpression.T.Literal column }
arguments(PATH):
diff --git a/lib/configuration/importConf.ml b/lib/configuration/importConf.ml
index 8516008..2df24bd 100644
--- a/lib/configuration/importConf.ml
+++ b/lib/configuration/importConf.ml
@@ -1,68 +1,17 @@
-open StdLabels
-module Syntax = Syntax
-module CTE = Cte
-module Table = ImportDataTypes.Table
-module Path = ImportDataTypes.Path
-module T = Read_conf
-module Expression = ImportExpression.T
-
-let latest_version = 1
-
module TomlReader = Read_conf.Make (Helpers.Toml.Decode)
-let t_of_toml : Otoml.t -> (Syntax.t, string) result =
+let t_of_toml : Otoml.t -> (ImporterSyntax.t, string) result =
fun toml ->
let version =
- Otoml.find_or ~default:latest_version toml
+ Otoml.find_or ~default:ImporterSyntax.latest_version toml
(Otoml.get_integer ~strict:false)
[ "version" ]
in
match version with
- | n when n = latest_version -> TomlReader.read toml
+ | n when n = ImporterSyntax.latest_version -> TomlReader.read toml
| _ ->
Printf.eprintf "Unsuported version : %d\n" version;
exit 1
-let dummy_conf =
- Syntax.
- {
- source = { file = ""; tab = 0; name = "" };
- version = latest_version;
- locale = Some "C";
- externals = [];
- columns = [];
- filters = [];
- sort = [];
- uniq = [];
- }
-
-let get_table_for_name : Syntax.t -> string option -> Table.t =
- fun conf name ->
- match name with
- | None -> conf.source
- | Some name ->
- if String.equal name conf.source.name then conf.source
- else
- let ext =
- List.find conf.externals ~f:(fun (ext : Syntax.Extern.t) ->
- String.equal name ext.target.name)
- in
- ext.target
-
-let root_table : Syntax.t -> Table.t = fun conf -> conf.source
-
-let get_dependancies_for_table : Syntax.t -> Table.t -> Syntax.Extern.t list =
- fun conf source ->
- let is_root = source = conf.source in
-
- List.filter conf.externals ~f:(fun (ext : Syntax.Extern.t) ->
- (* Enumerate the intern_key and check the source pointed by each column *)
- Expression.fold_values ext.intern_key ~init:false ~f:(fun acc expr ->
- if acc then acc
- else
- match expr.Syntax.Path.alias with
- | Some v -> String.equal v source.name
- | None -> is_root))
-
let expression_from_string s =
Read_conf.ExpressionParser.of_string Read_conf.ExpressionParser.path s
diff --git a/lib/configuration/importConf.mli b/lib/configuration/importConf.mli
index 40b985b..d2f65f2 100644
--- a/lib/configuration/importConf.mli
+++ b/lib/configuration/importConf.mli
@@ -1,18 +1,4 @@
-module Syntax = Syntax
-module CTE = Cte
-
-val dummy_conf : Syntax.t
-
-val root_table : Syntax.t -> ImportDataTypes.Table.t
-(** Get the root table, this table is the main table to load and each line in
- this table will be processed *)
-
-val t_of_toml : Otoml.t -> (Syntax.t, string) result
-val get_table_for_name : Syntax.t -> string option -> ImportDataTypes.Table.t
-
-val get_dependancies_for_table :
- Syntax.t -> ImportDataTypes.Table.t -> Syntax.Extern.t list
-(** Get all the externals refered by the source *)
+val t_of_toml : Otoml.t -> (ImporterSyntax.t, string) result
val expression_from_string :
string -> (ImportDataTypes.Path.t ImportExpression.T.t, string) result
diff --git a/lib/configuration/read_conf.ml b/lib/configuration/read_conf.ml
index 11f6726..d406b0e 100644
--- a/lib/configuration/read_conf.ml
+++ b/lib/configuration/read_conf.ml
@@ -222,7 +222,7 @@ module Make (S : Decoders.Decode.S) = struct
in
S.succeed
- Syntax.Extern.
+ ImporterSyntax.Extern.
{
intern_key;
extern_key;
@@ -256,7 +256,7 @@ module Make (S : Decoders.Decode.S) = struct
ExpressionParser.path)
in
S.succeed @@ fun version source externals locale ->
- Syntax.
+ ImporterSyntax.
{ version; source; externals; columns; filters; sort; uniq; locale }
method conf =
diff --git a/lib/containers/dune b/lib/containers/dune
index 70bbf0b..d9b06cf 100755
--- a/lib/containers/dune
+++ b/lib/containers/dune
@@ -2,7 +2,6 @@
(name importContainers)
(libraries
importDataTypes
- importConf
)
(preprocess (pps
ppx_deriving.ord
diff --git a/lib/containers/importContainers.ml b/lib/containers/importContainers.ml
index 6da8374..c90bc57 100644
--- a/lib/containers/importContainers.ml
+++ b/lib/containers/importContainers.ml
@@ -1,5 +1,3 @@
-module Conf = ImportConf
-module Syntax = Conf.Syntax
module Table = ImportDataTypes.Table
(** This key is used to create the table of each externals in the configuration.
diff --git a/lib/csv/csv.ml b/lib/csv/csv.ml
deleted file mode 100644
index db7329d..0000000
--- a/lib/csv/csv.ml
+++ /dev/null
@@ -1,30 +0,0 @@
-open StdLabels
-
-type t = int
-
-let column_of_char = function
- | 'A' .. 'Z' as c -> Char.code c - (Char.code 'A' - 1)
- | 'a' .. 'z' as c -> Char.code c - (Char.code 'a' - 1)
- | c -> raise (Invalid_argument ("column: " ^ Char.escaped c))
-
-let column_of_string : string -> int =
- fun s ->
- String.fold_left s ~init:0 ~f:(fun value c -> (value * 26) + column_of_char c)
-
-(** Accumulate the remaining for the successives divisions in a list. *)
-let rec _to_char ~b i =
- if i > 0 then
- let res = i mod 26 in
- let res = if res = 0 then 26 else res in
-
- let c = char_of_int @@ (res + 64) in
- (* The modulo is accumulated in the list head, which is the expected
- sequence *)
- let b = c :: b in
-
- _to_char ~b @@ ((i - res) / 26)
- else b
-
-let column_to_string i =
- let res = _to_char ~b:[] i in
- List.to_seq res |> String.of_seq
diff --git a/lib/csv/dune b/lib/csv/dune
deleted file mode 100755
index 2cdc868..0000000
--- a/lib/csv/dune
+++ /dev/null
@@ -1,7 +0,0 @@
-(library
- (name importCSV)
- (libraries
- re
- )
- (foreign_stubs (language c) (names format))
-)
diff --git a/lib/data_types/dune b/lib/data_types/dune
index b735171..d17ed8f 100644
--- a/lib/data_types/dune
+++ b/lib/data_types/dune
@@ -1,11 +1,11 @@
(library
(name importDataTypes)
(libraries
- importCSV
+ re
)
-
(preprocess (pps
ppx_deriving.ord
ppx_deriving.show
ppx_deriving.eq ))
+ (foreign_stubs (language c) (names format))
)
diff --git a/lib/csv/format.c b/lib/data_types/format.c
index 1394dc7..1394dc7 100644
--- a/lib/csv/format.c
+++ b/lib/data_types/format.c
diff --git a/lib/data_types/path.ml b/lib/data_types/path.ml
index 0a87109..0dc2b74 100644
--- a/lib/data_types/path.ml
+++ b/lib/data_types/path.ml
@@ -1,3 +1,5 @@
+open StdLabels
+
type column = int [@@deriving ord, show, eq]
type t = {
@@ -8,8 +10,29 @@ type t = {
}
[@@deriving ord, show, eq]
-let show { alias; column } =
- let column_text = ImportCSV.Csv.column_to_string column in
- match alias with
- | None -> ":" ^ column_text
- | Some value -> ":" ^ value ^ "." ^ column_text
+let column_of_char = function
+ | 'A' .. 'Z' as c -> Char.code c - (Char.code 'A' - 1)
+ | 'a' .. 'z' as c -> Char.code c - (Char.code 'a' - 1)
+ | c -> raise (Invalid_argument ("column: " ^ Char.escaped c))
+
+let column_of_string : string -> int =
+ fun s ->
+ String.fold_left s ~init:0 ~f:(fun value c -> (value * 26) + column_of_char c)
+
+(** Accumulate the remaining for the successives divisions in a list. *)
+let rec _to_char ~b i =
+ if i > 0 then
+ let res = i mod 26 in
+ let res = if res = 0 then 26 else res in
+
+ let c = char_of_int @@ (res + 64) in
+ (* The modulo is accumulated in the list head, which is the expected
+ sequence *)
+ let b = c :: b in
+
+ _to_char ~b @@ ((i - res) / 26)
+ else b
+
+let column_to_string i =
+ let res = _to_char ~b:[] i in
+ List.to_seq res |> String.of_seq
diff --git a/lib/data_types/path.mli b/lib/data_types/path.mli
new file mode 100644
index 0000000..5fd7542
--- /dev/null
+++ b/lib/data_types/path.mli
@@ -0,0 +1,12 @@
+type column = int [@@deriving ord, show, eq]
+
+type t = {
+ alias : string option;
+ (* External file to load, when the information is missing, load in
+ the current file *)
+ column : column;
+}
+[@@deriving ord, show, eq]
+
+val column_of_string : string -> int
+val column_to_string : int -> string
diff --git a/lib/csv/dataType.ml b/lib/data_types/value.ml
index 6bf85ae..6bf85ae 100644
--- a/lib/csv/dataType.ml
+++ b/lib/data_types/value.ml
diff --git a/lib/csv/dataType.mli b/lib/data_types/value.mli
index df5edf1..df5edf1 100644
--- a/lib/csv/dataType.mli
+++ b/lib/data_types/value.mli
diff --git a/lib/errors/dune b/lib/errors/dune
index c39f47f..98059b5 100644
--- a/lib/errors/dune
+++ b/lib/errors/dune
@@ -4,7 +4,6 @@
ppx_deriving.runtime
csv
sqlite3
- importCSV
importDataTypes
)
)
diff --git a/lib/errors/importErrors.ml b/lib/errors/importErrors.ml
index 7433e9b..a062f38 100644
--- a/lib/errors/importErrors.ml
+++ b/lib/errors/importErrors.ml
@@ -1,5 +1,4 @@
open StdLabels
-module CSV = ImportCSV
module Table = ImportDataTypes.Table
let bom = "\xEF\xBB\xBF"
@@ -9,7 +8,7 @@ type xlsError = {
row : int;
sheet : int;
target : Table.t option;
- value : CSV.DataType.t;
+ value : ImportDataTypes.Value.t;
exn : exn;
}
@@ -35,7 +34,8 @@ exception
let repr_error = function
| SqlError s -> Printf.sprintf "%s Error" (Sqlite3.Rc.to_string s)
| NullKey k ->
- Printf.sprintf "The key %s is null" (ImportCSV.Csv.column_to_string k)
+ Printf.sprintf "The key %s is null"
+ (ImportDataTypes.Path.column_to_string k)
| Unknown_source source ->
Printf.sprintf "The source %s is referenced without beiing declared"
source
@@ -74,7 +74,7 @@ let output_error : t -> xlsError -> unit =
string_of_int error.sheet;
string_of_int error.row;
target;
- CSV.DataType.to_string "C" error.value;
+ ImportDataTypes.Value.to_string "C" error.value;
repr_error error.exn;
]
diff --git a/lib/errors/importErrors.mli b/lib/errors/importErrors.mli
index 445361e..1afcacf 100644
--- a/lib/errors/importErrors.mli
+++ b/lib/errors/importErrors.mli
@@ -1,5 +1,3 @@
-module CSV = ImportCSV
-
exception SqlError of Sqlite3.Rc.t
exception MisplacedWindow
@@ -18,7 +16,7 @@ type xlsError = {
row : int;
sheet : int;
target : ImportDataTypes.Table.t option;
- value : CSV.DataType.t;
+ value : ImportDataTypes.Value.t;
exn : exn;
}
diff --git a/lib/expression/_readme.rst b/lib/expression/_readme.rst
index 729a950..59c4687 100644
--- a/lib/expression/_readme.rst
+++ b/lib/expression/_readme.rst
@@ -26,13 +26,6 @@ Simple transformations
Composed transformations
------------------------
-:Filter:
-
- Generate the filters in the query. This module identify if one the
- expression is actually a group window and handle a special case for this.
-
- This module relies on Ast
-
:Query:
Build an sql query. This module relies on Type_of
diff --git a/lib/expression/dune b/lib/expression/dune
index 8bf6e62..5f5a12c 100755
--- a/lib/expression/dune
+++ b/lib/expression/dune
@@ -2,7 +2,6 @@
(name importExpression)
(libraries
re
- importCSV
importDataTypes
importErrors
)
diff --git a/lib/expression/query.ml b/lib/expression/query.ml
index e648daf..8ff179d 100644
--- a/lib/expression/query.ml
+++ b/lib/expression/query.ml
@@ -6,7 +6,7 @@ open StdLabels
in order to tell if we need to bind the parameters in the query, or if we
can use plain literal as is (with some risk at the execution time. *)
type _ binded_query =
- | BindParam : ImportCSV.DataType.t Queue.t binded_query
+ | BindParam : ImportDataTypes.Value.t Queue.t binded_query
| NoParam : unit binded_query
module QueryParameter = struct
@@ -17,7 +17,7 @@ module QueryParameter = struct
The Raw can be generated from both BindParam or NoParam queries. *)
type t =
| Literal
- | Queue of ImportCSV.DataType.t Queue.t
+ | Queue of ImportDataTypes.Value.t Queue.t
| Raw of t
(** Wrap the given parameter mode into the raw mode *)
@@ -100,7 +100,7 @@ module Query = TypeBuilder.Make (struct
Format.fprintf formatter "'%s'" l
| QueryParameter.Queue queue ->
Format.fprintf formatter "?";
- Queue.add (ImportCSV.DataType.Content l) queue
+ Queue.add (ImportDataTypes.Value.Content l) queue
| QueryParameter.Raw _ -> Format.fprintf formatter "%s" l
let integer : string -> 'a Type_of.obs -> 'a repr =
diff --git a/lib/expression/query.mli b/lib/expression/query.mli
index fa789a9..a124be4 100644
--- a/lib/expression/query.mli
+++ b/lib/expression/query.mli
@@ -1,17 +1,17 @@
module QueryParameter : sig
- (** Internaly, we need to keep a different type for the Literal chunks
- (which requires to be quoted), and raw (which should be given as is to the
- sql engine)
+ (** Internaly, we need to keep a different type for the Literal chunks (which
+ requires to be quoted), and raw (which should be given as is to the sql
+ engine)
- The Raw can be generated from both BindParam or NoParam queries. *)
+ The Raw can be generated from both BindParam or NoParam queries. *)
type t =
| Literal
- | Queue of ImportCSV.DataType.t Queue.t
+ | Queue of ImportDataTypes.Value.t Queue.t
| Raw of t
end
type _ binded_query =
- | BindParam : ImportCSV.DataType.t Queue.t binded_query
+ | BindParam : ImportDataTypes.Value.t Queue.t binded_query
| NoParam : unit binded_query
val query_of_expression :
diff --git a/lib/file_handler/csv2sql.ml b/lib/file_handler/csv2sql.ml
index 42d84eb..a009f74 100644
--- a/lib/file_handler/csv2sql.ml
+++ b/lib/file_handler/csv2sql.ml
@@ -1,34 +1,33 @@
open StdLabels
module A = ImportAnalyser.Dependency
-module CSV = ImportCSV
module C = ImportContainers
-module Syntax = ImportConf.Syntax
module Db = ImportSQL.Db
-type state = CSV.DataType.t array State.t
+type state = ImportDataTypes.Value.t array State.t
let default_mapper :
- (ImportCSV.DataType.t, ImportCSV.DataType.t array) State.mapper =
- { get_row = Fun.id; get_value = Fun.id; default = ImportCSV.DataType.Null }
+ (ImportDataTypes.Value.t, ImportDataTypes.Value.t array) State.mapper =
+ { get_row = Fun.id; get_value = Fun.id; default = ImportDataTypes.Value.Null }
-let extract_values : string -> CSV.DataType.t =
+let extract_values : string -> ImportDataTypes.Value.t =
fun value ->
(* Test first if the content is empty *)
- if String.equal String.empty value then CSV.DataType.Null
+ if String.equal String.empty value then ImportDataTypes.Value.Null
else
(* else, try differents conversion in order to see which one works *)
match int_of_string_opt value with
- | Some i -> CSV.DataType.Integer i
+ | Some i -> ImportDataTypes.Value.Integer i
| None -> (
match float_of_string_opt value with
- | Some f -> CSV.DataType.Float f
+ | Some f -> ImportDataTypes.Value.Float f
| None ->
(* And finaly convert into date *)
- CSV.DataType.Content value)
+ ImportDataTypes.Value.Content value)
(** Initialize the state for the first row, count the column number and create
the table in the database *)
-let first_row : A.t -> _ Db.t -> state -> CSV.DataType.t list -> state =
+let first_row : A.t -> _ Db.t -> state -> ImportDataTypes.Value.t list -> state
+ =
fun mapping db acc row ->
(if acc.transaction then
match Db.commit db with
@@ -69,11 +68,11 @@ let read_csv_line :
let importInDatable :
log_error:ImportErrors.t ->
- conf:Syntax.t ->
+ conf:ImporterSyntax.t ->
dirname:string ->
A.t ->
'a Db.t ->
- CSV.DataType.t array option Lwt.t =
+ ImportDataTypes.Value.t array option Lwt.t =
fun ~log_error ~conf ~dirname mapping db ->
let file = Filename.concat dirname (A.table mapping).file in
@@ -86,7 +85,9 @@ let importInDatable :
This line could generate an error if the headers are not correctly defined.
*)
let header =
- List.map ~f:(fun v -> CSV.DataType.Content v) (Csv.next csv_channel)
+ List.map
+ ~f:(fun v -> ImportDataTypes.Value.Content v)
+ (Csv.next csv_channel)
in
let state =
@@ -110,7 +111,7 @@ let importInDatable :
with
| Csv.Failure (line, row, cause) as e ->
Printf.eprintf "Error %s on line %d — field : %s\n" cause line
- (ImportCSV.Csv.column_to_string row);
+ (ImportDataTypes.Path.column_to_string row);
raise e
in
ignore @@ State.clear ~log_error db mapping conf;
diff --git a/lib/file_handler/csv2sql.mli b/lib/file_handler/csv2sql.mli
index e09737b..8976034 100644
--- a/lib/file_handler/csv2sql.mli
+++ b/lib/file_handler/csv2sql.mli
@@ -1,10 +1,10 @@
val importInDatable :
log_error:ImportErrors.t ->
- conf:ImportConf.Syntax.t ->
+ conf:ImporterSyntax.t ->
dirname:string ->
ImportAnalyser.Dependency.t ->
_ ImportSQL.Db.t ->
- ImportCSV.DataType.t array option Lwt.t
-(** Load an excel spreadsheet in an SQLite database.
+ ImportDataTypes.Value.t array option Lwt.t
+(** Load an excel spreadsheet in an SQLite database.
-Return the header if at least one row where present *)
+ Return the header if at least one row where present *)
diff --git a/lib/file_handler/dune b/lib/file_handler/dune
index 02be751..80b2ff6 100755
--- a/lib/file_handler/dune
+++ b/lib/file_handler/dune
@@ -10,13 +10,11 @@
lwt
lwt.unix
helpers
- importAnalyser
- importConf
- importContainers
- importCSV
importDataTypes
+ importContainers
importErrors
- importExpression
+ importerSyntax
+ importAnalyser
importSQL
)
)
diff --git a/lib/file_handler/state.ml b/lib/file_handler/state.ml
index 5b43aff..7cf57da 100644
--- a/lib/file_handler/state.ml
+++ b/lib/file_handler/state.ml
@@ -18,7 +18,7 @@ type insert_result = {
type ('a, 'b) mapper = {
get_row : 'b -> 'a Array.t;
- get_value : 'a -> ImportCSV.DataType.t;
+ get_value : 'a -> ImportDataTypes.Value.t;
default : 'a;
}
@@ -105,14 +105,15 @@ let insert_row :
sheet = state.sheet_number;
row = state.row_number;
target = None;
- value = CSV.DataType.Content (String.concat ~sep:", " []);
+ value = ImportDataTypes.Value.Content (String.concat ~sep:", " []);
exn = e;
})
execution
-(** Load the row with all the informations associated with this sheet.
+(** Load the row with all the informations associated with this sheet.
- If an error has already been raised during the sheet, ignore this row only. *)
+ If an error has already been raised during the sheet, ignore this row only.
+*)
let run_row :
log_error:ImportErrors.t ->
mapper:(_, 'row) mapper ->
@@ -147,20 +148,20 @@ let clear :
log_error:ImportErrors.t ->
'a ImportSQL.Db.t ->
A.t ->
- ImportConf.Syntax.t ->
+ ImporterSyntax.t ->
unit ImportSQL.Db.result =
fun ~log_error db mapping conf ->
ImportSQL.Db.clear_duplicates db (A.table mapping) (A.keys mapping)
~f:(fun values ->
let line =
match snd @@ Array.get values 0 with
- | ImportCSV.DataType.Integer i -> i
+ | ImportDataTypes.Value.Integer i -> i
| _ -> -1
and value = snd @@ Array.get values 1
and target =
match snd @@ Array.get values 2 with
- | ImportCSV.DataType.Content s ->
- Some (ImportConf.get_table_for_name conf (Some s))
+ | ImportDataTypes.Value.Content s ->
+ Some (ImporterSyntax.get_table_for_name conf (Some s))
| _ -> None
in
let error =
diff --git a/lib/file_handler/state.mli b/lib/file_handler/state.mli
index f744c33..8e24b1b 100644
--- a/lib/file_handler/state.mli
+++ b/lib/file_handler/state.mli
@@ -15,7 +15,7 @@ type insert_result = {
type ('a, 'b) mapper = {
get_row : 'b -> 'a Array.t;
- get_value : 'a -> ImportCSV.DataType.t;
+ get_value : 'a -> ImportDataTypes.Value.t;
default : 'a;
}
@@ -41,6 +41,7 @@ val clear :
log_error:ImportErrors.t ->
'a ImportSQL.Db.t ->
ImportAnalyser.Dependency.t ->
- ImportConf.Syntax.t ->
+ ImporterSyntax.t ->
unit ImportSQL.Db.result
-(** Clean up the table after the insertion, check for the duplicates and external references *)
+(** Clean up the table after the insertion, check for the duplicates and
+ external references *)
diff --git a/lib/file_handler/xlsx2sql.ml b/lib/file_handler/xlsx2sql.ml
index f2d8f12..651059f 100644
--- a/lib/file_handler/xlsx2sql.ml
+++ b/lib/file_handler/xlsx2sql.ml
@@ -1,8 +1,6 @@
open StdLabels
module A = ImportAnalyser.Dependency
module C = ImportContainers
-module CSV = ImportCSV
-module Syntax = ImportConf.Syntax
module Db = ImportSQL.Db
let flags = Unix.[ O_RDONLY; O_NONBLOCK ]
@@ -10,24 +8,26 @@ let flags = Unix.[ O_RDONLY; O_NONBLOCK ]
let extractors =
SZXX.Xlsx.
{
- string = (fun _location s -> CSV.DataType.Content s);
+ string = (fun _location s -> ImportDataTypes.Value.Content s);
error =
- (fun _location s -> CSV.DataType.Error (Printf.sprintf "#ERROR# %s" s));
+ (fun _location s ->
+ ImportDataTypes.Value.Error (Printf.sprintf "#ERROR# %s" s));
boolean =
(fun _location s ->
let value = String.(equal s "1") in
- CSV.DataType.Content (string_of_bool value));
+ ImportDataTypes.Value.Content (string_of_bool value));
number =
(fun _location s ->
let f = Float.of_string s in
- if Float.is_integer f then CSV.DataType.Integer (Float.to_int f)
- else CSV.DataType.Float f);
- date = (fun _location s -> CSV.DataType.Content s);
- null = CSV.DataType.Null;
+ if Float.is_integer f then
+ ImportDataTypes.Value.Integer (Float.to_int f)
+ else ImportDataTypes.Value.Float f);
+ date = (fun _location s -> ImportDataTypes.Value.Content s);
+ null = ImportDataTypes.Value.Null;
formula =
(fun _location ~formula s ->
ignore formula;
- CSV.DataType.Content s);
+ ImportDataTypes.Value.Content s);
}
let feed_bigstring ic =
@@ -48,29 +48,31 @@ let is_delayed row =
| _ -> false)
let default_mapper :
- (ImportCSV.DataType.t, ImportCSV.DataType.t SZXX.Xlsx.row) State.mapper =
+ ( ImportDataTypes.Value.t,
+ ImportDataTypes.Value.t SZXX.Xlsx.row )
+ State.mapper =
{
get_value =
(function
- | ImportCSV.DataType.Content s ->
- ImportCSV.DataType.Content (SZXX.Xml.unescape s)
+ | ImportDataTypes.Value.Content s ->
+ ImportDataTypes.Value.Content (SZXX.Xml.unescape s)
| any -> any);
- default = ImportCSV.DataType.Null;
+ default = ImportDataTypes.Value.Null;
get_row = (fun v -> v.SZXX.Xlsx.data);
}
-type state = CSV.DataType.t SZXX.Xlsx.status SZXX.Xlsx.row State.t
+type state = ImportDataTypes.Value.t SZXX.Xlsx.status SZXX.Xlsx.row State.t
let delayed_mapper =
State.
{
get_value =
(function
- | SZXX.Xlsx.Available (CSV.DataType.Content s) ->
- CSV.DataType.Content (SZXX.Xml.unescape s)
+ | SZXX.Xlsx.Available (ImportDataTypes.Value.Content s) ->
+ ImportDataTypes.Value.Content (SZXX.Xml.unescape s)
| SZXX.Xlsx.Available value -> value
- | _ -> CSV.DataType.Null);
- default = SZXX.Xlsx.Available CSV.DataType.Null;
+ | _ -> ImportDataTypes.Value.Null);
+ default = SZXX.Xlsx.Available ImportDataTypes.Value.Null;
get_row = (fun v -> v.SZXX.Xlsx.data);
}
@@ -96,11 +98,11 @@ let first_row : A.t -> _ Db.t -> state -> 'a SZXX.Xlsx.row -> state =
let importInDatable :
log_error:Csv.out_channel Lazy.t ->
- conf:Syntax.t ->
+ conf:ImporterSyntax.t ->
dirname:string ->
A.t ->
'a Db.t ->
- CSV.DataType.t array option Lwt.t =
+ ImportDataTypes.Value.t array option Lwt.t =
fun ~log_error ~conf ~dirname mapping db ->
let file = Filename.concat dirname (A.table mapping).file in
diff --git a/lib/file_handler/xlsx2sql.mli b/lib/file_handler/xlsx2sql.mli
index e09737b..8976034 100644
--- a/lib/file_handler/xlsx2sql.mli
+++ b/lib/file_handler/xlsx2sql.mli
@@ -1,10 +1,10 @@
val importInDatable :
log_error:ImportErrors.t ->
- conf:ImportConf.Syntax.t ->
+ conf:ImporterSyntax.t ->
dirname:string ->
ImportAnalyser.Dependency.t ->
_ ImportSQL.Db.t ->
- ImportCSV.DataType.t array option Lwt.t
-(** Load an excel spreadsheet in an SQLite database.
+ ImportDataTypes.Value.t array option Lwt.t
+(** Load an excel spreadsheet in an SQLite database.
-Return the header if at least one row where present *)
+ Return the header if at least one row where present *)
diff --git a/lib/sql/db.ml b/lib/sql/db.ml
index f1aee8c..f2a2653 100644
--- a/lib/sql/db.ml
+++ b/lib/sql/db.ml
@@ -1,6 +1,4 @@
open StdLabels
-module CSV = ImportCSV
-module Syntax = ImportConf.Syntax
module Table = ImportDataTypes.Table
module Path = ImportDataTypes.Path
@@ -153,7 +151,7 @@ let eval_key :
'a t ->
Sqlite3.stmt option ->
ImportAnalyser.Dependency.key list ->
- (int * CSV.DataType.t) list ->
+ (int * ImportDataTypes.Value.t) list ->
(Sqlite3.stmt option * Sqlite3.Data.t list) T.result =
fun db stmt keys values ->
match keys with
@@ -204,7 +202,7 @@ let insert :
Sqlite3.db ->
Sqlite3.stmt ->
id:int ->
- (int * CSV.DataType.t) list ->
+ (int * ImportDataTypes.Value.t) list ->
unit T.result =
fun db statement ~id values ->
let** _ = T.savepoint db "PREVIOUS" in
@@ -255,9 +253,9 @@ let execute_query :
Ok statement
let query :
- f:((Path.t ImportExpression.T.t * CSV.DataType.t) array -> unit) ->
+ f:((Path.t ImportExpression.T.t * ImportDataTypes.Value.t) array -> unit) ->
Sqlite3.db ->
- Syntax.t ->
+ ImporterSyntax.t ->
unit T.result =
fun ~f db output ->
(* Extract the query from the configuration. *)
@@ -280,7 +278,7 @@ let query :
in
Ok ()
-let create_view : Sqlite3.db -> Syntax.t -> unit T.result =
+let create_view : Sqlite3.db -> ImporterSyntax.t -> unit T.result =
fun db output ->
ignore output;
let* drop = Sqlite3.exec db "DROP VIEW IF EXISTS 'result'" in
@@ -300,10 +298,10 @@ let create_view : Sqlite3.db -> Syntax.t -> unit T.result =
*)
let check_foreign :
- f:((string * CSV.DataType.t) array -> unit) ->
+ f:((string * ImportDataTypes.Value.t) array -> unit) ->
Sqlite3.db ->
- Syntax.t ->
- Syntax.Extern.t ->
+ ImporterSyntax.t ->
+ ImporterSyntax.Extern.t ->
unit T.result =
fun ~f db conf external_ ->
let query = ImportAnalyser.Query.check_external conf external_ in
@@ -318,7 +316,7 @@ let check_foreign :
|> T.to_result
let clear_duplicates :
- f:((string * ImportCSV.DataType.t) array -> unit) ->
+ f:((string * ImportDataTypes.Value.t) array -> unit) ->
'a t ->
ImportDataTypes.Table.t ->
ImportAnalyser.Dependency.key list ->
diff --git a/lib/sql/db.mli b/lib/sql/db.mli
index 91933c4..213fb27 100644
--- a/lib/sql/db.mli
+++ b/lib/sql/db.mli
@@ -1,5 +1,3 @@
-module Syntax = ImportConf.Syntax
-
type 'a t
type 'a result = ('a, exn) Result.t
@@ -32,7 +30,7 @@ val eval_key :
'a t ->
Sqlite3.stmt option ->
ImportAnalyser.Dependency.key list ->
- (int * ImportCSV.DataType.t) list ->
+ (int * ImportDataTypes.Value.t) list ->
(Sqlite3.stmt option * Sqlite3.Data.t list) result
(** Evaluate the keys in sqlite and get the results.
@@ -43,7 +41,7 @@ val insert :
'a t ->
Sqlite3.stmt ->
id:int ->
- (int * ImportCSV.DataType.t) list ->
+ (int * ImportDataTypes.Value.t) list ->
unit result
(** Insert a new row in the database.
@@ -63,10 +61,11 @@ val rollback : 'a t -> unit result
val query :
f:
- ((ImportDataTypes.Path.t ImportExpression.T.t * ImportCSV.DataType.t) array ->
+ ((ImportDataTypes.Path.t ImportExpression.T.t * ImportDataTypes.Value.t)
+ array ->
unit) ->
'a t ->
- Syntax.t ->
+ ImporterSyntax.t ->
unit result
(** This one the most important function from the application. The function will
transform the configuration into an sql query and will fetch the result from
@@ -74,18 +73,18 @@ val query :
The function [f] given in argument will be called for each line *)
-val create_view : 'a t -> Syntax.t -> unit result
+val create_view : 'a t -> ImporterSyntax.t -> unit result
(** Create a view which represent the result *)
val check_foreign :
- f:((string * ImportCSV.DataType.t) array -> unit) ->
+ f:((string * ImportDataTypes.Value.t) array -> unit) ->
'a t ->
- Syntax.t ->
- Syntax.Extern.t ->
+ ImporterSyntax.t ->
+ ImporterSyntax.Extern.t ->
unit result
val clear_duplicates :
- f:((string * ImportCSV.DataType.t) array -> unit) ->
+ f:((string * ImportDataTypes.Value.t) array -> unit) ->
'a t ->
ImportDataTypes.Table.t ->
ImportAnalyser.Dependency.key list ->
@@ -95,9 +94,9 @@ val clear_duplicates :
val insert_header :
'a t ->
ImportDataTypes.Table.t ->
- (int * ImportCSV.DataType.t) array ->
+ (int * ImportDataTypes.Value.t) array ->
unit T.result
val query_headers :
- 'a t -> ImportDataTypes.Table.t -> ImportCSV.DataType.t array T.result
+ 'a t -> ImportDataTypes.Table.t -> ImportDataTypes.Value.t array T.result
(** Get all the headers from the database (used or not) *)
diff --git a/lib/sql/dune b/lib/sql/dune
index 9f9f205..bd77858 100644
--- a/lib/sql/dune
+++ b/lib/sql/dune
@@ -4,12 +4,11 @@
re
sqlite3
calendar
- importAnalyser
- importCSV
- importConf
- importContainers
importDataTypes
+ importContainers
importErrors
importExpression
+ importerSyntax
+ importAnalyser
)
)
diff --git a/lib/sql/hashs.ml b/lib/sql/hashs.ml
index af1f092..eced2b4 100644
--- a/lib/sql/hashs.ml
+++ b/lib/sql/hashs.ml
@@ -1,10 +1,8 @@
-(**
- This module store the hash of the indexes ([extern_key]) for each table in
+(** This module store the hash of the indexes ([extern_key]) for each table in
order to update the file if the configuration changed.
The hashes are stored in a table named [hashes] and are evaluated just
- before inserting the values.
-*)
+ before inserting the values. *)
open StdLabels
module Table = ImportDataTypes.Table
@@ -75,5 +73,5 @@ let query : 'a T.t -> ImportDataTypes.Table.t -> int option T.result =
let* _ = T.to_result state in
match res with
- | Some (ImportCSV.DataType.Integer i) -> Ok (Some i)
+ | Some (ImportDataTypes.Value.Integer i) -> Ok (Some i)
| _ -> Ok None
diff --git a/lib/sql/header.ml b/lib/sql/header.ml
index 3cac5fb..1f4c9d6 100644
--- a/lib/sql/header.ml
+++ b/lib/sql/header.ml
@@ -13,7 +13,7 @@ let create_table : 'a T.t -> unit T.result =
let insert_header :
'a T.t ->
ImportDataTypes.Table.t ->
- (int * ImportCSV.DataType.t) array ->
+ (int * ImportDataTypes.Value.t) array ->
unit T.result =
fun db table values ->
let table_name = Table.name table in
@@ -48,7 +48,8 @@ let insert_header :
T.commit db
let query_headers :
- 'a T.t -> ImportDataTypes.Table.t -> ImportCSV.DataType.t array T.result =
+ 'a T.t -> ImportDataTypes.Table.t -> ImportDataTypes.Value.t array T.result
+ =
fun db table ->
let table_name = Table.name table in
let query =
diff --git a/lib/sql/t.ml b/lib/sql/t.ml
index 202c535..2297125 100644
--- a/lib/sql/t.ml
+++ b/lib/sql/t.ml
@@ -39,14 +39,14 @@ let reset : Sqlite3.stmt -> unit result =
fun statement -> to_result (Sqlite3.reset statement)
let of_datatype = function
- | ImportCSV.DataType.Float f -> Sqlite3.Data.FLOAT f
- | ImportCSV.DataType.Integer i -> Sqlite3.Data.INT (Int64.of_int i)
- | ImportCSV.DataType.Null -> Sqlite3.Data.NULL
- | ImportCSV.DataType.Error _ -> Sqlite3.Data.NULL
- | ImportCSV.DataType.Content s -> Sqlite3.Data.TEXT s
-
-let to_datatype : Sqlite3.Data.t -> ImportCSV.DataType.t = function
- | Sqlite3.Data.NONE | Sqlite3.Data.NULL -> ImportCSV.DataType.Null
- | Sqlite3.Data.INT i -> ImportCSV.DataType.Integer (Int64.to_int i)
- | Sqlite3.Data.FLOAT f -> ImportCSV.DataType.Float f
- | Sqlite3.Data.TEXT t | Sqlite3.Data.BLOB t -> ImportCSV.DataType.Content t
+ | ImportDataTypes.Value.Float f -> Sqlite3.Data.FLOAT f
+ | ImportDataTypes.Value.Integer i -> Sqlite3.Data.INT (Int64.of_int i)
+ | ImportDataTypes.Value.Null -> Sqlite3.Data.NULL
+ | ImportDataTypes.Value.Error _ -> Sqlite3.Data.NULL
+ | ImportDataTypes.Value.Content s -> Sqlite3.Data.TEXT s
+
+let to_datatype : Sqlite3.Data.t -> ImportDataTypes.Value.t = function
+ | Sqlite3.Data.NONE | Sqlite3.Data.NULL -> ImportDataTypes.Value.Null
+ | Sqlite3.Data.INT i -> ImportDataTypes.Value.Integer (Int64.to_int i)
+ | Sqlite3.Data.FLOAT f -> ImportDataTypes.Value.Float f
+ | Sqlite3.Data.TEXT t | Sqlite3.Data.BLOB t -> ImportDataTypes.Value.Content t
diff --git a/lib/configuration/cte.ml b/lib/syntax/cte.ml
index ff43d6d..ff43d6d 100644
--- a/lib/configuration/cte.ml
+++ b/lib/syntax/cte.ml
diff --git a/lib/configuration/cte.mli b/lib/syntax/cte.mli
index 0f2b3e3..39897ef 100644
--- a/lib/configuration/cte.mli
+++ b/lib/syntax/cte.mli
@@ -1,9 +1,6 @@
-module Path = ImportDataTypes.Path
-module Expression = ImportExpression.T
-
type t = {
- filters : Path.t Expression.t list;
- group : Path.t Expression.t option;
+ filters : ImportDataTypes.Path.t ImportExpression.T.t list;
+ group : ImportDataTypes.Path.t ImportExpression.T.t option;
}
(** Represent a filter to apply in the querry
@@ -17,4 +14,4 @@ type t = {
because all cf them can be evaluated at the same time, but as soon we have a
group function, the result become dependant of the previous ones. *)
-val of_filters : Path.t Expression.t list -> t list
+val of_filters : ImportDataTypes.Path.t ImportExpression.T.t list -> t list
diff --git a/lib/syntax/dune b/lib/syntax/dune
new file mode 100755
index 0000000..1459d0c
--- /dev/null
+++ b/lib/syntax/dune
@@ -0,0 +1,12 @@
+(library
+ (name importerSyntax)
+ (libraries
+ otoml
+ importDataTypes
+ importExpression
+ )
+ (preprocess (pps
+ ppx_deriving.show
+ ppx_deriving.eq
+ ))
+)
diff --git a/lib/configuration/syntax.ml b/lib/syntax/importerSyntax.ml
index ee47277..7788613 100644
--- a/lib/configuration/syntax.ml
+++ b/lib/syntax/importerSyntax.ml
@@ -2,6 +2,7 @@ open StdLabels
module E = ImportExpression.T
module Table = ImportDataTypes.Table
module Path = ImportDataTypes.Path
+module CTE = Cte
let toml_of_table Table.{ file; tab; name } =
let values = [ ("file", Otoml.string file); ("name", Otoml.string name) ] in
@@ -33,7 +34,7 @@ module Extern = struct
( "extern_key",
Otoml.string
@@ ImportExpression.Repr.repr ~top:true
- (fun v -> ":" ^ ImportCSV.Csv.column_to_string v)
+ (fun v -> ":" ^ ImportDataTypes.Path.column_to_string v)
extern.extern_key );
("file", Otoml.string extern.target.file);
("allow_missing", Otoml.boolean extern.allow_missing);
@@ -91,3 +92,46 @@ let repr t =
in
Otoml.table values
+
+let get_table_for_name : t -> string option -> Table.t =
+ fun conf name ->
+ match name with
+ | None -> conf.source
+ | Some name ->
+ if String.equal name conf.source.name then conf.source
+ else
+ let ext =
+ List.find conf.externals ~f:(fun (ext : Extern.t) ->
+ String.equal name ext.target.name)
+ in
+ ext.target
+
+let root_table : t -> Table.t = fun conf -> conf.source
+
+let get_dependancies_for_table : t -> Table.t -> Extern.t list =
+ fun conf source ->
+ let is_root = source = conf.source in
+
+ List.filter conf.externals ~f:(fun (ext : Extern.t) ->
+ (* Enumerate the intern_key and check the source pointed by each column *)
+ ImportExpression.T.fold_values ext.intern_key ~init:false
+ ~f:(fun acc expr ->
+ if acc then acc
+ else
+ match expr.Path.alias with
+ | Some v -> String.equal v source.name
+ | None -> is_root))
+
+let latest_version = 1
+
+let dummy_conf =
+ {
+ source = { file = ""; tab = 0; name = "" };
+ version = latest_version;
+ locale = Some "C";
+ externals = [];
+ columns = [];
+ filters = [];
+ sort = [];
+ uniq = [];
+ }
diff --git a/lib/syntax/importerSyntax.mli b/lib/syntax/importerSyntax.mli
new file mode 100644
index 0000000..49b7364
--- /dev/null
+++ b/lib/syntax/importerSyntax.mli
@@ -0,0 +1,37 @@
+module Extern : sig
+ type t = {
+ intern_key : ImportDataTypes.Path.t ImportExpression.T.t;
+ target : ImportDataTypes.Table.t;
+ extern_key : ImportDataTypes.Path.column ImportExpression.T.t;
+ allow_missing : bool;
+ match_rule : string option;
+ }
+ [@@deriving show, eq]
+end
+
+module CTE = Cte
+
+type t = {
+ version : int;
+ locale : string option;
+ source : ImportDataTypes.Table.t;
+ externals : Extern.t list;
+ columns : ImportDataTypes.Path.t ImportExpression.T.t list;
+ filters : ImportDataTypes.Path.t ImportExpression.T.t list;
+ sort : ImportDataTypes.Path.t ImportExpression.T.t list;
+ uniq : ImportDataTypes.Path.t ImportExpression.T.t list;
+}
+
+val repr : t -> Otoml.t
+
+val root_table : t -> ImportDataTypes.Table.t
+(** Get the root table, this table is the main table to load and each line in
+ this table will be processed *)
+
+val get_table_for_name : t -> string option -> ImportDataTypes.Table.t
+
+val get_dependancies_for_table : t -> ImportDataTypes.Table.t -> Extern.t list
+(** Get all the externals refered by the source *)
+
+val dummy_conf : t
+val latest_version : int
diff --git a/tests/analyser_dependency.ml b/tests/analyser_dependency.ml
index 8409d73..511b706 100644
--- a/tests/analyser_dependency.ml
+++ b/tests/analyser_dependency.ml
@@ -1,7 +1,7 @@
open StdLabels
module A = ImportAnalyser.Dependency
module Cont = ImportContainers
-module Syntax = ImportConf.Syntax
+module Syntax = ImporterSyntax
module Expression = ImportExpression.T
module Table = ImportDataTypes.Table
open ConfLoader
diff --git a/tests/analyser_filters.ml b/tests/analyser_filters.ml
index c329af8..864cab7 100644
--- a/tests/analyser_filters.ml
+++ b/tests/analyser_filters.ml
@@ -1,4 +1,3 @@
-module CTE = ImportConf.CTE
module Filters = ImportAnalyser.Filters
module Chunk = ImportAnalyser.Chunk
@@ -29,7 +28,9 @@ let empty_predicates () =
The path is identified as pointing to a numeric column, and the associated
query will replace null with 0. *)
let simple_filter () =
- let filter = CTE.of_filters Expression_builder.[ equal integer_one path' ] in
+ let filter =
+ ImporterSyntax.CTE.of_filters Expression_builder.[ equal integer_one path' ]
+ in
let chunk_links = Chunk.create () in
let chunk_predicates = Filters.generate_sql ~conf filter chunk_links in
@@ -47,7 +48,8 @@ let simple_filter () =
query will replace null with empty string. *)
let multiple_filters () =
let filter =
- CTE.of_filters Expression_builder.[ integer_one; equal path' literal_zero ]
+ ImporterSyntax.CTE.of_filters
+ Expression_builder.[ integer_one; equal path' literal_zero ]
in
let chunk_links = Chunk.create () in
let chunk_predicates = Filters.generate_sql ~conf filter chunk_links in
@@ -66,7 +68,7 @@ let multiple_filters () =
(** Create a simple configuration with a group expression and no other filters
*)
let group_filter () =
- let filter = CTE.of_filters [ group_expression ] in
+ let filter = ImporterSyntax.CTE.of_filters [ group_expression ] in
let chunk_links = Chunk.create () in
let chunk_predicates = Filters.generate_sql ~conf filter chunk_links in
@@ -90,7 +92,7 @@ let group_filter () =
let expression_with_group () =
let filter =
- CTE.of_filters
+ ImporterSyntax.CTE.of_filters
Expression_builder.[ equal integer_one path'; group_expression ]
in
let chunk_links = Chunk.create () in
@@ -118,7 +120,7 @@ let expression_with_group () =
included inside the CTE but in the main query. *)
let group_with_expression () =
let filter =
- CTE.of_filters
+ ImporterSyntax.CTE.of_filters
Expression_builder.[ group_expression; equal integer_one path' ]
in
let chunk_links = Chunk.create () in
@@ -145,7 +147,9 @@ let group_with_expression () =
(** Test the configuration with two group, each one generating it’s own CTE in
the query *)
let group_with_group () =
- let filter = CTE.of_filters [ group_expression; group_expression ] in
+ let filter =
+ ImporterSyntax.CTE.of_filters [ group_expression; group_expression ]
+ in
let chunk_links = Chunk.create () in
let chunk_predicates = Filters.generate_sql ~conf filter chunk_links in
diff --git a/tests/analyser_query_test.ml b/tests/analyser_query_test.ml
index 16a9bca..fd8914b 100644
--- a/tests/analyser_query_test.ml
+++ b/tests/analyser_query_test.ml
@@ -1,8 +1,7 @@
open StdLabels
module A = ImportAnalyser.Dependency
module Q = ImportAnalyser.Query
-module C = ImportConf
-module Syntax = ImportConf.Syntax
+module Syntax = ImporterSyntax
module Expr = Expression_builder
open Test_migration
diff --git a/tests/confLoader.ml b/tests/confLoader.ml
index 692cda9..13f9840 100644
--- a/tests/confLoader.ml
+++ b/tests/confLoader.ml
@@ -1,8 +1,8 @@
-let load' : string -> (ImportConf.Syntax.t, string) Result.t =
+let load' : string -> (ImporterSyntax.t, string) Result.t =
fun content -> Otoml.Parser.from_string content |> ImportConf.t_of_toml
(** Read the configuration in toml and return the internal representation *)
-let load : string -> ImportConf.Syntax.t =
+let load : string -> ImporterSyntax.t =
fun content -> Result.get_ok (load' content)
let conf =
@@ -37,7 +37,7 @@ let external_table_other =
ImportDataTypes.Table.{ file = "other.xlsx"; tab = 1; name = "other" }
let external_other =
- ImportConf.Syntax.Extern.
+ ImporterSyntax.Extern.
{
intern_key = Path { alias = None; column = 1 };
target = external_table_other;
@@ -50,7 +50,7 @@ let external_table_last =
ImportDataTypes.Table.{ file = "last.xlsx"; tab = 1; name = "last_file" }
let external_last =
- ImportConf.Syntax.Extern.
+ ImporterSyntax.Extern.
{
intern_key = Path { alias = Some "other"; column = 1 };
target = external_table_last;
diff --git a/tests/configuration_toml.ml b/tests/configuration_toml.ml
index e51c727..0a36faf 100644
--- a/tests/configuration_toml.ml
+++ b/tests/configuration_toml.ml
@@ -34,7 +34,7 @@ let test_suit =
| Error s -> raise (Failure s)
| Ok result ->
let expected =
- ImportConf.Syntax.Extern.
+ ImporterSyntax.Extern.
{
target = { file = ""; tab = 1; name = "target" };
extern_key = Literal "_B";
diff --git a/tests/dune b/tests/dune
index 78291eb..5c48308 100644
--- a/tests/dune
+++ b/tests/dune
@@ -6,10 +6,10 @@
otoml
sqlite3
fmt
+ importerSyntax
importConf
importAnalyser
importContainers
- importCSV
importDataTypes
importErrors
importExpression
diff --git a/tests/expression_repr.ml b/tests/expression_repr.ml
index 20a0484..9112704 100644
--- a/tests/expression_repr.ml
+++ b/tests/expression_repr.ml
@@ -4,7 +4,7 @@ module Expr = Expression_builder
module M = Expr.Make (ImportExpression.Repr.E)
open Test_migration
-let eval = M.eval ~path_repr:ImportCSV.Csv.column_to_string
+let eval = M.eval ~path_repr:ImportDataTypes.Path.column_to_string
let test_expr expr = ImportExpression.Repr.E.observe ~top:true expr
let assert_equal expected actual =
diff --git a/tests/importCSV_test.ml b/tests/importCSV_test.ml
index 8d83688..748c58b 100644
--- a/tests/importCSV_test.ml
+++ b/tests/importCSV_test.ml
@@ -1,4 +1,3 @@
-open ImportCSV
open Test_migration
let assert_equal = Alcotest.(check int "")
@@ -8,19 +7,23 @@ let float_repr =
let () =
Alcotest.(check string)
"Float repr" "3873921.620000"
- ImportCSV.DataType.(to_string "C" (Float 3873921.62))
+ ImportDataTypes.Value.(to_string "C" (Float 3873921.62))
in
()
let test_suit =
[
- ("Column A" >:: fun () -> assert_equal 1 (Csv.column_of_string "A"));
- ("Column a" >:: fun () -> assert_equal 1 (Csv.column_of_string "a"));
+ ( "Column A" >:: fun () ->
+ assert_equal 1 (ImportDataTypes.Path.column_of_string "A") );
+ ( "Column a" >:: fun () ->
+ assert_equal 1 (ImportDataTypes.Path.column_of_string "a") );
( "Column name" >:: fun () ->
let () =
for i = 1 to 1_000 do
- let column_name = Csv.column_to_string i in
- let column_index = Csv.column_of_string column_name in
+ let column_name = ImportDataTypes.Path.column_to_string i in
+ let column_index =
+ ImportDataTypes.Path.column_of_string column_name
+ in
assert_equal i column_index
done
diff --git a/tests/importConf_test.ml b/tests/importConf_test.ml
index 481b66c..5f98423 100644
--- a/tests/importConf_test.ml
+++ b/tests/importConf_test.ml
@@ -9,13 +9,14 @@ let check = Alcotest.(check (list Test_migration.extern_testable) "")
configuration. *)
let test_get_dependencies_for_source =
"get_dependancies_for_table" >:: fun _ ->
- let result = ImportConf.get_dependancies_for_table conf conf.source
+ let result = ImporterSyntax.get_dependancies_for_table conf conf.source
and expected = [ external_other ] in
check expected result
let test_get_dependencies_for_other =
"get_dependancies_for_table" >:: fun _ ->
- let result = ImportConf.get_dependancies_for_table conf external_table_other
+ let result =
+ ImporterSyntax.get_dependancies_for_table conf external_table_other
and expected = [ external_last ] in
check expected result
diff --git a/tests/sql_db.ml b/tests/sql_db.ml
index ab402bc..34f5b12 100644
--- a/tests/sql_db.ml
+++ b/tests/sql_db.ml
@@ -67,10 +67,10 @@ columns = [
":B",
":E"]|}
~input:
- ImportCSV.DataType.
+ ImportDataTypes.Value.
[ [ (0, Integer 123); (1, Integer 2); (4, Integer 5) ] ]
~expected:
- (Ok ImportCSV.DataType.[ [| Content "123_"; Integer 2; Integer 5 |] ])
+ (Ok ImportDataTypes.Value.[ [| Content "123_"; Integer 2; Integer 5 |] ])
(** Ensure the behavior of the sum function when a filter is given. It is
expected to accumulate the values over each line *)
@@ -87,7 +87,7 @@ columns = [
"sum(:C, [:B], [:A])",
]|}
~input:
- ImportCSV.DataType.
+ ImportDataTypes.Value.
[
[ (0, Integer 1); (1, Content "A"); (2, Integer 100) ];
[ (0, Integer 2); (1, Content "A"); (2, Integer 100) ];
@@ -95,7 +95,7 @@ columns = [
]
~expected:
(Ok
- ImportCSV.DataType.
+ ImportDataTypes.Value.
[
[| Integer 1; Integer 100 |];
[| Integer 2; Integer 200 |];
@@ -115,13 +115,18 @@ columns = [
"sum(:C, [], [])",
]|}
~input:
- ImportCSV.DataType.
+ ImportDataTypes.Value.
[
[ (0, Integer 1); (2, Integer 100) ];
[ (0, Integer 2); (2, Integer 100) ];
]
~expected:
- (Ok [ [| ImportCSV.DataType.Integer 1; ImportCSV.DataType.Integer 200 |] ])
+ (Ok
+ [
+ [|
+ ImportDataTypes.Value.Integer 1; ImportDataTypes.Value.Integer 200;
+ |];
+ ])
(** Ensure the behavior of the sum function when no filter is given. It is
expected to get the total sum for each line *)
@@ -138,7 +143,7 @@ columns = [
"sum(:C, [], [:A])",
]|}
~input:
- ImportCSV.DataType.
+ ImportDataTypes.Value.
[
[ (0, Integer 1); (2, Integer 100) ];
[ (0, Integer 2); (2, Integer 100) ];
@@ -146,8 +151,12 @@ columns = [
~expected:
(Ok
[
- [| ImportCSV.DataType.Integer 1; ImportCSV.DataType.Integer 100 |];
- [| ImportCSV.DataType.Integer 2; ImportCSV.DataType.Integer 200 |];
+ [|
+ ImportDataTypes.Value.Integer 1; ImportDataTypes.Value.Integer 100;
+ |];
+ [|
+ ImportDataTypes.Value.Integer 2; ImportDataTypes.Value.Integer 200;
+ |];
])
let sum_group =
@@ -163,7 +172,7 @@ columns = [
"sum(:C, [:A], [])",
]|}
~input:
- ImportCSV.DataType.
+ ImportDataTypes.Value.
[
[ (0, Integer 1); (2, Integer 100) ];
[ (0, Integer 1); (2, Integer 100) ];
@@ -171,7 +180,7 @@ columns = [
]
~expected:
(Ok
- ImportCSV.DataType.
+ ImportDataTypes.Value.
[
[| Integer 1; Integer 200 |];
[| Integer 1; Integer 200 |];
@@ -193,7 +202,7 @@ columns = [
"sum(:C, [:A], [])",
]|}
~input:
- ImportCSV.DataType.
+ ImportDataTypes.Value.
[
[ (0, Integer 1); (2, Integer 100) ];
[ (0, Integer 1); (2, Integer 100) ];
@@ -201,7 +210,7 @@ columns = [
]
~expected:
(Ok
- ImportCSV.DataType.
+ ImportDataTypes.Value.
[ [| Integer 1; Integer 200 |]; [| Integer 2; Integer 100 |] ])
let filter_group =
@@ -222,13 +231,13 @@ filters = [
|}
~input:
- ImportCSV.DataType.
+ ImportDataTypes.Value.
[
[ (0, Integer 1); (1, Integer 100) ];
[ (0, Integer 2); (1, Integer 150) ];
[ (0, Integer 3); (1, Integer 200) ];
]
- ~expected:(Ok ImportCSV.DataType.[ [| Integer 3 |] ])
+ ~expected:(Ok ImportDataTypes.Value.[ [| Integer 3 |] ])
(** The first filter will prevent the max value to pop, and only the second one
will be reported *)
@@ -251,13 +260,13 @@ filters = [
|}
~input:
- ImportCSV.DataType.
+ ImportDataTypes.Value.
[
[ (0, Integer 1); (1, Integer 100) ];
[ (0, Integer 2); (1, Integer 150) ];
[ (0, Integer 3); (1, Integer 200) ];
]
- ~expected:(Ok ImportCSV.DataType.[ [| Integer 2 |] ])
+ ~expected:(Ok ImportDataTypes.Value.[ [| Integer 2 |] ])
(** In this case, we first filter the line and keep only the max value, but the
second filter will match the result and will produce an empty list *)
@@ -280,7 +289,7 @@ filters = [
|}
~input:
- ImportCSV.DataType.
+ ImportDataTypes.Value.
[
[ (0, Integer 1); (1, Integer 100) ];
[ (0, Integer 2); (1, Integer 150) ];
@@ -311,13 +320,13 @@ filters = [
|}
~input:
- ImportCSV.DataType.
+ ImportDataTypes.Value.
[
[ (0, Integer 1); (1, Integer 100) ];
[ (0, Integer 2); (1, Integer 150) ];
[ (0, Integer 3); (1, Integer 200) ];
]
- ~expected:(Ok [ [| ImportCSV.DataType.Integer 1 |] ])
+ ~expected:(Ok [ [| ImportDataTypes.Value.Integer 1 |] ])
let test_suit =
[
diff --git a/tests/test_migration.ml b/tests/test_migration.ml
index e26f354..17e48cc 100644
--- a/tests/test_migration.ml
+++ b/tests/test_migration.ml
@@ -27,7 +27,7 @@ let sql_testable =
let csv_data_type_testable =
make_test
(module struct
- type t = ImportCSV.DataType.t =
+ type t = ImportDataTypes.Value.t =
| Null
| Error of string
| Content of string
@@ -38,7 +38,7 @@ let csv_data_type_testable =
let csv_result = Alcotest.(list @@ array @@ csv_data_type_testable)
let data_type_testable = make_test (module ImportDataTypes.Types)
-let extern_testable = make_test (module ImportConf.Syntax.Extern)
+let extern_testable = make_test (module ImporterSyntax.Extern)
let table_testable = make_test (module ImportDataTypes.Table)
let int_container_testable = make_test (module ImportContainers.IntSet)
@@ -70,11 +70,11 @@ let chunk =
let syntax =
make_test
(module struct
- type t = ImportConf.Syntax.t
+ type t = ImporterSyntax.t
let pp format t =
Format.fprintf format "%s"
- (Otoml.Printer.to_string (ImportConf.Syntax.repr t))
+ (Otoml.Printer.to_string (ImporterSyntax.repr t))
let equal t1 t2 = t1 = t2
end)