diff options
author | Sébastien Dailly <sebastien@dailly.me> | 2025-03-17 09:11:25 +0100 |
---|---|---|
committer | Sébastien Dailly <sebastien@dailly.me> | 2025-03-17 18:59:32 +0100 |
commit | 8b8b730d3ba98d6c9e4e6274844641043b5fefbb (patch) | |
tree | 4cb60dafa05b479d0ca287d501a51db88cecaaa4 | |
parent | 7bfbb67d83011f3e1845dcb9e44c3b6a5e93a9da (diff) |
Moved the syntax module in its own library
66 files changed, 442 insertions, 418 deletions
@@ -17,15 +17,15 @@ sqlite3
tools
helpers
+ importerSyntax
importConf
importAnalyser
importContainers
importDataTypes
- importCSV
importErrors
importExpression
- importFileHandler
importSQL
+ importFileHandler
)
(link_flags (:standard))
)
diff --git a/bin/importer.ml b/bin/importer.ml index 8549643..0da2ab7 100644 --- a/bin/importer.ml +++ b/bin/importer.ml @@ -24,18 +24,18 @@ let creation_date file = module Args = struct type arguments = { - configuration : ImportConf.Syntax.t; + configuration : ImporterSyntax.t; conf_name : string; bom : bool; print_conf : bool; mapping_date : float option; } - let load_conf : string -> ImportConf.Syntax.t = + let load_conf : string -> ImporterSyntax.t = fun file -> match Filename.extension file with | _ -> ( - let (conf : (Db.Syntax.t, string) result) = + let (conf : (ImporterSyntax.t, string) result) = let* configuration_file = try Ok (Otoml.Parser.from_file (exists file)) with | Otoml.Parse_error (position, message) -> @@ -60,7 +60,7 @@ module Args = struct | Ok e -> e) let load () = - let conf = ref ("", ImportConf.dummy_conf) + let conf = ref ("", ImporterSyntax.dummy_conf) and bom = ref true and usage = "importer [--conf configuration.toml]" and print_conf = ref false in @@ -101,11 +101,11 @@ end function is expected to convert the result into string in order to include the content in the output CSV. *) let printer : - string -> Path.t ImportExpression.T.t * ImportCSV.DataType.t -> string = + string -> Path.t ImportExpression.T.t * ImportDataTypes.Value.t -> string = fun locale (column, value) -> ignore column; (* Set the output locale accoording the configuration file *) - ImportCSV.DataType.to_string locale value + ImportDataTypes.Value.to_string locale value let bom = "\xEF\xBB\xBF" @@ -137,7 +137,7 @@ let process_table : table. *) match ImportSQL.Db.query_headers db source with | Ok v -> - let f = ImportCSV.DataType.to_string locale in + let f = ImportDataTypes.Value.to_string locale in let text_headers = Array.map v ~f in Headers.SheeetMap.add source text_headers map | Error _ -> map) @@ -163,24 +163,23 @@ let process_table : match headers_opt with | None -> map | Some v -> - let f = ImportCSV.DataType.to_string locale in + let f = ImportDataTypes.Value.to_string locale in let text_headers = Array.map v ~f in Headers.SheeetMap.add source text_headers map in headers let check_deps : - 'a Db.t -> Csv.out_channel Lazy.t -> ImportConf.Syntax.t -> Table.t -> unit - = + 'a Db.t -> Csv.out_channel Lazy.t -> ImporterSyntax.t -> Table.t -> unit = fun db log_error conf source -> (* For each external check if the values are loaded *) - let dependancies = ImportConf.get_dependancies_for_table conf source in + let dependancies = ImporterSyntax.get_dependancies_for_table conf source in List.iter dependancies ~f:(fun ext -> - match ext.ImportConf.Syntax.Extern.allow_missing with + match ext.ImporterSyntax.Extern.allow_missing with | true -> () | false -> ( Printf.printf "Checking dependancies for %s\n%!" - ext.ImportConf.Syntax.Extern.target.ImportDataTypes.Table.name; + ext.ImporterSyntax.Extern.target.ImportDataTypes.Table.name; try ignore @@ Db.check_foreign db conf ext ~f:(fun values -> @@ -188,7 +187,7 @@ let check_deps : let row = match snd (Array.get values 0) with - | ImportCSV.DataType.Integer i -> i + | ImportDataTypes.Value.Integer i -> i | _ -> -1 and value = snd (Array.get values 1) in let error = @@ -198,11 +197,11 @@ let check_deps : sheet = source.Table.tab; row; value; - target = Some ext.ImportConf.Syntax.Extern.target; + target = Some ext.ImporterSyntax.Extern.target; exn = Failure (Printf.sprintf "Key '%s' not found" - (CSV.DataType.to_string "C" value)); + (ImportDataTypes.Value.to_string "C" value)); } in @@ -232,7 +231,7 @@ let () = (* With the printconf option, we do not need to open any file *) if conf.print_conf then ( - let toml = ImportConf.Syntax.repr conf.configuration in + let toml = ImporterSyntax.repr conf.configuration in Otoml.Printer.to_channel ~collapse_tables:true stdout toml; exit 0); diff --git a/lib/analysers/chunk.ml b/lib/analysers/chunk.ml index 2fa4808..cefa6d8 100644 --- a/lib/analysers/chunk.ml +++ b/lib/analysers/chunk.ml @@ -8,13 +8,13 @@ type t = { b : Buffer.t; - parameters : ImportCSV.DataType.t Queue.t; + parameters : ImportDataTypes.Value.t Queue.t; } let create : unit -> t = fun () -> { b = Buffer.create 16; parameters = Queue.create () } -let create' : Buffer.t -> ImportCSV.DataType.t Queue.t -> t = +let create' : Buffer.t -> ImportDataTypes.Value.t Queue.t -> t = fun b parameters -> { b; parameters } (* Append the element from [tail] at the end of [head] @@ -39,10 +39,9 @@ let copy : t -> t = Buffer.add_buffer b t.b; { b; parameters } -let add_parameters : t -> ImportCSV.DataType.t Seq.t -> unit = +let add_parameters : t -> ImportDataTypes.Value.t Seq.t -> unit = fun t p -> Queue.add_seq t.parameters p -module Syntax = ImportConf.Syntax module Table = ImportDataTypes.Table module Q = ImportExpression.Query open StdLabels @@ -53,7 +52,8 @@ open StdLabels - the join query in order to load the data from the external column - the column corresponding to the key in order to identify the missing links later. *) -let join_external : conf:Syntax.t -> join_buffer:t -> Syntax.Extern.t -> unit = +let join_external : + conf:ImporterSyntax.t -> join_buffer:t -> ImporterSyntax.Extern.t -> unit = fun ~conf ~join_buffer external_ -> let extern_table = Table.name external_.target in @@ -67,8 +67,8 @@ let join_external : conf:Syntax.t -> join_buffer:t -> Syntax.Extern.t -> unit = in add_parameters join_buffer (Queue.to_seq q))) - (Table.print_column external_.Syntax.Extern.target - ("key_" ^ external_.Syntax.Extern.target.name)); + (Table.print_column external_.ImporterSyntax.Extern.target + ("key_" ^ external_.ImporterSyntax.Extern.target.name)); Format.pp_print_flush formatter () @@ -78,7 +78,7 @@ let join_external : conf:Syntax.t -> join_buffer:t -> Syntax.Extern.t -> unit = SQLite is able to optimize the query and do not load the table not used in the select clause. *) let create_from_statement_of_chunck : - ?externals:Syntax.Extern.t list -> Syntax.t -> t -> unit = + ?externals:ImporterSyntax.Extern.t list -> ImporterSyntax.t -> t -> unit = fun ?externals conf c -> let externals = Option.value externals ~default:conf.externals in add_string c "\nFROM '"; @@ -91,7 +91,10 @@ let create_from_statement_of_chunck : List.iter externals ~f:(join_external ~conf ~join_buffer:c) let add_expression : - conf:Syntax.t -> t -> ImportDataTypes.Path.t ImportExpression.T.t -> unit = + conf:ImporterSyntax.t -> + t -> + ImportDataTypes.Path.t ImportExpression.T.t -> + unit = fun ~conf group expression -> let formatter = Format.formatter_of_buffer group.b in let queue = diff --git a/lib/analysers/chunk.mli b/lib/analysers/chunk.mli index d4f69e7..13a748a 100644 --- a/lib/analysers/chunk.mli +++ b/lib/analysers/chunk.mli @@ -2,11 +2,11 @@ type t = { b : Buffer.t; - parameters : ImportCSV.DataType.t Queue.t; + parameters : ImportDataTypes.Value.t Queue.t; } val create : unit -> t -val create' : Buffer.t -> ImportCSV.DataType.t Queue.t -> t +val create' : Buffer.t -> ImportDataTypes.Value.t Queue.t -> t val append : head:t -> tail:t -> unit (** Append the element from [tail] at the end of [head] @@ -19,7 +19,7 @@ val add_string : t -> string -> unit val copy : t -> t val create_from_statement_of_chunck : - ?externals:ImportConf.Syntax.Extern.t list -> ImportConf.Syntax.t -> t -> unit + ?externals:ImporterSyntax.Extern.t list -> ImporterSyntax.t -> t -> unit (** Create the from part of the query, adding all the declared externals (even when not required) @@ -27,7 +27,7 @@ val create_from_statement_of_chunck : the select clause. *) val add_expression : - conf:ImportConf.Syntax.t -> + conf:ImporterSyntax.t -> t -> ImportDataTypes.Path.t ImportExpression.T.t -> unit diff --git a/lib/analysers/dependency.ml b/lib/analysers/dependency.ml index d0ea8b3..38bc23c 100644 --- a/lib/analysers/dependency.ml +++ b/lib/analysers/dependency.ml @@ -1,6 +1,5 @@ open StdLabels module IntSet = ImportContainers.IntSet -module Syntax = ImportConf.Syntax module Table = ImportDataTypes.Table module Path = ImportDataTypes.Path module Expression = ImportExpression.T @@ -61,11 +60,15 @@ type 'a expression_extractor = { The function may raise [Unknown_source] if the the path describe an unknown table. *) let add_path_in_map : - f:'a expression_extractor -> conf:Syntax.t -> 'a -> build_map -> build_map = + f:'a expression_extractor -> + conf:ImporterSyntax.t -> + 'a -> + build_map -> + build_map = fun ~f ~conf path map -> let table_source, column = f.of_path path in let table = - try ImportConf.get_table_for_name conf table_source with + try ImporterSyntax.get_table_for_name conf table_source with | Not_found -> raise (ImportErrors.Unknown_source (Option.get table_source)) in @@ -81,7 +84,7 @@ let add_path_in_map : let add_expression_in_map : f:'a expression_extractor -> - conf:Syntax.t -> + conf:ImporterSyntax.t -> 'a Expression.t -> build_map -> build_map = @@ -91,7 +94,7 @@ let add_expression_in_map : let add_columns_in_map : f:'a expression_extractor -> - conf:Syntax.t -> + conf:ImporterSyntax.t -> 'a Expression.t list -> build_map -> build_map = @@ -109,9 +112,9 @@ let add_columns_in_map : This function is called for each path declared inside the expression. *) let add_dependancies : - conf:Syntax.t -> Syntax.Extern.t -> deps -> Path.t -> deps = + conf:ImporterSyntax.t -> ImporterSyntax.Extern.t -> deps -> Path.t -> deps = fun ~conf extern graph path -> - let source_table = ImportConf.get_table_for_name conf path.Path.alias in + let source_table = ImporterSyntax.get_table_for_name conf path.Path.alias in let source = ImportContainers.Source.from_table source_table in let target = ImportContainers.Source.from_table extern.target in @@ -121,14 +124,17 @@ let add_dependancies : | _ -> (target, [ source ]) :: graph let add_external_in_map : - conf:Syntax.t -> Syntax.Extern.t -> build_map * deps -> build_map * deps = + conf:ImporterSyntax.t -> + ImporterSyntax.Extern.t -> + build_map * deps -> + build_map * deps = fun ~conf extern (map, graph) -> let dest = ImportContainers.KeyName.from_table extern.target in (* Pre-check that every source is already declared in the configuration. *) let _ = Expression.fold_values extern.intern_key ~init:() ~f:(fun () path -> try - let _ = ImportConf.get_table_for_name conf path.Path.alias in + let _ = ImporterSyntax.get_table_for_name conf path.Path.alias in () with | Not_found -> ( @@ -177,7 +183,7 @@ let add_external_in_map : { of_path = (fun Path.{ alias; column } -> - let table = ImportConf.get_table_for_name conf alias in + let table = ImporterSyntax.get_table_for_name conf alias in (Some table.name, column)); to_mapping = (fun mapping column -> @@ -194,11 +200,11 @@ let mapper = of_path = (fun ({ alias; column } : Path.t) -> (alias, column)); } -let get_mapping : Syntax.t -> build_map * deps = +let get_mapping : ImporterSyntax.t -> build_map * deps = fun conf -> - let root = ImportContainers.Source.from_table (ImportConf.root_table conf) + let root = ImportContainers.Source.from_table (ImporterSyntax.root_table conf) and root' = - ImportContainers.KeyName.from_table (ImportConf.root_table conf) + ImportContainers.KeyName.from_table (ImporterSyntax.root_table conf) in let graph = [ (root, []) ] in @@ -207,7 +213,7 @@ let get_mapping : Syntax.t -> build_map * deps = let init = ( ImportContainers.Externals.singleton root' { - table = ImportConf.root_table conf; + table = ImporterSyntax.root_table conf; columns = IntSet.empty; keys = []; }, @@ -229,7 +235,7 @@ let get_mapping : Syntax.t -> build_map * deps = in (map, graph) -let get_process_order : Syntax.t -> t list = +let get_process_order : ImporterSyntax.t -> t list = fun map -> let map, graph = get_mapping map in diff --git a/lib/analysers/dependency.mli b/lib/analysers/dependency.mli index bc761ae..522436c 100644 --- a/lib/analysers/dependency.mli +++ b/lib/analysers/dependency.mli @@ -1,6 +1,6 @@ type t -val get_process_order : ImportConf.Syntax.t -> t list +val get_process_order : ImporterSyntax.t -> t list (** Extract the file list to process, following the identified dependancies. Try to load first the document which does not required another spreadsheet, and keep going in the topological order diff --git a/lib/analysers/dune b/lib/analysers/dune index 382dd6b..3ba018c 100755 --- a/lib/analysers/dune +++ b/lib/analysers/dune @@ -1,12 +1,11 @@ (library
(name importAnalyser)
(libraries
- importConf
- importContainers
- importCSV
importDataTypes
- importExpression
importErrors
+ importExpression
+ importerSyntax
+ importContainers
tsort
)
(preprocess (pps
diff --git a/lib/analysers/filters.ml b/lib/analysers/filters.ml index 4e8b175..7044798 100644 --- a/lib/analysers/filters.ml +++ b/lib/analysers/filters.ml @@ -1,14 +1,12 @@ (** Build a fragment of the query match a filter *) -module Syntax = ImportConf.Syntax module Path = ImportDataTypes.Path module Expression = ImportExpression -module CTE = ImportConf.CTE open StdLabels (** Add a list of expressions into the group *) let rec add_filters : - conf:Syntax.t -> Chunk.t -> Path.t Expression.T.t list -> unit = + conf:ImporterSyntax.t -> Chunk.t -> Path.t Expression.T.t list -> unit = fun ~conf group -> function | [] -> () | hd :: [] -> Chunk.add_expression ~conf group hd @@ -25,7 +23,7 @@ type 'a cte_acc = { latest_expression : Path.t Expression.T.t list; } -let add_inner : conf:Syntax.t -> int -> Buffer.t -> unit = +let add_inner : conf:ImporterSyntax.t -> int -> Buffer.t -> unit = fun ~conf n b -> let name = "filter" ^ string_of_int n in (* We use an INNER JOIN here because we want to be sure to get all the rows @@ -39,15 +37,15 @@ let add_inner : conf:Syntax.t -> int -> Buffer.t -> unit = Buffer.add_string b ".id\n" let print : - conf:Syntax.t -> + conf:ImporterSyntax.t -> (Chunk.t * Chunk.t) cte_acc -> - CTE.t -> + ImporterSyntax.CTE.t -> (Chunk.t * Chunk.t) cte_acc = fun ~conf acc cte -> let predicates, query = acc.acc in let n = acc.n in let cte_index = - match cte.CTE.group with + match cte.ImporterSyntax.CTE.group with | Some expression -> begin if acc.has_previous then Chunk.add_string query ", " @@ -73,11 +71,11 @@ let print : end; begin - match cte.CTE.filters with + match cte.ImporterSyntax.CTE.filters with | [] -> () | _ -> Chunk.add_string query " WHERE "; - add_filters ~conf query cte.CTE.filters + add_filters ~conf query cte.ImporterSyntax.CTE.filters end; Chunk.add_string query ")\n"; Some acc.n @@ -85,7 +83,7 @@ let print : (* Do not add the filters in the CTE (we don’t have any) but in the main query *) Chunk.add_string predicates "WHERE "; - add_filters ~conf predicates cte.CTE.filters; + add_filters ~conf predicates cte.ImporterSyntax.CTE.filters; acc.cte_index in { @@ -93,10 +91,11 @@ let print : has_previous = true; n = acc.n + 1; cte_index; - latest_expression = cte.CTE.filters; + latest_expression = cte.ImporterSyntax.CTE.filters; } -let generate_sql : conf:Syntax.t -> CTE.t list -> Chunk.t -> Chunk.t = +let generate_sql : + conf:ImporterSyntax.t -> ImporterSyntax.CTE.t list -> Chunk.t -> Chunk.t = fun ~conf filters links' -> let predicates = Chunk.create () and links = Chunk.create () in let eval = diff --git a/lib/analysers/filters.mli b/lib/analysers/filters.mli index 7783799..3a81202 100644 --- a/lib/analysers/filters.mli +++ b/lib/analysers/filters.mli @@ -1,2 +1,2 @@ val generate_sql : - conf:ImportConf.Syntax.t -> ImportConf.CTE.t list -> Chunk.t -> Chunk.t + conf:ImporterSyntax.t -> ImporterSyntax.CTE.t list -> Chunk.t -> Chunk.t diff --git a/lib/analysers/headers.ml b/lib/analysers/headers.ml index 916dfee..cbeddfb 100644 --- a/lib/analysers/headers.ml +++ b/lib/analysers/headers.ml @@ -1,7 +1,5 @@ open StdLabels -module I = ImportConf module E = ImportExpression.T -module Syntax = ImportConf.Syntax module Table = ImportDataTypes.Table module Path = ImportDataTypes.Path @@ -15,7 +13,7 @@ end) type content = string array type t = content SheeetMap.t -(** The map associate a line of headers for each table. +(** The map associate a line of headers for each table. The header are always in string. *) @@ -23,9 +21,8 @@ type t = content SheeetMap.t and will reformat the first line with the values from the cell. The functions will not be evaluated (instead they will be displayed "as is". - When there is no value for this path, return empty string. - *) -let columns : Syntax.t -> t -> string list = + When there is no value for this path, return empty string. *) +let columns : ImporterSyntax.t -> t -> string list = fun conf t -> (* We build here a custom printer which search in the array for the column name. @@ -33,7 +30,7 @@ let columns : Syntax.t -> t -> string list = This function will be given as argument in the expression printer. *) let f : Path.t -> Buffer.t -> unit = fun path b -> - let source = I.get_table_for_name conf path.alias in + let source = ImporterSyntax.get_table_for_name conf path.alias in match SheeetMap.find_opt source t with | None -> () @@ -42,12 +39,13 @@ let columns : Syntax.t -> t -> string list = | _ -> prerr_endline @@ Printf.sprintf "No header found for :%s.%s" - (Option.value ~default:(I.root_table conf).Table.name + (Option.value + ~default:(ImporterSyntax.root_table conf).Table.name path.alias) - (ImportCSV.Csv.column_to_string path.column)) + (ImportDataTypes.Path.column_to_string path.column)) in - List.map conf.Syntax.columns ~f:(fun column -> + List.map conf.ImporterSyntax.columns ~f:(fun column -> let b = Buffer.create 4 in ImportExpression.Headers.headers_of_expression b f column; diff --git a/lib/analysers/headers.mli b/lib/analysers/headers.mli index 03e384b..b9149a9 100644 --- a/lib/analysers/headers.mli +++ b/lib/analysers/headers.mli @@ -2,10 +2,9 @@ module SheeetMap : Map.S with type key = ImportDataTypes.Table.t type t = string array SheeetMap.t -val columns : ImportConf.Syntax.t -> t -> string list +val columns : ImporterSyntax.t -> t -> string list (** Get the headers. The function has to be called after reading each document, and will reformat the first line with the values from the cell. The functions will not be evaluated (instead they will be displayed "as is". - When there is no value for this path, return empty string. - *) + When there is no value for this path, return empty string. *) diff --git a/lib/analysers/printers.ml b/lib/analysers/printers.ml index 1c73c13..3dc2f02 100644 --- a/lib/analysers/printers.ml +++ b/lib/analysers/printers.ml @@ -1,10 +1,9 @@ -module Syntax = ImportConf.Syntax module Table = ImportDataTypes.Table module Path = ImportDataTypes.Path -let path : conf:Syntax.t -> Format.formatter -> Path.t -> unit = +let path : conf:ImporterSyntax.t -> Format.formatter -> Path.t -> unit = fun ~conf buffer { alias; column } -> - let table = ImportConf.get_table_for_name conf alias in + let table = ImporterSyntax.get_table_for_name conf alias in Format.fprintf buffer "%s" (Table.print_column table ("col_" ^ string_of_int column)) diff --git a/lib/analysers/printers.mli b/lib/analysers/printers.mli index 102bb91..3916e1c 100644 --- a/lib/analysers/printers.mli +++ b/lib/analysers/printers.mli @@ -1,5 +1,5 @@ val path : - conf:ImportConf.Syntax.t -> Format.formatter -> ImportDataTypes.Path.t -> unit + conf:ImporterSyntax.t -> Format.formatter -> ImportDataTypes.Path.t -> unit (** Represent a path in a SQL query. This function is given in the Expression.Query module. *) diff --git a/lib/analysers/query.ml b/lib/analysers/query.ml index e24da78..f89f5f0 100644 --- a/lib/analysers/query.ml +++ b/lib/analysers/query.ml @@ -1,15 +1,15 @@ open StdLabels module Expression = ImportExpression module Q = Expression.Query -module Syntax = ImportConf.Syntax module Table = ImportDataTypes.Table module Path = ImportDataTypes.Path (* Collect all the tables pointed by the expression. *) -let pointed_tables : Syntax.t -> 'a Expression.T.t -> (Table.t * string) list = +let pointed_tables : + ImporterSyntax.t -> 'a Expression.T.t -> (Table.t * string) list = fun conf expression -> Expression.T.fold_values expression ~init:[] ~f:(fun acc path -> - let table = ImportConf.get_table_for_name conf path.Path.alias in + let table = ImporterSyntax.get_table_for_name conf path.Path.alias in let table_name = Table.name table in (table, table_name) :: acc) |> List.sort_uniq ~cmp:Stdlib.compare @@ -37,7 +37,7 @@ let create_table : Dependency.t -> string = type query = { q : string; - parameters : ImportCSV.DataType.t Seq.t; + parameters : ImportDataTypes.Value.t Seq.t; } let rec take_elements : @@ -88,9 +88,9 @@ let clean_window : The select query will name each column with an alias, and the map allow to find which source is pointed by this alias. *) -let select : Syntax.t -> query * Path.t ImportExpression.T.t array = +let select : ImporterSyntax.t -> query * Path.t ImportExpression.T.t array = fun conf -> - let filter = ImportConf.CTE.of_filters conf.filters in + let filter = ImporterSyntax.CTE.of_filters conf.filters in (* For each column in the configuration file, add the corresponding element in the query. @@ -136,7 +136,7 @@ let select : Syntax.t -> query * Path.t ImportExpression.T.t array = Chunk.append ~head:request_header ~tail:filters; let formatter = Format.formatter_of_buffer b in - (match conf.Syntax.uniq with + (match conf.ImporterSyntax.uniq with | [] -> () | uniq -> Format.fprintf formatter "\nGROUP BY %a" @@ -149,7 +149,7 @@ let select : Syntax.t -> query * Path.t ImportExpression.T.t array = in Queue.transfer seq parameters)) uniq); - (match conf.Syntax.sort with + (match conf.ImporterSyntax.sort with | [] -> () | sort -> Format.fprintf formatter "\nORDER BY %a" @@ -166,15 +166,16 @@ let select : Syntax.t -> query * Path.t ImportExpression.T.t array = ({ q = Buffer.contents b; parameters = Queue.to_seq parameters }, headers) -let check_external : Syntax.t -> Syntax.Extern.t -> query = +let check_external : ImporterSyntax.t -> ImporterSyntax.Extern.t -> query = fun conf external_ -> let internal_chunk = Chunk.create () in - Chunk.add_expression ~conf internal_chunk external_.Syntax.Extern.intern_key; + Chunk.add_expression ~conf internal_chunk + external_.ImporterSyntax.Extern.intern_key; let external_key_buffer = Buffer.create 16 in Buffer.add_string external_key_buffer - (Table.print_column external_.Syntax.Extern.target - ("key_" ^ external_.Syntax.Extern.target.name)); + (Table.print_column external_.ImporterSyntax.Extern.target + ("key_" ^ external_.ImporterSyntax.Extern.target.name)); let pointed_tables = pointed_tables conf external_.intern_key in @@ -190,7 +191,9 @@ let check_external : Syntax.t -> Syntax.Extern.t -> query = This not the usual way to proceed (we start from the source and link the externals) *) let rec collect_links : - Syntax.Extern.t -> Syntax.Extern.t list -> Syntax.Extern.t list = + ImporterSyntax.Extern.t -> + ImporterSyntax.Extern.t list -> + ImporterSyntax.Extern.t list = fun table init -> let res = (* Do not add the same external if the value is already present *) @@ -200,16 +203,16 @@ let check_external : Syntax.t -> Syntax.Extern.t -> query = | Some _ -> init in - Expression.T.fold_values ~init table.Syntax.Extern.intern_key + Expression.T.fold_values ~init table.ImporterSyntax.Extern.intern_key ~f:(fun acc expr -> match expr.Path.alias with | None -> acc | Some _ as path -> ( - let table = ImportConf.get_table_for_name conf path in + let table = ImporterSyntax.get_table_for_name conf path in (* Look for this table in the externals *) let external_opt = - List.find_opt conf.Syntax.externals ~f:(fun t -> - t.Syntax.Extern.target == table) + List.find_opt conf.ImporterSyntax.externals ~f:(fun t -> + t.ImporterSyntax.Extern.target == table) in match external_opt with | None -> acc diff --git a/lib/analysers/query.mli b/lib/analysers/query.mli index d158867..491c891 100644 --- a/lib/analysers/query.mli +++ b/lib/analysers/query.mli @@ -2,16 +2,15 @@ val create_table : Dependency.t -> string type query = { q : string; (** The query to execute *) - parameters : ImportCSV.DataType.t Seq.t; + parameters : ImportDataTypes.Value.t Seq.t; } (** This type represent a query to execute. [q] is the template to run, and shall be run with all the binded parameters. *) val select : - ImportConf.Syntax.t -> - query * ImportDataTypes.Path.t ImportExpression.T.t array + ImporterSyntax.t -> query * ImportDataTypes.Path.t ImportExpression.T.t array -val check_external : ImportConf.Syntax.t -> ImportConf.Syntax.Extern.t -> query +val check_external : ImporterSyntax.t -> ImporterSyntax.Extern.t -> query (** Create a query which select all the missing key in an external *) val build_key_insert : Buffer.t -> Dependency.key -> unit diff --git a/lib/configuration/dune b/lib/configuration/dune index 6a0bc61..7f78cca 100755 --- a/lib/configuration/dune +++ b/lib/configuration/dune @@ -4,17 +4,14 @@ decoders
otoml
menhirLib
- importCSV
- re
helpers
importDataTypes
- importExpression
importErrors
+ importExpression
+ importerSyntax
)
(preprocess (pps
ppx_deriving.ord
- ppx_deriving.show
- ppx_deriving.eq
))
)
diff --git a/lib/configuration/expression_parser.mly b/lib/configuration/expression_parser.mly index 18b79c8..1761cce 100644 --- a/lib/configuration/expression_parser.mly +++ b/lib/configuration/expression_parser.mly @@ -36,8 +36,8 @@ path_: | COLUMN column = IDENT { ImportExpression.T.Path - Syntax.Path.{ alias = None - ; column = ImportCSV.Csv.column_of_string column + ImportDataTypes.Path.{ alias = None + ; column = ImportDataTypes.Path.column_of_string column } } @@ -46,14 +46,14 @@ path_: DOT column = IDENT { ImportExpression.T.Path - Syntax.Path.{ alias = Some table - ; column = ImportCSV.Csv.column_of_string column} + ImportDataTypes.Path.{ alias = Some table + ; column = ImportDataTypes.Path.column_of_string column} } column_: | COLUMN column = IDENT - { try ImportExpression.T.Path (ImportCSV.Csv.column_of_string column) + { try ImportExpression.T.Path (ImportDataTypes.Path.column_of_string column) with _ -> ImportExpression.T.Literal column } arguments(PATH): diff --git a/lib/configuration/importConf.ml b/lib/configuration/importConf.ml index 8516008..2df24bd 100644 --- a/lib/configuration/importConf.ml +++ b/lib/configuration/importConf.ml @@ -1,68 +1,17 @@ -open StdLabels -module Syntax = Syntax -module CTE = Cte -module Table = ImportDataTypes.Table -module Path = ImportDataTypes.Path -module T = Read_conf -module Expression = ImportExpression.T - -let latest_version = 1 - module TomlReader = Read_conf.Make (Helpers.Toml.Decode) -let t_of_toml : Otoml.t -> (Syntax.t, string) result = +let t_of_toml : Otoml.t -> (ImporterSyntax.t, string) result = fun toml -> let version = - Otoml.find_or ~default:latest_version toml + Otoml.find_or ~default:ImporterSyntax.latest_version toml (Otoml.get_integer ~strict:false) [ "version" ] in match version with - | n when n = latest_version -> TomlReader.read toml + | n when n = ImporterSyntax.latest_version -> TomlReader.read toml | _ -> Printf.eprintf "Unsuported version : %d\n" version; exit 1 -let dummy_conf = - Syntax. - { - source = { file = ""; tab = 0; name = "" }; - version = latest_version; - locale = Some "C"; - externals = []; - columns = []; - filters = []; - sort = []; - uniq = []; - } - -let get_table_for_name : Syntax.t -> string option -> Table.t = - fun conf name -> - match name with - | None -> conf.source - | Some name -> - if String.equal name conf.source.name then conf.source - else - let ext = - List.find conf.externals ~f:(fun (ext : Syntax.Extern.t) -> - String.equal name ext.target.name) - in - ext.target - -let root_table : Syntax.t -> Table.t = fun conf -> conf.source - -let get_dependancies_for_table : Syntax.t -> Table.t -> Syntax.Extern.t list = - fun conf source -> - let is_root = source = conf.source in - - List.filter conf.externals ~f:(fun (ext : Syntax.Extern.t) -> - (* Enumerate the intern_key and check the source pointed by each column *) - Expression.fold_values ext.intern_key ~init:false ~f:(fun acc expr -> - if acc then acc - else - match expr.Syntax.Path.alias with - | Some v -> String.equal v source.name - | None -> is_root)) - let expression_from_string s = Read_conf.ExpressionParser.of_string Read_conf.ExpressionParser.path s diff --git a/lib/configuration/importConf.mli b/lib/configuration/importConf.mli index 40b985b..d2f65f2 100644 --- a/lib/configuration/importConf.mli +++ b/lib/configuration/importConf.mli @@ -1,18 +1,4 @@ -module Syntax = Syntax -module CTE = Cte - -val dummy_conf : Syntax.t - -val root_table : Syntax.t -> ImportDataTypes.Table.t -(** Get the root table, this table is the main table to load and each line in - this table will be processed *) - -val t_of_toml : Otoml.t -> (Syntax.t, string) result -val get_table_for_name : Syntax.t -> string option -> ImportDataTypes.Table.t - -val get_dependancies_for_table : - Syntax.t -> ImportDataTypes.Table.t -> Syntax.Extern.t list -(** Get all the externals refered by the source *) +val t_of_toml : Otoml.t -> (ImporterSyntax.t, string) result val expression_from_string : string -> (ImportDataTypes.Path.t ImportExpression.T.t, string) result diff --git a/lib/configuration/read_conf.ml b/lib/configuration/read_conf.ml index 11f6726..d406b0e 100644 --- a/lib/configuration/read_conf.ml +++ b/lib/configuration/read_conf.ml @@ -222,7 +222,7 @@ module Make (S : Decoders.Decode.S) = struct in S.succeed - Syntax.Extern. + ImporterSyntax.Extern. { intern_key; extern_key; @@ -256,7 +256,7 @@ module Make (S : Decoders.Decode.S) = struct ExpressionParser.path) in S.succeed @@ fun version source externals locale -> - Syntax. + ImporterSyntax. { version; source; externals; columns; filters; sort; uniq; locale } method conf = diff --git a/lib/containers/dune b/lib/containers/dune index 70bbf0b..d9b06cf 100755 --- a/lib/containers/dune +++ b/lib/containers/dune @@ -2,7 +2,6 @@ (name importContainers)
(libraries
importDataTypes
- importConf
)
(preprocess (pps
ppx_deriving.ord
diff --git a/lib/containers/importContainers.ml b/lib/containers/importContainers.ml index 6da8374..c90bc57 100644 --- a/lib/containers/importContainers.ml +++ b/lib/containers/importContainers.ml @@ -1,5 +1,3 @@ -module Conf = ImportConf -module Syntax = Conf.Syntax module Table = ImportDataTypes.Table (** This key is used to create the table of each externals in the configuration. diff --git a/lib/csv/csv.ml b/lib/csv/csv.ml deleted file mode 100644 index db7329d..0000000 --- a/lib/csv/csv.ml +++ /dev/null @@ -1,30 +0,0 @@ -open StdLabels - -type t = int - -let column_of_char = function - | 'A' .. 'Z' as c -> Char.code c - (Char.code 'A' - 1) - | 'a' .. 'z' as c -> Char.code c - (Char.code 'a' - 1) - | c -> raise (Invalid_argument ("column: " ^ Char.escaped c)) - -let column_of_string : string -> int = - fun s -> - String.fold_left s ~init:0 ~f:(fun value c -> (value * 26) + column_of_char c) - -(** Accumulate the remaining for the successives divisions in a list. *) -let rec _to_char ~b i = - if i > 0 then - let res = i mod 26 in - let res = if res = 0 then 26 else res in - - let c = char_of_int @@ (res + 64) in - (* The modulo is accumulated in the list head, which is the expected - sequence *) - let b = c :: b in - - _to_char ~b @@ ((i - res) / 26) - else b - -let column_to_string i = - let res = _to_char ~b:[] i in - List.to_seq res |> String.of_seq diff --git a/lib/csv/dune b/lib/csv/dune deleted file mode 100755 index 2cdc868..0000000 --- a/lib/csv/dune +++ /dev/null @@ -1,7 +0,0 @@ -(library
- (name importCSV)
- (libraries
- re
- )
- (foreign_stubs (language c) (names format))
-)
diff --git a/lib/data_types/dune b/lib/data_types/dune index b735171..d17ed8f 100644 --- a/lib/data_types/dune +++ b/lib/data_types/dune @@ -1,11 +1,11 @@ (library (name importDataTypes) (libraries - importCSV + re ) - (preprocess (pps ppx_deriving.ord ppx_deriving.show ppx_deriving.eq )) + (foreign_stubs (language c) (names format)) ) diff --git a/lib/csv/format.c b/lib/data_types/format.c index 1394dc7..1394dc7 100644 --- a/lib/csv/format.c +++ b/lib/data_types/format.c diff --git a/lib/data_types/path.ml b/lib/data_types/path.ml index 0a87109..0dc2b74 100644 --- a/lib/data_types/path.ml +++ b/lib/data_types/path.ml @@ -1,3 +1,5 @@ +open StdLabels + type column = int [@@deriving ord, show, eq] type t = { @@ -8,8 +10,29 @@ type t = { } [@@deriving ord, show, eq] -let show { alias; column } = - let column_text = ImportCSV.Csv.column_to_string column in - match alias with - | None -> ":" ^ column_text - | Some value -> ":" ^ value ^ "." ^ column_text +let column_of_char = function + | 'A' .. 'Z' as c -> Char.code c - (Char.code 'A' - 1) + | 'a' .. 'z' as c -> Char.code c - (Char.code 'a' - 1) + | c -> raise (Invalid_argument ("column: " ^ Char.escaped c)) + +let column_of_string : string -> int = + fun s -> + String.fold_left s ~init:0 ~f:(fun value c -> (value * 26) + column_of_char c) + +(** Accumulate the remaining for the successives divisions in a list. *) +let rec _to_char ~b i = + if i > 0 then + let res = i mod 26 in + let res = if res = 0 then 26 else res in + + let c = char_of_int @@ (res + 64) in + (* The modulo is accumulated in the list head, which is the expected + sequence *) + let b = c :: b in + + _to_char ~b @@ ((i - res) / 26) + else b + +let column_to_string i = + let res = _to_char ~b:[] i in + List.to_seq res |> String.of_seq diff --git a/lib/data_types/path.mli b/lib/data_types/path.mli new file mode 100644 index 0000000..5fd7542 --- /dev/null +++ b/lib/data_types/path.mli @@ -0,0 +1,12 @@ +type column = int [@@deriving ord, show, eq] + +type t = { + alias : string option; + (* External file to load, when the information is missing, load in + the current file *) + column : column; +} +[@@deriving ord, show, eq] + +val column_of_string : string -> int +val column_to_string : int -> string diff --git a/lib/csv/dataType.ml b/lib/data_types/value.ml index 6bf85ae..6bf85ae 100644 --- a/lib/csv/dataType.ml +++ b/lib/data_types/value.ml diff --git a/lib/csv/dataType.mli b/lib/data_types/value.mli index df5edf1..df5edf1 100644 --- a/lib/csv/dataType.mli +++ b/lib/data_types/value.mli diff --git a/lib/errors/dune b/lib/errors/dune index c39f47f..98059b5 100644 --- a/lib/errors/dune +++ b/lib/errors/dune @@ -4,7 +4,6 @@ ppx_deriving.runtime csv sqlite3 - importCSV importDataTypes ) ) diff --git a/lib/errors/importErrors.ml b/lib/errors/importErrors.ml index 7433e9b..a062f38 100644 --- a/lib/errors/importErrors.ml +++ b/lib/errors/importErrors.ml @@ -1,5 +1,4 @@ open StdLabels -module CSV = ImportCSV module Table = ImportDataTypes.Table let bom = "\xEF\xBB\xBF" @@ -9,7 +8,7 @@ type xlsError = { row : int; sheet : int; target : Table.t option; - value : CSV.DataType.t; + value : ImportDataTypes.Value.t; exn : exn; } @@ -35,7 +34,8 @@ exception let repr_error = function | SqlError s -> Printf.sprintf "%s Error" (Sqlite3.Rc.to_string s) | NullKey k -> - Printf.sprintf "The key %s is null" (ImportCSV.Csv.column_to_string k) + Printf.sprintf "The key %s is null" + (ImportDataTypes.Path.column_to_string k) | Unknown_source source -> Printf.sprintf "The source %s is referenced without beiing declared" source @@ -74,7 +74,7 @@ let output_error : t -> xlsError -> unit = string_of_int error.sheet; string_of_int error.row; target; - CSV.DataType.to_string "C" error.value; + ImportDataTypes.Value.to_string "C" error.value; repr_error error.exn; ] diff --git a/lib/errors/importErrors.mli b/lib/errors/importErrors.mli index 445361e..1afcacf 100644 --- a/lib/errors/importErrors.mli +++ b/lib/errors/importErrors.mli @@ -1,5 +1,3 @@ -module CSV = ImportCSV - exception SqlError of Sqlite3.Rc.t exception MisplacedWindow @@ -18,7 +16,7 @@ type xlsError = { row : int; sheet : int; target : ImportDataTypes.Table.t option; - value : CSV.DataType.t; + value : ImportDataTypes.Value.t; exn : exn; } diff --git a/lib/expression/_readme.rst b/lib/expression/_readme.rst index 729a950..59c4687 100644 --- a/lib/expression/_readme.rst +++ b/lib/expression/_readme.rst @@ -26,13 +26,6 @@ Simple transformations Composed transformations ------------------------ -:Filter: - - Generate the filters in the query. This module identify if one the - expression is actually a group window and handle a special case for this. - - This module relies on Ast - :Query: Build an sql query. This module relies on Type_of diff --git a/lib/expression/dune b/lib/expression/dune index 8bf6e62..5f5a12c 100755 --- a/lib/expression/dune +++ b/lib/expression/dune @@ -2,7 +2,6 @@ (name importExpression)
(libraries
re
- importCSV
importDataTypes
importErrors
)
diff --git a/lib/expression/query.ml b/lib/expression/query.ml index e648daf..8ff179d 100644 --- a/lib/expression/query.ml +++ b/lib/expression/query.ml @@ -6,7 +6,7 @@ open StdLabels in order to tell if we need to bind the parameters in the query, or if we can use plain literal as is (with some risk at the execution time. *) type _ binded_query = - | BindParam : ImportCSV.DataType.t Queue.t binded_query + | BindParam : ImportDataTypes.Value.t Queue.t binded_query | NoParam : unit binded_query module QueryParameter = struct @@ -17,7 +17,7 @@ module QueryParameter = struct The Raw can be generated from both BindParam or NoParam queries. *) type t = | Literal - | Queue of ImportCSV.DataType.t Queue.t + | Queue of ImportDataTypes.Value.t Queue.t | Raw of t (** Wrap the given parameter mode into the raw mode *) @@ -100,7 +100,7 @@ module Query = TypeBuilder.Make (struct Format.fprintf formatter "'%s'" l | QueryParameter.Queue queue -> Format.fprintf formatter "?"; - Queue.add (ImportCSV.DataType.Content l) queue + Queue.add (ImportDataTypes.Value.Content l) queue | QueryParameter.Raw _ -> Format.fprintf formatter "%s" l let integer : string -> 'a Type_of.obs -> 'a repr = diff --git a/lib/expression/query.mli b/lib/expression/query.mli index fa789a9..a124be4 100644 --- a/lib/expression/query.mli +++ b/lib/expression/query.mli @@ -1,17 +1,17 @@ module QueryParameter : sig - (** Internaly, we need to keep a different type for the Literal chunks - (which requires to be quoted), and raw (which should be given as is to the - sql engine) + (** Internaly, we need to keep a different type for the Literal chunks (which + requires to be quoted), and raw (which should be given as is to the sql + engine) - The Raw can be generated from both BindParam or NoParam queries. *) + The Raw can be generated from both BindParam or NoParam queries. *) type t = | Literal - | Queue of ImportCSV.DataType.t Queue.t + | Queue of ImportDataTypes.Value.t Queue.t | Raw of t end type _ binded_query = - | BindParam : ImportCSV.DataType.t Queue.t binded_query + | BindParam : ImportDataTypes.Value.t Queue.t binded_query | NoParam : unit binded_query val query_of_expression : diff --git a/lib/file_handler/csv2sql.ml b/lib/file_handler/csv2sql.ml index 42d84eb..a009f74 100644 --- a/lib/file_handler/csv2sql.ml +++ b/lib/file_handler/csv2sql.ml @@ -1,34 +1,33 @@ open StdLabels module A = ImportAnalyser.Dependency -module CSV = ImportCSV module C = ImportContainers -module Syntax = ImportConf.Syntax module Db = ImportSQL.Db -type state = CSV.DataType.t array State.t +type state = ImportDataTypes.Value.t array State.t let default_mapper : - (ImportCSV.DataType.t, ImportCSV.DataType.t array) State.mapper = - { get_row = Fun.id; get_value = Fun.id; default = ImportCSV.DataType.Null } + (ImportDataTypes.Value.t, ImportDataTypes.Value.t array) State.mapper = + { get_row = Fun.id; get_value = Fun.id; default = ImportDataTypes.Value.Null } -let extract_values : string -> CSV.DataType.t = +let extract_values : string -> ImportDataTypes.Value.t = fun value -> (* Test first if the content is empty *) - if String.equal String.empty value then CSV.DataType.Null + if String.equal String.empty value then ImportDataTypes.Value.Null else (* else, try differents conversion in order to see which one works *) match int_of_string_opt value with - | Some i -> CSV.DataType.Integer i + | Some i -> ImportDataTypes.Value.Integer i | None -> ( match float_of_string_opt value with - | Some f -> CSV.DataType.Float f + | Some f -> ImportDataTypes.Value.Float f | None -> (* And finaly convert into date *) - CSV.DataType.Content value) + ImportDataTypes.Value.Content value) (** Initialize the state for the first row, count the column number and create the table in the database *) -let first_row : A.t -> _ Db.t -> state -> CSV.DataType.t list -> state = +let first_row : A.t -> _ Db.t -> state -> ImportDataTypes.Value.t list -> state + = fun mapping db acc row -> (if acc.transaction then match Db.commit db with @@ -69,11 +68,11 @@ let read_csv_line : let importInDatable : log_error:ImportErrors.t -> - conf:Syntax.t -> + conf:ImporterSyntax.t -> dirname:string -> A.t -> 'a Db.t -> - CSV.DataType.t array option Lwt.t = + ImportDataTypes.Value.t array option Lwt.t = fun ~log_error ~conf ~dirname mapping db -> let file = Filename.concat dirname (A.table mapping).file in @@ -86,7 +85,9 @@ let importInDatable : This line could generate an error if the headers are not correctly defined. *) let header = - List.map ~f:(fun v -> CSV.DataType.Content v) (Csv.next csv_channel) + List.map + ~f:(fun v -> ImportDataTypes.Value.Content v) + (Csv.next csv_channel) in let state = @@ -110,7 +111,7 @@ let importInDatable : with | Csv.Failure (line, row, cause) as e -> Printf.eprintf "Error %s on line %d — field : %s\n" cause line - (ImportCSV.Csv.column_to_string row); + (ImportDataTypes.Path.column_to_string row); raise e in ignore @@ State.clear ~log_error db mapping conf; diff --git a/lib/file_handler/csv2sql.mli b/lib/file_handler/csv2sql.mli index e09737b..8976034 100644 --- a/lib/file_handler/csv2sql.mli +++ b/lib/file_handler/csv2sql.mli @@ -1,10 +1,10 @@ val importInDatable : log_error:ImportErrors.t -> - conf:ImportConf.Syntax.t -> + conf:ImporterSyntax.t -> dirname:string -> ImportAnalyser.Dependency.t -> _ ImportSQL.Db.t -> - ImportCSV.DataType.t array option Lwt.t -(** Load an excel spreadsheet in an SQLite database. + ImportDataTypes.Value.t array option Lwt.t +(** Load an excel spreadsheet in an SQLite database. -Return the header if at least one row where present *) + Return the header if at least one row where present *) diff --git a/lib/file_handler/dune b/lib/file_handler/dune index 02be751..80b2ff6 100755 --- a/lib/file_handler/dune +++ b/lib/file_handler/dune @@ -10,13 +10,11 @@ lwt
lwt.unix
helpers
- importAnalyser
- importConf
- importContainers
- importCSV
importDataTypes
+ importContainers
importErrors
- importExpression
+ importerSyntax
+ importAnalyser
importSQL
)
)
diff --git a/lib/file_handler/state.ml b/lib/file_handler/state.ml index 5b43aff..7cf57da 100644 --- a/lib/file_handler/state.ml +++ b/lib/file_handler/state.ml @@ -18,7 +18,7 @@ type insert_result = { type ('a, 'b) mapper = { get_row : 'b -> 'a Array.t; - get_value : 'a -> ImportCSV.DataType.t; + get_value : 'a -> ImportDataTypes.Value.t; default : 'a; } @@ -105,14 +105,15 @@ let insert_row : sheet = state.sheet_number; row = state.row_number; target = None; - value = CSV.DataType.Content (String.concat ~sep:", " []); + value = ImportDataTypes.Value.Content (String.concat ~sep:", " []); exn = e; }) execution -(** Load the row with all the informations associated with this sheet. +(** Load the row with all the informations associated with this sheet. - If an error has already been raised during the sheet, ignore this row only. *) + If an error has already been raised during the sheet, ignore this row only. +*) let run_row : log_error:ImportErrors.t -> mapper:(_, 'row) mapper -> @@ -147,20 +148,20 @@ let clear : log_error:ImportErrors.t -> 'a ImportSQL.Db.t -> A.t -> - ImportConf.Syntax.t -> + ImporterSyntax.t -> unit ImportSQL.Db.result = fun ~log_error db mapping conf -> ImportSQL.Db.clear_duplicates db (A.table mapping) (A.keys mapping) ~f:(fun values -> let line = match snd @@ Array.get values 0 with - | ImportCSV.DataType.Integer i -> i + | ImportDataTypes.Value.Integer i -> i | _ -> -1 and value = snd @@ Array.get values 1 and target = match snd @@ Array.get values 2 with - | ImportCSV.DataType.Content s -> - Some (ImportConf.get_table_for_name conf (Some s)) + | ImportDataTypes.Value.Content s -> + Some (ImporterSyntax.get_table_for_name conf (Some s)) | _ -> None in let error = diff --git a/lib/file_handler/state.mli b/lib/file_handler/state.mli index f744c33..8e24b1b 100644 --- a/lib/file_handler/state.mli +++ b/lib/file_handler/state.mli @@ -15,7 +15,7 @@ type insert_result = { type ('a, 'b) mapper = { get_row : 'b -> 'a Array.t; - get_value : 'a -> ImportCSV.DataType.t; + get_value : 'a -> ImportDataTypes.Value.t; default : 'a; } @@ -41,6 +41,7 @@ val clear : log_error:ImportErrors.t -> 'a ImportSQL.Db.t -> ImportAnalyser.Dependency.t -> - ImportConf.Syntax.t -> + ImporterSyntax.t -> unit ImportSQL.Db.result -(** Clean up the table after the insertion, check for the duplicates and external references *) +(** Clean up the table after the insertion, check for the duplicates and + external references *) diff --git a/lib/file_handler/xlsx2sql.ml b/lib/file_handler/xlsx2sql.ml index f2d8f12..651059f 100644 --- a/lib/file_handler/xlsx2sql.ml +++ b/lib/file_handler/xlsx2sql.ml @@ -1,8 +1,6 @@ open StdLabels module A = ImportAnalyser.Dependency module C = ImportContainers -module CSV = ImportCSV -module Syntax = ImportConf.Syntax module Db = ImportSQL.Db let flags = Unix.[ O_RDONLY; O_NONBLOCK ] @@ -10,24 +8,26 @@ let flags = Unix.[ O_RDONLY; O_NONBLOCK ] let extractors = SZXX.Xlsx. { - string = (fun _location s -> CSV.DataType.Content s); + string = (fun _location s -> ImportDataTypes.Value.Content s); error = - (fun _location s -> CSV.DataType.Error (Printf.sprintf "#ERROR# %s" s)); + (fun _location s -> + ImportDataTypes.Value.Error (Printf.sprintf "#ERROR# %s" s)); boolean = (fun _location s -> let value = String.(equal s "1") in - CSV.DataType.Content (string_of_bool value)); + ImportDataTypes.Value.Content (string_of_bool value)); number = (fun _location s -> let f = Float.of_string s in - if Float.is_integer f then CSV.DataType.Integer (Float.to_int f) - else CSV.DataType.Float f); - date = (fun _location s -> CSV.DataType.Content s); - null = CSV.DataType.Null; + if Float.is_integer f then + ImportDataTypes.Value.Integer (Float.to_int f) + else ImportDataTypes.Value.Float f); + date = (fun _location s -> ImportDataTypes.Value.Content s); + null = ImportDataTypes.Value.Null; formula = (fun _location ~formula s -> ignore formula; - CSV.DataType.Content s); + ImportDataTypes.Value.Content s); } let feed_bigstring ic = @@ -48,29 +48,31 @@ let is_delayed row = | _ -> false) let default_mapper : - (ImportCSV.DataType.t, ImportCSV.DataType.t SZXX.Xlsx.row) State.mapper = + ( ImportDataTypes.Value.t, + ImportDataTypes.Value.t SZXX.Xlsx.row ) + State.mapper = { get_value = (function - | ImportCSV.DataType.Content s -> - ImportCSV.DataType.Content (SZXX.Xml.unescape s) + | ImportDataTypes.Value.Content s -> + ImportDataTypes.Value.Content (SZXX.Xml.unescape s) | any -> any); - default = ImportCSV.DataType.Null; + default = ImportDataTypes.Value.Null; get_row = (fun v -> v.SZXX.Xlsx.data); } -type state = CSV.DataType.t SZXX.Xlsx.status SZXX.Xlsx.row State.t +type state = ImportDataTypes.Value.t SZXX.Xlsx.status SZXX.Xlsx.row State.t let delayed_mapper = State. { get_value = (function - | SZXX.Xlsx.Available (CSV.DataType.Content s) -> - CSV.DataType.Content (SZXX.Xml.unescape s) + | SZXX.Xlsx.Available (ImportDataTypes.Value.Content s) -> + ImportDataTypes.Value.Content (SZXX.Xml.unescape s) | SZXX.Xlsx.Available value -> value - | _ -> CSV.DataType.Null); - default = SZXX.Xlsx.Available CSV.DataType.Null; + | _ -> ImportDataTypes.Value.Null); + default = SZXX.Xlsx.Available ImportDataTypes.Value.Null; get_row = (fun v -> v.SZXX.Xlsx.data); } @@ -96,11 +98,11 @@ let first_row : A.t -> _ Db.t -> state -> 'a SZXX.Xlsx.row -> state = let importInDatable : log_error:Csv.out_channel Lazy.t -> - conf:Syntax.t -> + conf:ImporterSyntax.t -> dirname:string -> A.t -> 'a Db.t -> - CSV.DataType.t array option Lwt.t = + ImportDataTypes.Value.t array option Lwt.t = fun ~log_error ~conf ~dirname mapping db -> let file = Filename.concat dirname (A.table mapping).file in diff --git a/lib/file_handler/xlsx2sql.mli b/lib/file_handler/xlsx2sql.mli index e09737b..8976034 100644 --- a/lib/file_handler/xlsx2sql.mli +++ b/lib/file_handler/xlsx2sql.mli @@ -1,10 +1,10 @@ val importInDatable : log_error:ImportErrors.t -> - conf:ImportConf.Syntax.t -> + conf:ImporterSyntax.t -> dirname:string -> ImportAnalyser.Dependency.t -> _ ImportSQL.Db.t -> - ImportCSV.DataType.t array option Lwt.t -(** Load an excel spreadsheet in an SQLite database. + ImportDataTypes.Value.t array option Lwt.t +(** Load an excel spreadsheet in an SQLite database. -Return the header if at least one row where present *) + Return the header if at least one row where present *) diff --git a/lib/sql/db.ml b/lib/sql/db.ml index f1aee8c..f2a2653 100644 --- a/lib/sql/db.ml +++ b/lib/sql/db.ml @@ -1,6 +1,4 @@ open StdLabels -module CSV = ImportCSV -module Syntax = ImportConf.Syntax module Table = ImportDataTypes.Table module Path = ImportDataTypes.Path @@ -153,7 +151,7 @@ let eval_key : 'a t -> Sqlite3.stmt option -> ImportAnalyser.Dependency.key list -> - (int * CSV.DataType.t) list -> + (int * ImportDataTypes.Value.t) list -> (Sqlite3.stmt option * Sqlite3.Data.t list) T.result = fun db stmt keys values -> match keys with @@ -204,7 +202,7 @@ let insert : Sqlite3.db -> Sqlite3.stmt -> id:int -> - (int * CSV.DataType.t) list -> + (int * ImportDataTypes.Value.t) list -> unit T.result = fun db statement ~id values -> let** _ = T.savepoint db "PREVIOUS" in @@ -255,9 +253,9 @@ let execute_query : Ok statement let query : - f:((Path.t ImportExpression.T.t * CSV.DataType.t) array -> unit) -> + f:((Path.t ImportExpression.T.t * ImportDataTypes.Value.t) array -> unit) -> Sqlite3.db -> - Syntax.t -> + ImporterSyntax.t -> unit T.result = fun ~f db output -> (* Extract the query from the configuration. *) @@ -280,7 +278,7 @@ let query : in Ok () -let create_view : Sqlite3.db -> Syntax.t -> unit T.result = +let create_view : Sqlite3.db -> ImporterSyntax.t -> unit T.result = fun db output -> ignore output; let* drop = Sqlite3.exec db "DROP VIEW IF EXISTS 'result'" in @@ -300,10 +298,10 @@ let create_view : Sqlite3.db -> Syntax.t -> unit T.result = *) let check_foreign : - f:((string * CSV.DataType.t) array -> unit) -> + f:((string * ImportDataTypes.Value.t) array -> unit) -> Sqlite3.db -> - Syntax.t -> - Syntax.Extern.t -> + ImporterSyntax.t -> + ImporterSyntax.Extern.t -> unit T.result = fun ~f db conf external_ -> let query = ImportAnalyser.Query.check_external conf external_ in @@ -318,7 +316,7 @@ let check_foreign : |> T.to_result let clear_duplicates : - f:((string * ImportCSV.DataType.t) array -> unit) -> + f:((string * ImportDataTypes.Value.t) array -> unit) -> 'a t -> ImportDataTypes.Table.t -> ImportAnalyser.Dependency.key list -> diff --git a/lib/sql/db.mli b/lib/sql/db.mli index 91933c4..213fb27 100644 --- a/lib/sql/db.mli +++ b/lib/sql/db.mli @@ -1,5 +1,3 @@ -module Syntax = ImportConf.Syntax - type 'a t type 'a result = ('a, exn) Result.t @@ -32,7 +30,7 @@ val eval_key : 'a t -> Sqlite3.stmt option -> ImportAnalyser.Dependency.key list -> - (int * ImportCSV.DataType.t) list -> + (int * ImportDataTypes.Value.t) list -> (Sqlite3.stmt option * Sqlite3.Data.t list) result (** Evaluate the keys in sqlite and get the results. @@ -43,7 +41,7 @@ val insert : 'a t -> Sqlite3.stmt -> id:int -> - (int * ImportCSV.DataType.t) list -> + (int * ImportDataTypes.Value.t) list -> unit result (** Insert a new row in the database. @@ -63,10 +61,11 @@ val rollback : 'a t -> unit result val query : f: - ((ImportDataTypes.Path.t ImportExpression.T.t * ImportCSV.DataType.t) array -> + ((ImportDataTypes.Path.t ImportExpression.T.t * ImportDataTypes.Value.t) + array -> unit) -> 'a t -> - Syntax.t -> + ImporterSyntax.t -> unit result (** This one the most important function from the application. The function will transform the configuration into an sql query and will fetch the result from @@ -74,18 +73,18 @@ val query : The function [f] given in argument will be called for each line *) -val create_view : 'a t -> Syntax.t -> unit result +val create_view : 'a t -> ImporterSyntax.t -> unit result (** Create a view which represent the result *) val check_foreign : - f:((string * ImportCSV.DataType.t) array -> unit) -> + f:((string * ImportDataTypes.Value.t) array -> unit) -> 'a t -> - Syntax.t -> - Syntax.Extern.t -> + ImporterSyntax.t -> + ImporterSyntax.Extern.t -> unit result val clear_duplicates : - f:((string * ImportCSV.DataType.t) array -> unit) -> + f:((string * ImportDataTypes.Value.t) array -> unit) -> 'a t -> ImportDataTypes.Table.t -> ImportAnalyser.Dependency.key list -> @@ -95,9 +94,9 @@ val clear_duplicates : val insert_header : 'a t -> ImportDataTypes.Table.t -> - (int * ImportCSV.DataType.t) array -> + (int * ImportDataTypes.Value.t) array -> unit T.result val query_headers : - 'a t -> ImportDataTypes.Table.t -> ImportCSV.DataType.t array T.result + 'a t -> ImportDataTypes.Table.t -> ImportDataTypes.Value.t array T.result (** Get all the headers from the database (used or not) *) diff --git a/lib/sql/dune b/lib/sql/dune index 9f9f205..bd77858 100644 --- a/lib/sql/dune +++ b/lib/sql/dune @@ -4,12 +4,11 @@ re sqlite3 calendar - importAnalyser - importCSV - importConf - importContainers importDataTypes + importContainers importErrors importExpression + importerSyntax + importAnalyser ) ) diff --git a/lib/sql/hashs.ml b/lib/sql/hashs.ml index af1f092..eced2b4 100644 --- a/lib/sql/hashs.ml +++ b/lib/sql/hashs.ml @@ -1,10 +1,8 @@ -(** - This module store the hash of the indexes ([extern_key]) for each table in +(** This module store the hash of the indexes ([extern_key]) for each table in order to update the file if the configuration changed. The hashes are stored in a table named [hashes] and are evaluated just - before inserting the values. -*) + before inserting the values. *) open StdLabels module Table = ImportDataTypes.Table @@ -75,5 +73,5 @@ let query : 'a T.t -> ImportDataTypes.Table.t -> int option T.result = let* _ = T.to_result state in match res with - | Some (ImportCSV.DataType.Integer i) -> Ok (Some i) + | Some (ImportDataTypes.Value.Integer i) -> Ok (Some i) | _ -> Ok None diff --git a/lib/sql/header.ml b/lib/sql/header.ml index 3cac5fb..1f4c9d6 100644 --- a/lib/sql/header.ml +++ b/lib/sql/header.ml @@ -13,7 +13,7 @@ let create_table : 'a T.t -> unit T.result = let insert_header : 'a T.t -> ImportDataTypes.Table.t -> - (int * ImportCSV.DataType.t) array -> + (int * ImportDataTypes.Value.t) array -> unit T.result = fun db table values -> let table_name = Table.name table in @@ -48,7 +48,8 @@ let insert_header : T.commit db let query_headers : - 'a T.t -> ImportDataTypes.Table.t -> ImportCSV.DataType.t array T.result = + 'a T.t -> ImportDataTypes.Table.t -> ImportDataTypes.Value.t array T.result + = fun db table -> let table_name = Table.name table in let query = diff --git a/lib/sql/t.ml b/lib/sql/t.ml index 202c535..2297125 100644 --- a/lib/sql/t.ml +++ b/lib/sql/t.ml @@ -39,14 +39,14 @@ let reset : Sqlite3.stmt -> unit result = fun statement -> to_result (Sqlite3.reset statement) let of_datatype = function - | ImportCSV.DataType.Float f -> Sqlite3.Data.FLOAT f - | ImportCSV.DataType.Integer i -> Sqlite3.Data.INT (Int64.of_int i) - | ImportCSV.DataType.Null -> Sqlite3.Data.NULL - | ImportCSV.DataType.Error _ -> Sqlite3.Data.NULL - | ImportCSV.DataType.Content s -> Sqlite3.Data.TEXT s - -let to_datatype : Sqlite3.Data.t -> ImportCSV.DataType.t = function - | Sqlite3.Data.NONE | Sqlite3.Data.NULL -> ImportCSV.DataType.Null - | Sqlite3.Data.INT i -> ImportCSV.DataType.Integer (Int64.to_int i) - | Sqlite3.Data.FLOAT f -> ImportCSV.DataType.Float f - | Sqlite3.Data.TEXT t | Sqlite3.Data.BLOB t -> ImportCSV.DataType.Content t + | ImportDataTypes.Value.Float f -> Sqlite3.Data.FLOAT f + | ImportDataTypes.Value.Integer i -> Sqlite3.Data.INT (Int64.of_int i) + | ImportDataTypes.Value.Null -> Sqlite3.Data.NULL + | ImportDataTypes.Value.Error _ -> Sqlite3.Data.NULL + | ImportDataTypes.Value.Content s -> Sqlite3.Data.TEXT s + +let to_datatype : Sqlite3.Data.t -> ImportDataTypes.Value.t = function + | Sqlite3.Data.NONE | Sqlite3.Data.NULL -> ImportDataTypes.Value.Null + | Sqlite3.Data.INT i -> ImportDataTypes.Value.Integer (Int64.to_int i) + | Sqlite3.Data.FLOAT f -> ImportDataTypes.Value.Float f + | Sqlite3.Data.TEXT t | Sqlite3.Data.BLOB t -> ImportDataTypes.Value.Content t diff --git a/lib/configuration/cte.ml b/lib/syntax/cte.ml index ff43d6d..ff43d6d 100644 --- a/lib/configuration/cte.ml +++ b/lib/syntax/cte.ml diff --git a/lib/configuration/cte.mli b/lib/syntax/cte.mli index 0f2b3e3..39897ef 100644 --- a/lib/configuration/cte.mli +++ b/lib/syntax/cte.mli @@ -1,9 +1,6 @@ -module Path = ImportDataTypes.Path -module Expression = ImportExpression.T - type t = { - filters : Path.t Expression.t list; - group : Path.t Expression.t option; + filters : ImportDataTypes.Path.t ImportExpression.T.t list; + group : ImportDataTypes.Path.t ImportExpression.T.t option; } (** Represent a filter to apply in the querry @@ -17,4 +14,4 @@ type t = { because all cf them can be evaluated at the same time, but as soon we have a group function, the result become dependant of the previous ones. *) -val of_filters : Path.t Expression.t list -> t list +val of_filters : ImportDataTypes.Path.t ImportExpression.T.t list -> t list diff --git a/lib/syntax/dune b/lib/syntax/dune new file mode 100755 index 0000000..1459d0c --- /dev/null +++ b/lib/syntax/dune @@ -0,0 +1,12 @@ +(library
+ (name importerSyntax)
+ (libraries
+ otoml
+ importDataTypes
+ importExpression
+ )
+ (preprocess (pps
+ ppx_deriving.show
+ ppx_deriving.eq
+ ))
+)
diff --git a/lib/configuration/syntax.ml b/lib/syntax/importerSyntax.ml index ee47277..7788613 100644 --- a/lib/configuration/syntax.ml +++ b/lib/syntax/importerSyntax.ml @@ -2,6 +2,7 @@ open StdLabels module E = ImportExpression.T module Table = ImportDataTypes.Table module Path = ImportDataTypes.Path +module CTE = Cte let toml_of_table Table.{ file; tab; name } = let values = [ ("file", Otoml.string file); ("name", Otoml.string name) ] in @@ -33,7 +34,7 @@ module Extern = struct ( "extern_key", Otoml.string @@ ImportExpression.Repr.repr ~top:true - (fun v -> ":" ^ ImportCSV.Csv.column_to_string v) + (fun v -> ":" ^ ImportDataTypes.Path.column_to_string v) extern.extern_key ); ("file", Otoml.string extern.target.file); ("allow_missing", Otoml.boolean extern.allow_missing); @@ -91,3 +92,46 @@ let repr t = in Otoml.table values + +let get_table_for_name : t -> string option -> Table.t = + fun conf name -> + match name with + | None -> conf.source + | Some name -> + if String.equal name conf.source.name then conf.source + else + let ext = + List.find conf.externals ~f:(fun (ext : Extern.t) -> + String.equal name ext.target.name) + in + ext.target + +let root_table : t -> Table.t = fun conf -> conf.source + +let get_dependancies_for_table : t -> Table.t -> Extern.t list = + fun conf source -> + let is_root = source = conf.source in + + List.filter conf.externals ~f:(fun (ext : Extern.t) -> + (* Enumerate the intern_key and check the source pointed by each column *) + ImportExpression.T.fold_values ext.intern_key ~init:false + ~f:(fun acc expr -> + if acc then acc + else + match expr.Path.alias with + | Some v -> String.equal v source.name + | None -> is_root)) + +let latest_version = 1 + +let dummy_conf = + { + source = { file = ""; tab = 0; name = "" }; + version = latest_version; + locale = Some "C"; + externals = []; + columns = []; + filters = []; + sort = []; + uniq = []; + } diff --git a/lib/syntax/importerSyntax.mli b/lib/syntax/importerSyntax.mli new file mode 100644 index 0000000..49b7364 --- /dev/null +++ b/lib/syntax/importerSyntax.mli @@ -0,0 +1,37 @@ +module Extern : sig + type t = { + intern_key : ImportDataTypes.Path.t ImportExpression.T.t; + target : ImportDataTypes.Table.t; + extern_key : ImportDataTypes.Path.column ImportExpression.T.t; + allow_missing : bool; + match_rule : string option; + } + [@@deriving show, eq] +end + +module CTE = Cte + +type t = { + version : int; + locale : string option; + source : ImportDataTypes.Table.t; + externals : Extern.t list; + columns : ImportDataTypes.Path.t ImportExpression.T.t list; + filters : ImportDataTypes.Path.t ImportExpression.T.t list; + sort : ImportDataTypes.Path.t ImportExpression.T.t list; + uniq : ImportDataTypes.Path.t ImportExpression.T.t list; +} + +val repr : t -> Otoml.t + +val root_table : t -> ImportDataTypes.Table.t +(** Get the root table, this table is the main table to load and each line in + this table will be processed *) + +val get_table_for_name : t -> string option -> ImportDataTypes.Table.t + +val get_dependancies_for_table : t -> ImportDataTypes.Table.t -> Extern.t list +(** Get all the externals refered by the source *) + +val dummy_conf : t +val latest_version : int diff --git a/tests/analyser_dependency.ml b/tests/analyser_dependency.ml index 8409d73..511b706 100644 --- a/tests/analyser_dependency.ml +++ b/tests/analyser_dependency.ml @@ -1,7 +1,7 @@ open StdLabels module A = ImportAnalyser.Dependency module Cont = ImportContainers -module Syntax = ImportConf.Syntax +module Syntax = ImporterSyntax module Expression = ImportExpression.T module Table = ImportDataTypes.Table open ConfLoader diff --git a/tests/analyser_filters.ml b/tests/analyser_filters.ml index c329af8..864cab7 100644 --- a/tests/analyser_filters.ml +++ b/tests/analyser_filters.ml @@ -1,4 +1,3 @@ -module CTE = ImportConf.CTE module Filters = ImportAnalyser.Filters module Chunk = ImportAnalyser.Chunk @@ -29,7 +28,9 @@ let empty_predicates () = The path is identified as pointing to a numeric column, and the associated query will replace null with 0. *) let simple_filter () = - let filter = CTE.of_filters Expression_builder.[ equal integer_one path' ] in + let filter = + ImporterSyntax.CTE.of_filters Expression_builder.[ equal integer_one path' ] + in let chunk_links = Chunk.create () in let chunk_predicates = Filters.generate_sql ~conf filter chunk_links in @@ -47,7 +48,8 @@ let simple_filter () = query will replace null with empty string. *) let multiple_filters () = let filter = - CTE.of_filters Expression_builder.[ integer_one; equal path' literal_zero ] + ImporterSyntax.CTE.of_filters + Expression_builder.[ integer_one; equal path' literal_zero ] in let chunk_links = Chunk.create () in let chunk_predicates = Filters.generate_sql ~conf filter chunk_links in @@ -66,7 +68,7 @@ let multiple_filters () = (** Create a simple configuration with a group expression and no other filters *) let group_filter () = - let filter = CTE.of_filters [ group_expression ] in + let filter = ImporterSyntax.CTE.of_filters [ group_expression ] in let chunk_links = Chunk.create () in let chunk_predicates = Filters.generate_sql ~conf filter chunk_links in @@ -90,7 +92,7 @@ let group_filter () = let expression_with_group () = let filter = - CTE.of_filters + ImporterSyntax.CTE.of_filters Expression_builder.[ equal integer_one path'; group_expression ] in let chunk_links = Chunk.create () in @@ -118,7 +120,7 @@ let expression_with_group () = included inside the CTE but in the main query. *) let group_with_expression () = let filter = - CTE.of_filters + ImporterSyntax.CTE.of_filters Expression_builder.[ group_expression; equal integer_one path' ] in let chunk_links = Chunk.create () in @@ -145,7 +147,9 @@ let group_with_expression () = (** Test the configuration with two group, each one generating it’s own CTE in the query *) let group_with_group () = - let filter = CTE.of_filters [ group_expression; group_expression ] in + let filter = + ImporterSyntax.CTE.of_filters [ group_expression; group_expression ] + in let chunk_links = Chunk.create () in let chunk_predicates = Filters.generate_sql ~conf filter chunk_links in diff --git a/tests/analyser_query_test.ml b/tests/analyser_query_test.ml index 16a9bca..fd8914b 100644 --- a/tests/analyser_query_test.ml +++ b/tests/analyser_query_test.ml @@ -1,8 +1,7 @@ open StdLabels module A = ImportAnalyser.Dependency module Q = ImportAnalyser.Query -module C = ImportConf -module Syntax = ImportConf.Syntax +module Syntax = ImporterSyntax module Expr = Expression_builder open Test_migration diff --git a/tests/confLoader.ml b/tests/confLoader.ml index 692cda9..13f9840 100644 --- a/tests/confLoader.ml +++ b/tests/confLoader.ml @@ -1,8 +1,8 @@ -let load' : string -> (ImportConf.Syntax.t, string) Result.t = +let load' : string -> (ImporterSyntax.t, string) Result.t = fun content -> Otoml.Parser.from_string content |> ImportConf.t_of_toml (** Read the configuration in toml and return the internal representation *) -let load : string -> ImportConf.Syntax.t = +let load : string -> ImporterSyntax.t = fun content -> Result.get_ok (load' content) let conf = @@ -37,7 +37,7 @@ let external_table_other = ImportDataTypes.Table.{ file = "other.xlsx"; tab = 1; name = "other" } let external_other = - ImportConf.Syntax.Extern. + ImporterSyntax.Extern. { intern_key = Path { alias = None; column = 1 }; target = external_table_other; @@ -50,7 +50,7 @@ let external_table_last = ImportDataTypes.Table.{ file = "last.xlsx"; tab = 1; name = "last_file" } let external_last = - ImportConf.Syntax.Extern. + ImporterSyntax.Extern. { intern_key = Path { alias = Some "other"; column = 1 }; target = external_table_last; diff --git a/tests/configuration_toml.ml b/tests/configuration_toml.ml index e51c727..0a36faf 100644 --- a/tests/configuration_toml.ml +++ b/tests/configuration_toml.ml @@ -34,7 +34,7 @@ let test_suit = | Error s -> raise (Failure s) | Ok result -> let expected = - ImportConf.Syntax.Extern. + ImporterSyntax.Extern. { target = { file = ""; tab = 1; name = "target" }; extern_key = Literal "_B"; @@ -6,10 +6,10 @@ otoml sqlite3 fmt + importerSyntax importConf importAnalyser importContainers - importCSV importDataTypes importErrors importExpression diff --git a/tests/expression_repr.ml b/tests/expression_repr.ml index 20a0484..9112704 100644 --- a/tests/expression_repr.ml +++ b/tests/expression_repr.ml @@ -4,7 +4,7 @@ module Expr = Expression_builder module M = Expr.Make (ImportExpression.Repr.E) open Test_migration -let eval = M.eval ~path_repr:ImportCSV.Csv.column_to_string +let eval = M.eval ~path_repr:ImportDataTypes.Path.column_to_string let test_expr expr = ImportExpression.Repr.E.observe ~top:true expr let assert_equal expected actual = diff --git a/tests/importCSV_test.ml b/tests/importCSV_test.ml index 8d83688..748c58b 100644 --- a/tests/importCSV_test.ml +++ b/tests/importCSV_test.ml @@ -1,4 +1,3 @@ -open ImportCSV open Test_migration let assert_equal = Alcotest.(check int "") @@ -8,19 +7,23 @@ let float_repr = let () = Alcotest.(check string) "Float repr" "3873921.620000" - ImportCSV.DataType.(to_string "C" (Float 3873921.62)) + ImportDataTypes.Value.(to_string "C" (Float 3873921.62)) in () let test_suit = [ - ("Column A" >:: fun () -> assert_equal 1 (Csv.column_of_string "A")); - ("Column a" >:: fun () -> assert_equal 1 (Csv.column_of_string "a")); + ( "Column A" >:: fun () -> + assert_equal 1 (ImportDataTypes.Path.column_of_string "A") ); + ( "Column a" >:: fun () -> + assert_equal 1 (ImportDataTypes.Path.column_of_string "a") ); ( "Column name" >:: fun () -> let () = for i = 1 to 1_000 do - let column_name = Csv.column_to_string i in - let column_index = Csv.column_of_string column_name in + let column_name = ImportDataTypes.Path.column_to_string i in + let column_index = + ImportDataTypes.Path.column_of_string column_name + in assert_equal i column_index done diff --git a/tests/importConf_test.ml b/tests/importConf_test.ml index 481b66c..5f98423 100644 --- a/tests/importConf_test.ml +++ b/tests/importConf_test.ml @@ -9,13 +9,14 @@ let check = Alcotest.(check (list Test_migration.extern_testable) "") configuration. *) let test_get_dependencies_for_source = "get_dependancies_for_table" >:: fun _ -> - let result = ImportConf.get_dependancies_for_table conf conf.source + let result = ImporterSyntax.get_dependancies_for_table conf conf.source and expected = [ external_other ] in check expected result let test_get_dependencies_for_other = "get_dependancies_for_table" >:: fun _ -> - let result = ImportConf.get_dependancies_for_table conf external_table_other + let result = + ImporterSyntax.get_dependancies_for_table conf external_table_other and expected = [ external_last ] in check expected result diff --git a/tests/sql_db.ml b/tests/sql_db.ml index ab402bc..34f5b12 100644 --- a/tests/sql_db.ml +++ b/tests/sql_db.ml @@ -67,10 +67,10 @@ columns = [ ":B", ":E"]|} ~input: - ImportCSV.DataType. + ImportDataTypes.Value. [ [ (0, Integer 123); (1, Integer 2); (4, Integer 5) ] ] ~expected: - (Ok ImportCSV.DataType.[ [| Content "123_"; Integer 2; Integer 5 |] ]) + (Ok ImportDataTypes.Value.[ [| Content "123_"; Integer 2; Integer 5 |] ]) (** Ensure the behavior of the sum function when a filter is given. It is expected to accumulate the values over each line *) @@ -87,7 +87,7 @@ columns = [ "sum(:C, [:B], [:A])", ]|} ~input: - ImportCSV.DataType. + ImportDataTypes.Value. [ [ (0, Integer 1); (1, Content "A"); (2, Integer 100) ]; [ (0, Integer 2); (1, Content "A"); (2, Integer 100) ]; @@ -95,7 +95,7 @@ columns = [ ] ~expected: (Ok - ImportCSV.DataType. + ImportDataTypes.Value. [ [| Integer 1; Integer 100 |]; [| Integer 2; Integer 200 |]; @@ -115,13 +115,18 @@ columns = [ "sum(:C, [], [])", ]|} ~input: - ImportCSV.DataType. + ImportDataTypes.Value. [ [ (0, Integer 1); (2, Integer 100) ]; [ (0, Integer 2); (2, Integer 100) ]; ] ~expected: - (Ok [ [| ImportCSV.DataType.Integer 1; ImportCSV.DataType.Integer 200 |] ]) + (Ok + [ + [| + ImportDataTypes.Value.Integer 1; ImportDataTypes.Value.Integer 200; + |]; + ]) (** Ensure the behavior of the sum function when no filter is given. It is expected to get the total sum for each line *) @@ -138,7 +143,7 @@ columns = [ "sum(:C, [], [:A])", ]|} ~input: - ImportCSV.DataType. + ImportDataTypes.Value. [ [ (0, Integer 1); (2, Integer 100) ]; [ (0, Integer 2); (2, Integer 100) ]; @@ -146,8 +151,12 @@ columns = [ ~expected: (Ok [ - [| ImportCSV.DataType.Integer 1; ImportCSV.DataType.Integer 100 |]; - [| ImportCSV.DataType.Integer 2; ImportCSV.DataType.Integer 200 |]; + [| + ImportDataTypes.Value.Integer 1; ImportDataTypes.Value.Integer 100; + |]; + [| + ImportDataTypes.Value.Integer 2; ImportDataTypes.Value.Integer 200; + |]; ]) let sum_group = @@ -163,7 +172,7 @@ columns = [ "sum(:C, [:A], [])", ]|} ~input: - ImportCSV.DataType. + ImportDataTypes.Value. [ [ (0, Integer 1); (2, Integer 100) ]; [ (0, Integer 1); (2, Integer 100) ]; @@ -171,7 +180,7 @@ columns = [ ] ~expected: (Ok - ImportCSV.DataType. + ImportDataTypes.Value. [ [| Integer 1; Integer 200 |]; [| Integer 1; Integer 200 |]; @@ -193,7 +202,7 @@ columns = [ "sum(:C, [:A], [])", ]|} ~input: - ImportCSV.DataType. + ImportDataTypes.Value. [ [ (0, Integer 1); (2, Integer 100) ]; [ (0, Integer 1); (2, Integer 100) ]; @@ -201,7 +210,7 @@ columns = [ ] ~expected: (Ok - ImportCSV.DataType. + ImportDataTypes.Value. [ [| Integer 1; Integer 200 |]; [| Integer 2; Integer 100 |] ]) let filter_group = @@ -222,13 +231,13 @@ filters = [ |} ~input: - ImportCSV.DataType. + ImportDataTypes.Value. [ [ (0, Integer 1); (1, Integer 100) ]; [ (0, Integer 2); (1, Integer 150) ]; [ (0, Integer 3); (1, Integer 200) ]; ] - ~expected:(Ok ImportCSV.DataType.[ [| Integer 3 |] ]) + ~expected:(Ok ImportDataTypes.Value.[ [| Integer 3 |] ]) (** The first filter will prevent the max value to pop, and only the second one will be reported *) @@ -251,13 +260,13 @@ filters = [ |} ~input: - ImportCSV.DataType. + ImportDataTypes.Value. [ [ (0, Integer 1); (1, Integer 100) ]; [ (0, Integer 2); (1, Integer 150) ]; [ (0, Integer 3); (1, Integer 200) ]; ] - ~expected:(Ok ImportCSV.DataType.[ [| Integer 2 |] ]) + ~expected:(Ok ImportDataTypes.Value.[ [| Integer 2 |] ]) (** In this case, we first filter the line and keep only the max value, but the second filter will match the result and will produce an empty list *) @@ -280,7 +289,7 @@ filters = [ |} ~input: - ImportCSV.DataType. + ImportDataTypes.Value. [ [ (0, Integer 1); (1, Integer 100) ]; [ (0, Integer 2); (1, Integer 150) ]; @@ -311,13 +320,13 @@ filters = [ |} ~input: - ImportCSV.DataType. + ImportDataTypes.Value. [ [ (0, Integer 1); (1, Integer 100) ]; [ (0, Integer 2); (1, Integer 150) ]; [ (0, Integer 3); (1, Integer 200) ]; ] - ~expected:(Ok [ [| ImportCSV.DataType.Integer 1 |] ]) + ~expected:(Ok [ [| ImportDataTypes.Value.Integer 1 |] ]) let test_suit = [ diff --git a/tests/test_migration.ml b/tests/test_migration.ml index e26f354..17e48cc 100644 --- a/tests/test_migration.ml +++ b/tests/test_migration.ml @@ -27,7 +27,7 @@ let sql_testable = let csv_data_type_testable = make_test (module struct - type t = ImportCSV.DataType.t = + type t = ImportDataTypes.Value.t = | Null | Error of string | Content of string @@ -38,7 +38,7 @@ let csv_data_type_testable = let csv_result = Alcotest.(list @@ array @@ csv_data_type_testable) let data_type_testable = make_test (module ImportDataTypes.Types) -let extern_testable = make_test (module ImportConf.Syntax.Extern) +let extern_testable = make_test (module ImporterSyntax.Extern) let table_testable = make_test (module ImportDataTypes.Table) let int_container_testable = make_test (module ImportContainers.IntSet) @@ -70,11 +70,11 @@ let chunk = let syntax = make_test (module struct - type t = ImportConf.Syntax.t + type t = ImporterSyntax.t let pp format t = Format.fprintf format "%s" - (Otoml.Printer.to_string (ImportConf.Syntax.repr t)) + (Otoml.Printer.to_string (ImporterSyntax.repr t)) let equal t1 t2 = t1 = t2 end) |