From 69d9b6ada15af41fa5db179f71c2bb284c643f96 Mon Sep 17 00:00:00 2001 From: Sébastien Dailly Date: Fri, 6 Feb 2026 19:03:27 +0100 Subject: Added a dry-run mode which does not requires the data --- bin/importer.ml | 184 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 106 insertions(+), 78 deletions(-) (limited to 'bin') diff --git a/bin/importer.ml b/bin/importer.ml index 260d83b..ee06eb2 100644 --- a/bin/importer.ml +++ b/bin/importer.ml @@ -29,10 +29,11 @@ module Args = struct bom : bool; print_conf : bool; mapping_date : float option; + dry_run : bool; } - let load_conf : string -> ImporterSyntax.t = - fun file -> + let load_conf : bool ref -> string -> ImporterSyntax.t = + fun dry_run file -> let dirname = Filename.dirname file in match Filename.extension file with | _ -> ( @@ -57,7 +58,8 @@ module Args = struct ~context: { checkFile = - (fun f -> Sys.file_exists (Filename.concat dirname f)); + (fun f -> + !dry_run || Sys.file_exists (Filename.concat dirname f)); loadFile = (fun f -> Otoml.Parser.from_file (Filename.concat dirname f)); } @@ -69,16 +71,18 @@ module Args = struct exit 1 | Ok e -> e) - let load () = - let conf = ref ("", ImporterSyntax.dummy_conf) - and bom = ref true + let load : unit -> arguments = + fun () -> + let bom = ref true and usage = "importer [--conf configuration.toml]" - and print_conf = ref false in + and print_conf = ref false + and dry_run = ref false + and configuration_file = ref "" in let annon_fun _filename = print_endline usage; exit 1 - and set_conf file = conf := (file, load_conf file) in + and set_conf file = configuration_file := file in let speclist = [ ( "--version", @@ -90,18 +94,21 @@ module Args = struct ("--conf", Arg.String set_conf, "Configuration file"); ("-c", Arg.String set_conf, "Configuration file"); ("--no-bom", Arg.Clear bom, "Do not insert a BOM in the CSV"); + ("--dry-run", Arg.Set dry_run, "Do not read the files"); ( "--print-conf", Arg.Set print_conf, "Reformat the configuration file and exit" ); ] in let () = Arg.parse speclist annon_fun usage in + let conf = (!configuration_file, load_conf dry_run !configuration_file) in { - configuration = snd !conf; + configuration = snd conf; bom = !bom; - conf_name = fst !conf; + conf_name = fst conf; print_conf = !print_conf; mapping_date = None; + dry_run = !dry_run; } end @@ -154,17 +161,23 @@ let process_table : | _ -> Printf.printf "Loading document %s %!" source.name; let headers_opt = - let extension = String.lowercase_ascii (Filename.extension file) in - match extension with - | ".xlsx" -> + match conf.dry_run with + | true -> Lwt_main.run - @@ ImportFileHandler.Xlsx2sql.importInDatable ~dirname + @@ ImportFileHandler.Dry_run.importInDatable ~dirname ~conf:conf.configuration ~log_error mapping db - | ".csv" -> - Lwt_main.run - @@ ImportFileHandler.Csv2sql.importInDatable ~dirname - ~conf:conf.configuration ~log_error mapping db - | _ -> raise (ImportErrors.Unknown_source extension) + | false -> ( + let extension = String.lowercase_ascii (Filename.extension file) in + match extension with + | ".xlsx" -> + Lwt_main.run + @@ ImportFileHandler.Xlsx2sql.importInDatable ~dirname + ~conf:conf.configuration ~log_error mapping db + | ".csv" -> + Lwt_main.run + @@ ImportFileHandler.Csv2sql.importInDatable ~dirname + ~conf:conf.configuration ~log_error mapping db + | _ -> raise (ImportErrors.Unknown_source extension)) in Helpers.Console.close_cursor (); @@ -193,29 +206,29 @@ let check_deps : try ignore @@ Db.check_foreign db conf ext ~f:(fun values -> - Helpers.Console.update_cursor (); - - let row = - match snd (Array.get values 0) with - | ImportDataTypes.Value.Integer i -> i - | _ -> -1 - and value = snd (Array.get values 1) in - let error = - ImportErrors. - { - source; - sheet = source.Table.tab; - row; - value; - target = Some ext.ImporterSyntax.Extern.target; - exn = - Failure - (Printf.sprintf "Key '%s' not found" - (ImportDataTypes.Value.to_string "C" value)); - } - in - - ImportErrors.output_error log_error error); + Helpers.Console.update_cursor (); + + let row = + match snd (Array.get values 0) with + | ImportDataTypes.Value.Integer i -> i + | _ -> -1 + and value = snd (Array.get values 1) in + let error = + ImportErrors. + { + source; + sheet = source.Table.tab; + row; + value; + target = Some ext.ImporterSyntax.Extern.target; + exn = + Failure + (Printf.sprintf "Key '%s' not found" + (ImportDataTypes.Value.to_string "C" value)); + } + in + + ImportErrors.output_error log_error error); Helpers.Console.close_cursor () with | Sqlite3.Error _ -> @@ -245,7 +258,12 @@ let () = Otoml.Printer.to_channel ~collapse_tables:true stdout toml; exit 0); - let sqlfile = Filename.concat dirname (prefix ^ ".sqlite") in + let sqlfile = + match conf.dry_run with + | false -> Filename.concat dirname (prefix ^ ".sqlite") + | true -> ":memory:" + in + let conf = { conf with mapping_date = creation_date sqlfile } in (* The configuration is loaded and valid, we create the errors log file *) @@ -258,7 +276,11 @@ let () = *) let out_channel = try - Out_channel.open_bin (Filename.concat dirname (String.cat prefix ".csv")) + match conf.dry_run with + | false -> + Out_channel.open_bin + (Filename.concat dirname (String.cat prefix ".csv")) + | true -> Stdlib.stdout with | Sys_error e -> prerr_endline e; @@ -280,40 +302,46 @@ let () = (* Create the database *) ignore @@ Db.with_db sqlfile (fun db -> - let headers = - List.fold_left process_order ~init:Headers.SheeetMap.empty - ~f:(process_table db dirname log_error conf) - in - let () = - check_deps db log_error conf.configuration - conf.configuration.source - in - - let first_line = Headers.columns conf.configuration headers in - Csv.output_record out_csv first_line; - - (* Run the query *) - ignore @@ Db.create_view db conf.configuration; - Printf.printf "Extracting results %!"; - - let locale = Option.value ~default:"" conf.configuration.locale in - match - Db.query db conf.configuration ~f:(fun v -> - let arr = - Array.to_seq v |> Seq.map (printer locale) |> List.of_seq - in - - Helpers.Console.update_cursor (); - Csv.output_record out_csv arr) - with - | Ok () -> - Helpers.Console.close_cursor (); - Ok () - | Error e -> - Helpers.Console.close_cursor (); - print_endline @@ ImportErrors.repr_error e; - - Ok ())); + let headers = + List.fold_left process_order ~init:Headers.SheeetMap.empty + ~f:(process_table db dirname log_error conf) + in + let () = + check_deps db log_error conf.configuration conf.configuration.source + in + + let () = + match conf.dry_run with + | false -> + let first_line = Headers.columns conf.configuration headers in + Csv.output_record out_csv first_line + | true -> () + in + + (* Run the query *) + ignore @@ Db.create_view db conf.configuration; + Printf.printf "Extracting results %!"; + + let locale = Option.value ~default:"" conf.configuration.locale in + match + Db.query db conf.configuration ~f:(fun v -> + let arr = + Array.to_seq v |> Seq.map (printer locale) |> List.of_seq + in + + Helpers.Console.update_cursor (); + match conf.dry_run with + | false -> Csv.output_record out_csv arr + | true -> ()) + with + | Ok () -> + Helpers.Console.close_cursor (); + Ok () + | Error e -> + Helpers.Console.close_cursor (); + print_endline @@ ImportErrors.repr_error e; + + Ok ())); (* Close the error file *) match Lazy.is_val log_error with -- cgit v1.2.3