From 6b377719c10d5ab3343fd5221f99a4a21008e25a Mon Sep 17 00:00:00 2001
From: Sébastien Dailly <sebastien@dailly.me>
Date: Thu, 14 Mar 2024 08:26:58 +0100
Subject: Initial commit

---
 lib/file_handler/state.ml | 178 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 178 insertions(+)
 create mode 100644 lib/file_handler/state.ml

(limited to 'lib/file_handler/state.ml')

diff --git a/lib/file_handler/state.ml b/lib/file_handler/state.ml
new file mode 100644
index 0000000..5b43aff
--- /dev/null
+++ b/lib/file_handler/state.ml
@@ -0,0 +1,178 @@
+open StdLabels
+module Table = ImportDataTypes.Table
+
+type 'a t = {
+  header : 'a option;
+  transaction : bool;
+  insert_stmt : Sqlite3.stmt option;
+  check_key_stmt : Sqlite3.stmt option;
+  row_number : int;
+  sheet_number : int;
+  delayed : 'a list;
+}
+
+type insert_result = {
+  insert_stmt : Sqlite3.stmt option;
+  check_key_stmt : Sqlite3.stmt option;
+}
+
+type ('a, 'b) mapper = {
+  get_row : 'b -> 'a Array.t;
+  get_value : 'a -> ImportCSV.DataType.t;
+  default : 'a;
+}
+
+module A = ImportAnalyser.Dependency
+
+let insert_row :
+    mapper:(_, 'row) mapper ->
+    A.t ->
+    _ ImportSQL.Db.t ->
+    'row ->
+    _ t ->
+    (insert_result, ImportErrors.xlsError) result =
+ fun ~mapper mapping db row state ->
+  (* Extract all columns referenced in the keys or the columns to extract *)
+  let keys_id =
+    List.fold_left (A.keys mapping) ~init:ImportContainers.IntSet.empty
+      ~f:(fun acc (keys : A.key) ->
+        let columns = keys.A.columns in
+        ImportContainers.IntSet.union acc (Lazy.force columns))
+  and columns_id = A.columns mapping in
+  let ids = ImportContainers.IntSet.(union keys_id columns_id |> elements) in
+
+  (* Filter only the required columns in the row *)
+  let values =
+    List.map ids ~f:(fun i ->
+        let index = i - 1 in
+        let value =
+          try Array.get (mapper.get_row row) index with
+          | Stdlib.Invalid_argument _ ->
+              (* If we have more headers than data, assume the value are NULL.
+                 This can happen when all the line tail is empty, Excel can
+                 give us a truncated line instead of a series of NULL *)
+              mapper.default
+        in
+        (index, mapper.get_value value))
+  in
+  let keys = A.keys mapping in
+
+  let execution =
+    let ( let* ) = Result.bind in
+    let* check_key_stmt, result =
+      ImportSQL.Db.eval_key db state.check_key_stmt keys values
+    in
+    let no_null =
+      (* We check if we have at least one key which is not null — and in such
+         case we ignore the line.
+
+         If multiple keys are presents, we ensure there is at least one non
+         null here.
+      *)
+      match result with
+      | [] -> true
+      | _ ->
+          List.exists result ~f:(function
+            | Sqlite3.Data.FLOAT _ | Sqlite3.Data.INT _ -> true
+            | Sqlite3.Data.BLOB t | Sqlite3.Data.TEXT t ->
+                not (String.equal "" t)
+            | Sqlite3.Data.NONE | Sqlite3.Data.NULL -> false)
+    in
+    let* _ =
+      match no_null with
+      | true -> Ok ()
+      | false -> Error (Failure "The key is null")
+    in
+
+    let* statement =
+      match state.insert_stmt with
+      | None -> ImportSQL.Db.prepare_insert db mapping
+      | Some v -> Ok v
+    in
+    let* _ = ImportSQL.Db.insert db statement ~id:state.row_number values in
+    let* _ = ImportSQL.Db.reset statement in
+
+    Helpers.Console.update_cursor ();
+    Ok { insert_stmt = Some statement; check_key_stmt }
+  in
+
+  (* In case of error, wrap the exception to get the line *)
+  Result.map_error
+    (fun e ->
+      ImportErrors.
+        {
+          source = ImportAnalyser.Dependency.table mapping;
+          sheet = state.sheet_number;
+          row = state.row_number;
+          target = None;
+          value = CSV.DataType.Content (String.concat ~sep:", " []);
+          exn = e;
+        })
+    execution
+
+(** Load the row with all the informations associated with this sheet. 
+
+ If an error has already been raised during the sheet, ignore this row only. *)
+let run_row :
+    log_error:ImportErrors.t ->
+    mapper:(_, 'row) mapper ->
+    A.t ->
+    _ ImportSQL.Db.t ->
+    'row ->
+    'a t ->
+    'a t =
+ fun ~log_error ~mapper mapping db row state ->
+  match insert_row ~mapper mapping db row state with
+  | Ok { insert_stmt; check_key_stmt } ->
+      {
+        state with
+        insert_stmt;
+        check_key_stmt;
+        row_number = state.row_number + 1;
+      }
+  | Error e ->
+      Option.iter (fun v -> ignore @@ ImportSQL.Db.finalize v) state.insert_stmt;
+      Option.iter
+        (fun v -> ignore @@ ImportSQL.Db.finalize v)
+        state.check_key_stmt;
+      ImportErrors.output_error log_error e;
+      {
+        state with
+        insert_stmt = None;
+        check_key_stmt = None;
+        row_number = state.row_number + 1;
+      }
+
+let clear :
+    log_error:ImportErrors.t ->
+    'a ImportSQL.Db.t ->
+    A.t ->
+    ImportConf.Syntax.t ->
+    unit ImportSQL.Db.result =
+ fun ~log_error db mapping conf ->
+  ImportSQL.Db.clear_duplicates db (A.table mapping) (A.keys mapping)
+    ~f:(fun values ->
+      let line =
+        match snd @@ Array.get values 0 with
+        | ImportCSV.DataType.Integer i -> i
+        | _ -> -1
+      and value = snd @@ Array.get values 1
+      and target =
+        match snd @@ Array.get values 2 with
+        | ImportCSV.DataType.Content s ->
+            Some (ImportConf.get_table_for_name conf (Some s))
+        | _ -> None
+      in
+      let error =
+        ImportErrors.
+          {
+            source = A.table mapping;
+            sheet = (A.table mapping).tab;
+            row = line;
+            target;
+            value;
+            exn = Failure "Duplicated key";
+          }
+      in
+
+      ImportErrors.output_error log_error error)
-- 
cgit v1.2.3