open StdLabels module Table = ImportDataTypes.Table type 'a t = { header : 'a option; transaction : bool; insert_stmt : Sqlite3.stmt option; check_key_stmt : Sqlite3.stmt option; row_number : int; sheet_number : int; delayed : 'a list; } type insert_result = { insert_stmt : Sqlite3.stmt option; check_key_stmt : Sqlite3.stmt option; } type ('a, 'b) mapper = { get_row : 'b -> 'a Array.t; get_value : 'a -> ImportCSV.DataType.t; default : 'a; } module A = ImportAnalyser.Dependency let insert_row : mapper:(_, 'row) mapper -> A.t -> _ ImportSQL.Db.t -> 'row -> _ t -> (insert_result, ImportErrors.xlsError) result = fun ~mapper mapping db row state -> (* Extract all columns referenced in the keys or the columns to extract *) let keys_id = List.fold_left (A.keys mapping) ~init:ImportContainers.IntSet.empty ~f:(fun acc (keys : A.key) -> let columns = keys.A.columns in ImportContainers.IntSet.union acc (Lazy.force columns)) and columns_id = A.columns mapping in let ids = ImportContainers.IntSet.(union keys_id columns_id |> elements) in (* Filter only the required columns in the row *) let values = List.map ids ~f:(fun i -> let index = i - 1 in let value = try Array.get (mapper.get_row row) index with | Stdlib.Invalid_argument _ -> (* If we have more headers than data, assume the value are NULL. This can happen when all the line tail is empty, Excel can give us a truncated line instead of a series of NULL *) mapper.default in (index, mapper.get_value value)) in let keys = A.keys mapping in let execution = let ( let* ) = Result.bind in let* check_key_stmt, result = ImportSQL.Db.eval_key db state.check_key_stmt keys values in let no_null = (* We check if we have at least one key which is not null — and in such case we ignore the line. If multiple keys are presents, we ensure there is at least one non null here. *) match result with | [] -> true | _ -> List.exists result ~f:(function | Sqlite3.Data.FLOAT _ | Sqlite3.Data.INT _ -> true | Sqlite3.Data.BLOB t | Sqlite3.Data.TEXT t -> not (String.equal "" t) | Sqlite3.Data.NONE | Sqlite3.Data.NULL -> false) in let* _ = match no_null with | true -> Ok () | false -> Error (Failure "The key is null") in let* statement = match state.insert_stmt with | None -> ImportSQL.Db.prepare_insert db mapping | Some v -> Ok v in let* _ = ImportSQL.Db.insert db statement ~id:state.row_number values in let* _ = ImportSQL.Db.reset statement in Helpers.Console.update_cursor (); Ok { insert_stmt = Some statement; check_key_stmt } in (* In case of error, wrap the exception to get the line *) Result.map_error (fun e -> ImportErrors. { source = ImportAnalyser.Dependency.table mapping; sheet = state.sheet_number; row = state.row_number; target = None; value = CSV.DataType.Content (String.concat ~sep:", " []); exn = e; }) execution (** Load the row with all the informations associated with this sheet. If an error has already been raised during the sheet, ignore this row only. *) let run_row : log_error:ImportErrors.t -> mapper:(_, 'row) mapper -> A.t -> _ ImportSQL.Db.t -> 'row -> 'a t -> 'a t = fun ~log_error ~mapper mapping db row state -> match insert_row ~mapper mapping db row state with | Ok { insert_stmt; check_key_stmt } -> { state with insert_stmt; check_key_stmt; row_number = state.row_number + 1; } | Error e -> Option.iter (fun v -> ignore @@ ImportSQL.Db.finalize v) state.insert_stmt; Option.iter (fun v -> ignore @@ ImportSQL.Db.finalize v) state.check_key_stmt; ImportErrors.output_error log_error e; { state with insert_stmt = None; check_key_stmt = None; row_number = state.row_number + 1; } let clear : log_error:ImportErrors.t -> 'a ImportSQL.Db.t -> A.t -> ImportConf.Syntax.t -> unit ImportSQL.Db.result = fun ~log_error db mapping conf -> ImportSQL.Db.clear_duplicates db (A.table mapping) (A.keys mapping) ~f:(fun values -> let line = match snd @@ Array.get values 0 with | ImportCSV.DataType.Integer i -> i | _ -> -1 and value = snd @@ Array.get values 1 and target = match snd @@ Array.get values 2 with | ImportCSV.DataType.Content s -> Some (ImportConf.get_table_for_name conf (Some s)) | _ -> None in let error = ImportErrors. { source = A.table mapping; sheet = (A.table mapping).tab; row = line; target; value; exn = Failure "Duplicated key"; } in ImportErrors.output_error log_error error)