1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
|
open StdLabels
module Table = ImportDataTypes.Table
type 'a t = {
header : 'a option;
transaction : bool;
insert_stmt : Sqlite3.stmt option;
check_key_stmt : Sqlite3.stmt option;
row_number : int;
sheet_number : int;
delayed : 'a list;
}
type insert_result = {
insert_stmt : Sqlite3.stmt option;
check_key_stmt : Sqlite3.stmt option;
}
type ('a, 'b) mapper = {
get_row : 'b -> 'a Array.t;
get_value : 'a -> ImportCSV.DataType.t;
default : 'a;
}
module A = ImportAnalyser.Dependency
let insert_row :
mapper:(_, 'row) mapper ->
A.t ->
_ ImportSQL.Db.t ->
'row ->
_ t ->
(insert_result, ImportErrors.xlsError) result =
fun ~mapper mapping db row state ->
(* Extract all columns referenced in the keys or the columns to extract *)
let keys_id =
List.fold_left (A.keys mapping) ~init:ImportContainers.IntSet.empty
~f:(fun acc (keys : A.key) ->
let columns = keys.A.columns in
ImportContainers.IntSet.union acc (Lazy.force columns))
and columns_id = A.columns mapping in
let ids = ImportContainers.IntSet.(union keys_id columns_id |> elements) in
(* Filter only the required columns in the row *)
let values =
List.map ids ~f:(fun i ->
let index = i - 1 in
let value =
try Array.get (mapper.get_row row) index with
| Stdlib.Invalid_argument _ ->
(* If we have more headers than data, assume the value are NULL.
This can happen when all the line tail is empty, Excel can
give us a truncated line instead of a series of NULL *)
mapper.default
in
(index, mapper.get_value value))
in
let keys = A.keys mapping in
let execution =
let ( let* ) = Result.bind in
let* check_key_stmt, result =
ImportSQL.Db.eval_key db state.check_key_stmt keys values
in
let no_null =
(* We check if we have at least one key which is not null — and in such
case we ignore the line.
If multiple keys are presents, we ensure there is at least one non
null here.
*)
match result with
| [] -> true
| _ ->
List.exists result ~f:(function
| Sqlite3.Data.FLOAT _ | Sqlite3.Data.INT _ -> true
| Sqlite3.Data.BLOB t | Sqlite3.Data.TEXT t ->
not (String.equal "" t)
| Sqlite3.Data.NONE | Sqlite3.Data.NULL -> false)
in
let* _ =
match no_null with
| true -> Ok ()
| false -> Error (Failure "The key is null")
in
let* statement =
match state.insert_stmt with
| None -> ImportSQL.Db.prepare_insert db mapping
| Some v -> Ok v
in
let* _ = ImportSQL.Db.insert db statement ~id:state.row_number values in
let* _ = ImportSQL.Db.reset statement in
Helpers.Console.update_cursor ();
Ok { insert_stmt = Some statement; check_key_stmt }
in
(* In case of error, wrap the exception to get the line *)
Result.map_error
(fun e ->
ImportErrors.
{
source = ImportAnalyser.Dependency.table mapping;
sheet = state.sheet_number;
row = state.row_number;
target = None;
value = CSV.DataType.Content (String.concat ~sep:", " []);
exn = e;
})
execution
(** Load the row with all the informations associated with this sheet.
If an error has already been raised during the sheet, ignore this row only. *)
let run_row :
log_error:ImportErrors.t ->
mapper:(_, 'row) mapper ->
A.t ->
_ ImportSQL.Db.t ->
'row ->
'a t ->
'a t =
fun ~log_error ~mapper mapping db row state ->
match insert_row ~mapper mapping db row state with
| Ok { insert_stmt; check_key_stmt } ->
{
state with
insert_stmt;
check_key_stmt;
row_number = state.row_number + 1;
}
| Error e ->
Option.iter (fun v -> ignore @@ ImportSQL.Db.finalize v) state.insert_stmt;
Option.iter
(fun v -> ignore @@ ImportSQL.Db.finalize v)
state.check_key_stmt;
ImportErrors.output_error log_error e;
{
state with
insert_stmt = None;
check_key_stmt = None;
row_number = state.row_number + 1;
}
let clear :
log_error:ImportErrors.t ->
'a ImportSQL.Db.t ->
A.t ->
ImportConf.Syntax.t ->
unit ImportSQL.Db.result =
fun ~log_error db mapping conf ->
ImportSQL.Db.clear_duplicates db (A.table mapping) (A.keys mapping)
~f:(fun values ->
let line =
match snd @@ Array.get values 0 with
| ImportCSV.DataType.Integer i -> i
| _ -> -1
and value = snd @@ Array.get values 1
and target =
match snd @@ Array.get values 2 with
| ImportCSV.DataType.Content s ->
Some (ImportConf.get_table_for_name conf (Some s))
| _ -> None
in
let error =
ImportErrors.
{
source = A.table mapping;
sheet = (A.table mapping).tab;
row = line;
target;
value;
exn = Failure "Duplicated key";
}
in
ImportErrors.output_error log_error error)
|
