import_jsonrpc: Add --allow-import-errors

NEWS.rst

@@ -11,7 +11,7 @@
 
 ``docker_build`` also tag with any current tag if any.
 
-``import_jsonrpc``: Add ``--allow-missing-refs`` to log warnings on missing XMLIDs instead of stopping with a blocking error.
+``import_jsonrpc``: Add ``--allow-import-errors`` & ``--allow-missing-refs`` to log warnings without blocking errors.
 
 13.3.0
 ------

odoo_scripts/import_jsonrpc.py

@@ -36,6 +36,7 @@
     model_filenames: List[Dict[str, str]],
     delimiter: str = ",",
     extra_context: Dict[str, str] = dict(),
+    allow_import_errors: bool = False,
     allow_missing_refs: bool = False,
 ) -> int:
     o, session_id = odoo_login(
@@ -119,6 +120,9 @@
             else [(5, 0, 0)]
         )
 
+    def _one_line_error(error):
+        return str(error).replace("\n", " ").replace("\r", " ")
+
     o.env.context["tz"] = ""
     o.env.context.update(extra_context)
     # this dictionary is used to store the import.yaml loaded data to avoid
@@ -163,6 +167,7 @@
         o.env.context["lang"] = lang
         created: List[int] = list()
         wrote: List[int] = list()
+        import_errors: List[str] = list()  # used with allow_import_errors
         this_file_delimiter = yaml_delimiter if yaml_delimiter else delimiter
         with open(csv_file, "r") as csvfile:
             reader = csv.DictReader(
@@ -235,9 +240,14 @@
                     _logger.debug("Write %s (%d)", model, write_id)
                     try:
                         o.env[model].write(write_id, values)
-                    except RPCError:
-                        _logger.error("Current values (%s): %s", xmlid, values)
-                        raise
-                    wrote.append(write_id)
+                        wrote.append(write_id)
+                    except RPCError as error:
+                        if allow_import_errors:
+                            error_msg = f"{xmlid}: {_one_line_error(error)}"
+                            _logger.warning(error_msg)
+                            import_errors.append(error_msg)
+                        else:
+                            _logger.error("Values (%s): %s", xmlid, values)
+                            raise
                 else:
                     _logger.debug("Creating a %s", model)
@@ -242,4 +252,5 @@
                 else:
                     _logger.debug("Creating a %s", model)
+                    created_id = None
                     try:
                         created_id = o.env[model].create(values)
@@ -244,10 +255,15 @@
                     try:
                         created_id = o.env[model].create(values)
-                    except RPCError:
-                        _logger.error("Current values (%s): %s", xmlid, values)
-                        raise
-                    created.append(created_id)
-                    if xmlid:
+                        created.append(created_id)
+                    except RPCError as error:
+                        if allow_import_errors:
+                            error_msg = f"{xmlid}: {_one_line_error(error)}"
+                            _logger.warning(error_msg)
+                            import_errors.append(error_msg)
+                        else:
+                            _logger.error("Values (%s): %s", xmlid, values)
+                            raise
+                    if created_id and xmlid:
                         _logger.debug(
                             "Adding xmlid %s for %d", xmlid, created_id
                         )
@@ -260,7 +276,7 @@
                             }
                         )
                         xmlid_cache[xmlid] = created_id
-                if (len(created) + len(wrote)) % 100 == 0:
+                if (len(created) + len(wrote) + len(import_errors)) % 100 == 0:
                     _logger.info(
                         "%s progress: created %d, wrote %d",
                         model,
@@ -271,6 +287,10 @@
         _logger.info(
             "%s: created %d, wrote %d", model, len(created), len(wrote)
         )
+        if import_errors:
+            _logger.warning("%s: %d errors", model, len(import_errors))
+            with open(f"{csv_file}-errors.txt", "w") as error_f:
+                error_f.write("\n".join(import_errors))
         if old_context:
             key_to_pop = list()
             for k in o.env.context:
@@ -318,6 +338,15 @@
         "--context", help="Extra context (key=value)", action="append"
     )
     parser.add_argument(
+        "--allow-import-errors",
+        help=(
+            "Log errors when a record could not be created / updated but keep "
+            "going through the rest of the file; then write them into "
+            '"<filename>-errors.txt".'
+        ),
+        action="store_true",
+    )
+    parser.add_argument(
         "--allow-missing-refs",
         help=(
             "Log warnings on missing XMLIDs instead of stopping with a "
@@ -348,6 +377,7 @@
         ),
         delimiter=nmspc.delimiter,
         extra_context=extra_context,
+        allow_import_errors=nmspc.allow_import_errors,
         allow_missing_refs=nmspc.allow_missing_refs,
     )