add option in import_jsonrpc for CSV delimiter

Also include missing script in setup

docker_build_copy

@@ -6,11 +6,10 @@
 # template version 2.7
 
 # TODO add a way to bypass the value, maybe with a key in the setup.cfg file
-here=$(dirname $0)
-project_home=$(cd $here && cd .. && echo $PWD)
+project_home=$PWD
 static_dir=$project_home/static/
 odoo_modules="$(python3 -B -c "import configparser ; c = configparser.ConfigParser() ; c.read('${project_home}/setup.cfg') ; print (c.has_option('odoo_scripts', 'modules') and ' '.join(c.get('odoo_scripts', 'modules').split()) or '')")"
 
 # Copy modules when specified
 if [[ -n "$odoo_modules" ]];
 then
@@ -11,10 +10,16 @@
 static_dir=$project_home/static/
 odoo_modules="$(python3 -B -c "import configparser ; c = configparser.ConfigParser() ; c.read('${project_home}/setup.cfg') ; print (c.has_option('odoo_scripts', 'modules') and ' '.join(c.get('odoo_scripts', 'modules').split()) or '')")"
 
 # Copy modules when specified
 if [[ -n "$odoo_modules" ]];
 then
-    ${here}/list_modules
+    if ! [ -x "$(command -v list_modules)" ];
+    then
+      # TODO test that this works
+      $(dirname $0)/list_modules
+    else
+      list_modules
+    fi
     mkdir -p $project_home/odoo_modules
     set -e
     rsync --delete -C --exclude='.hg*' --exclude='.git*' --links --exclude='*.pyc' -r --times $(eval echo $odoo_modules) $project_home/odoo_modules

list_modules

@@ -6,11 +6,5 @@
 
 # template version 2.7
 
-here=$(dirname $0)
-project_home=$(cd $here && cd .. && echo $PWD)
-if type "python3" > /dev/null;
-then
-    python3 -B -c "import configparser ; c = configparser.ConfigParser() ; c.read('${project_home}/setup.cfg') ; print(','.join(c.get('odoo_scripts', 'module_list').split()))" > $project_home/odoo_modules_list
-else
-    python2 -B -c "import ConfigParser ; c = ConfigParser.SafeConfigParser() ; c.read('${project_home}/setup.cfg') ; print(','.join(c.get('odoo_scripts', 'module_list').split()))" > $project_home/odoo_modules_list
-fi
+project_home=$PWD
+python3 -B -c "import configparser ; c = configparser.ConfigParser() ; c.read('${project_home}/setup.cfg') ; print(','.join(c.get('odoo_scripts', 'module_list').split()))" > $project_home/odoo_modules_list

odoo_scripts/import_jsonrpc.py

@@ -5,7 +5,7 @@
 import csv
 import logging
 import sys
-from typing import Callable, Dict, List, Tuple
+from typing import Callable, Dict, List, Tuple, Union
 
 from . import logging_from_verbose, odoo_connect_parser, odoo_login
 from .importing import add_importing_file_parsing, extract_info_from_parsed
@@ -26,6 +26,7 @@
     port: int,
     timeout: int,
     model_filenames: List[Tuple[str, str, str]],
+    delimiter: str = ",",
 ) -> int:
     o, session_id = odoo_login(
         database, host, login, password, port, protocol, timeout
@@ -64,5 +65,12 @@
     def _convert_int(value) -> int:
         return int(value)
 
-    def _convert_many2many():
+    def _convert_many2one(xmlid: str) -> int:
+        if xmlid:
+            return _ref(xmlid)
+        return None
+
+    def _convert_many2many(
+        xmlids: str
+    ) -> List[Tuple[int, int, Union[int, List[int]]]]:
         return (
@@ -68,7 +76,5 @@
         return (
-            lambda xmlids: [
-                (6, 0, [_ref(xmlid) for xmlid in xmlids.split(",")])
-            ]
+            [(6, 0, [_ref(xmlid) for xmlid in xmlids.split(",")])]
             if xmlids
             else [(5, 0, 0)]
         )
@@ -81,7 +87,9 @@
         created: List[int] = list()
         wrote: List[int] = list()
         with open(csv_file, "r") as csvfile:
-            reader = csv.DictReader(csvfile, delimiter=",", quotechar='"')
+            reader = csv.DictReader(
+                csvfile, delimiter=delimiter, quotechar='"'
+            )
             # stored in this dict is the name of the odoo field and
             # a conversion method for data
             headers: Dict[str, Tuple[str, Callable]] = dict()
@@ -107,7 +115,7 @@
                             elif ttype in ("char", "text", "selection"):
                                 converter = _convert_identity
                             elif ttype in ("many2one",):
-                                converter = _ref
+                                converter = _convert_many2one
                             elif ttype in ("many2many",):
                                 converter = _convert_many2many
                             elif ttype in ("integer",):
@@ -149,6 +157,15 @@
                                 "res_id": created_id,
                             }
                         )
+                        xmlid_cache[xmlid] = created_id
+                if (len(created) + len(model)) % 100 == 0:
+                    _logger.info(
+                        "%s progress: created %d, wrote %d",
+                        model,
+                        len(created),
+                        len(wrote),
+                    )
+
         _logger.info(
             "%s: created %d, wrote %d", model, len(created), len(wrote)
         )
@@ -182,6 +199,9 @@
     )
     parser = odoo_connect_parser(program_license, program_version_message)
     add_importing_file_parsing(parser)
+    parser.add_argument(
+        "--delimiter", help="CSV delimiter [default: %(default)s]", default=","
+    )
 
     nmspc = parser.parse_args(argv)
     logging_from_verbose(nmspc)
@@ -195,6 +215,7 @@
         timeout=nmspc.timeout,
         database=nmspc.database,
         model_filenames=extract_info_from_parsed(nmspc),
+        delimiter=nmspc.delimiter,
     )
 
 

setup.py

@@ -30,6 +30,7 @@
         "docker_build_copy",
         "docker_build_clean",
         "update_duplicate_sources",
+        "list_modules",
     ],
     python_requires=">=3.6",
 )