# HG changeset patch
# User Vincent Hatakeyama <vincent.hatakeyama@xcg-consulting.fr>
# Date 1591005425 -7200
# Mon Jun 01 11:57:05 2020 +0200
# Node ID eeda6a70ac8faadfc2cc23eec463621b68627c13
# Parent 324e97070580f9bbe280a269c5f61069dcff3d37
✨ add context option to import_json and also allow delimiter and context options to be in a file
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -38,6 +38,8 @@
- sleep 35
# then test the setup files
- import_jsonrpc -v --host localhost --password admin -d test_setup --protocol jsonrpc -p 8069 --directory tests/import
+ - import_jsonrpc -v --host localhost --password admin -d test_setup --protocol jsonrpc -p 8069 --file tests/context_import/res.company.csv --context test=value
+ - import_jsonrpc -v --host localhost --password admin -d test_setup --protocol jsonrpc -p 8069 --directory tests/import_emptyyaml --delimiter ";"
docker:
stage: build
diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -163,6 +163,9 @@
- many2one and many2many fields must use the xmlid of the field they are referencing, with the column name in the form field_name/id or field_name:id.
- one2many fields in the same file are not supported for now,
- date and datetime must be in ISO format.
+- context does not include base.import usual import key, so they need to be added as argument or in the configuration file.
+
+import_jsonrpc tries to find an `import.yaml` file alongside any imported file. It will use the delimiter indicated in the file if it finds one and will add the context key to the context used in the jsonrpc calls.
setup.cfg
=========
diff --git a/odoo_scripts/import_jsonrpc.py b/odoo_scripts/import_jsonrpc.py
--- a/odoo_scripts/import_jsonrpc.py
+++ b/odoo_scripts/import_jsonrpc.py
@@ -5,8 +5,10 @@
import csv
import logging
import sys
+from os import path
from typing import Callable, Dict, List, Tuple, Union
+import yaml
from dateutil.parser import isoparse
from odoorpc.error import RPCError
@@ -30,6 +32,7 @@
timeout: int,
model_filenames: List[Tuple[str, str, str]],
delimiter: str = ",",
+ extra_context: Dict[str, str] = dict(),
) -> int:
o, session_id = odoo_login(
database, host, login, password, port, protocol, timeout
@@ -107,16 +110,50 @@
)
o.env.context["tz"] = ""
+ o.env.context.update(extra_context)
+ # this dictionary is used to store the import.yaml loaded data to avoid
+ # accessing the disk too frequently
+ import_yaml_cache = dict()
for model, csv_file, lang in model_filenames:
_logger.info(
"Importing - %s in model %s (lang %s)", csv_file, model, lang
)
+ yaml_filename = path.join(path.dirname(csv_file), "import.yaml")
+ yaml_delimiter = old_context = None
+ if path.exists(yaml_filename):
+ if yaml_filename not in import_yaml_cache:
+ _logger.info(
+ "Importing - reading options from yaml file %s",
+ yaml_filename,
+ )
+ with open(yaml_filename, "r") as stream:
+ data = yaml.load(stream, Loader=yaml.BaseLoader)
+ import_yaml_cache[yaml_filename] = (
+ data if data is not None else dict()
+ )
+ else:
+ _logger.info(
+ "Importing - using options from yaml file %s",
+ yaml_filename,
+ )
+ yaml_data = import_yaml_cache[yaml_filename]
+ yaml_delimiter = yaml_data.get("delimiter", None)
+ extra_context_yaml = yaml_data.get("context", None)
+ if extra_context_yaml:
+ old_context = dict(o.env.context)
+ _logger.debug("Old context %s", old_context)
+ o.env.context.update(extra_context_yaml)
+ _logger.debug("Updated context %s", o.env.context)
+ else:
+ _logger.debug("Importing - no %s file to read from", yaml_filename)
+
o.env.context["lang"] = lang
created: List[int] = list()
wrote: List[int] = list()
+ this_file_delimiter = yaml_delimiter if yaml_delimiter else delimiter
with open(csv_file, "r") as csvfile:
reader = csv.DictReader(
- csvfile, delimiter=delimiter, quotechar='"'
+ csvfile, delimiter=this_file_delimiter, quotechar='"'
)
# stored in this dict is the name of the odoo field and
# a conversion method for data
@@ -131,7 +168,7 @@
):
odoo_field_name = column_name[:-3]
if odoo_field_name != "id":
- _logger.debug('%s: %s', model, odoo_field_name)
+ _logger.debug("%s: %s", model, odoo_field_name)
field = _get_imf(model, odoo_field_name)
ttype = field.ttype
_logger.info(
@@ -152,10 +189,7 @@
converter = _convert_identity
else:
converter = _convert_identity_nullable
- elif ttype in (
- "date",
- "datetime",
- ):
+ elif ttype in ("date", "datetime"):
if field.required:
converter = _convert_date
else:
@@ -224,6 +258,17 @@
_logger.info(
"%s: created %d, wrote %d", model, len(created), len(wrote)
)
+ if old_context:
+ key_to_pop = list()
+ for k in o.env.context:
+ if k in old_context:
+ if o.env.context[k] != old_context[k]:
+ o.env.context[k] = old_context[k]
+ else:
+ key_to_pop.append(k)
+ for k in key_to_pop:
+ o.env.context.pop(k)
+ _logger.debug("restored context: %s", o.env.context)
return 0
@@ -257,10 +302,19 @@
parser.add_argument(
"--delimiter", help="CSV delimiter [default: %(default)s]", default=","
)
+ parser.add_argument(
+ "--context", help="Extra context (key=value)", action="append"
+ )
nmspc = parser.parse_args(argv)
logging_from_verbose(nmspc)
+ extra_context = dict()
+ if nmspc.context:
+ for context in nmspc.context:
+ k, v = context.split("=", 1)
+ extra_context[k] = v
+
return import_with_jsonrpc(
login=nmspc.login,
password=nmspc.password,
@@ -269,8 +323,9 @@
protocol=nmspc.protocol,
timeout=nmspc.timeout,
database=nmspc.database,
- model_filenames=extract_info_from_parsed(nmspc),
+ model_filenames=extract_info_from_parsed(nmspc, r"(.*/)?import\.yaml"),
delimiter=nmspc.delimiter,
+ extra_context=extra_context,
)
diff --git a/odoo_scripts/importing.py b/odoo_scripts/importing.py
--- a/odoo_scripts/importing.py
+++ b/odoo_scripts/importing.py
@@ -17,18 +17,28 @@
)
-def extract_info_from_parsed(nmspc) -> List[Tuple[str, str, str]]:
- return extract_info_from_filename(nmspc.file, nmspc.directory)
+def extract_info_from_parsed(
+ nmspc: argparse.Namespace, expected_filenames: str = None
+) -> List[Tuple[str, str, str]]:
+ """
+ :param nmspc: result of parser.parse_args
+ :param expected_filenames: a pattern to use in re.compile
+ :return: list of (model, filename, lang)
+ """
+ return extract_info_from_filename(
+ nmspc.file, nmspc.directory, expected_filenames
+ )
def extract_info_from_filename(
- files, directories
+ files, directories, expected_filenames: str = None
) -> List[Tuple[str, str, str]]:
"""from filename or directories, extract model and lang
:param files:
:param directories:
- :return: list of model, filename, lang
+ :param expected_filenames: a pattern to use in re.compile
+ :return: list of (model, filename, lang)
"""
model_filenames: List[Tuple[str, str, str]] = list()
# files need to be named according to the following regexp
@@ -46,6 +56,11 @@
for directory in directories:
for root, subdirectories, files in os.walk(directory):
file_list.extend(os.path.join(root, f) for f in sorted(files))
+ expected_filenames_pattern = (
+ re.compile(expected_filenames)
+ if expected_filenames is not None
+ else None
+ )
for filename in file_list:
_logger.debug("%s to import", filename)
result = pattern.fullmatch(filename)
@@ -54,5 +69,15 @@
lang = result.group("lang")
model_filenames.append((model, filename, lang))
else:
- _logger.warning('Unrecognized file name "%s", ignored', filename)
+ if (
+ expected_filenames_pattern
+ and expected_filenames_pattern.match(filename)
+ ):
+ _logger.debug(
+ 'Expected file name "%s" found, ignored', filename
+ )
+ else:
+ _logger.warning(
+ 'Unrecognized file name "%s", ignored', filename
+ )
return model_filenames
diff --git a/tests/context_import/import.yaml b/tests/context_import/import.yaml
new file mode 100644
--- /dev/null
+++ b/tests/context_import/import.yaml
@@ -0,0 +1,3 @@
+delimiter: ;
+context:
+ value2: test
diff --git a/tests/import/2 res.company.csv b/tests/context_import/res.company.csv
copy from tests/import/2 res.company.csv
copy to tests/context_import/res.company.csv
--- a/tests/import/2 res.company.csv
+++ b/tests/context_import/res.company.csv
@@ -1,4 +1,2 @@
-id,name,currency_id/id
-base.main_company,Import Test,base.EUR
-company_uk,UK Import Test,base.GBP
-company_us,US Import Test,base.USD
+id;name
+company_2;Import Test
diff --git a/tests/import/1 res.lang.csv b/tests/import_emptyyaml/1 res.lang.csv
copy from tests/import/1 res.lang.csv
copy to tests/import_emptyyaml/1 res.lang.csv
--- a/tests/import/1 res.lang.csv
+++ b/tests/import_emptyyaml/1 res.lang.csv
@@ -1,3 +1,2 @@
-id,active
-base.lang_en_GB,1
-base.lang_fr,1
+id;active
+base.lang_es;1
diff --git a/tests/import_emptyyaml/import.yaml b/tests/import_emptyyaml/import.yaml
new file mode 100644