# HG changeset patch
# User Vincent Hatakeyama <vincent.hatakeyama@xcg-consulting.fr>
# Date 1678101583 -3600
# Mon Mar 06 12:19:43 2023 +0100
# Node ID e9c42cce0f109ddb48aa85e12122bd17c6f06bdf
# Parent ea0e6eb01497a1dca25ebe08ba784c12a2741c06
✨ add support for python_packages
diff --git a/NEWS.rst b/NEWS.rst
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -2,6 +2,11 @@
History
=======
+20.0.0
+------
+
+Support for Odoo 16.0 super project: add new python_packages configuration key.
+
19.1.0
------
diff --git a/odoo_scripts/conf2reST.py b/odoo_scripts/conf2reST.py
--- a/odoo_scripts/conf2reST.py
+++ b/odoo_scripts/conf2reST.py
@@ -6,7 +6,7 @@
import os
import subprocess
import sys
-from typing import Dict
+from typing import Dict, List, Optional
import yaml
@@ -15,29 +15,19 @@
_logger = logging.getLogger(__name__)
-__version__ = "1.1.1"
+__version__ = "1.2.0"
__date__ = "2016-07-15"
-__updated__ = "2022-12-16"
+__updated__ = "2023-03-06"
-def main(argv=None): # IGNORE:C0111
+def main(argv: Optional[List[str]] = None) -> int:
"""Parse arguments and launch conversion"""
- if argv is None:
- argv = sys.argv
- else:
- sys.argv.extend(argv)
+ program_version_message = f"%(prog)s {__version__} ({__updated__})"
+ program_shortdesc = __doc__.split(".", maxsplit=1)[0]
+ program_license = f"""{program_shortdesc}
- program_version = __version__
- program_build_date = str(__updated__)
- program_version_message = "%%(prog)s %s (%s)" % (
- program_version,
- program_build_date,
- )
- program_shortdesc = __doc__.split("\n", maxsplit=2)[1]
- program_license = """%s
-
- Created by Vincent Hatakeyama on %s.
- Copyright 2016, 2019, 2020, 2022 XCG Consulting. All rights reserved.
+ Created by Vincent Hatakeyama on {__date__}.
+ Copyright 2016, 2019, 2020, 2022, 2023 XCG Consulting. All rights reserved.
Licensed under the MIT License
@@ -45,10 +35,7 @@
or conditions of any kind, either express or implied.
USAGE
-""" % (
- program_shortdesc,
- str(__date__),
- )
+"""
# Argument parsing
parser = basic_parser(description=program_license, version=program_version_message)
parser.add_argument(
@@ -59,9 +46,10 @@
)
parser.add_argument("-o", "--output", help="Output file name")
- nmspc = parser.parse_args()
+ nmspc = parser.parse_args(argv)
apply(nmspc)
conf2rst(nmspc.directory, nmspc.output)
+ return 0
def conf2rst(directory: str, output=None):
@@ -74,24 +62,40 @@
configuration = Configuration(directory)
addon_dirs = configuration.addons_path
- # pip3 freeze
- reqs = {}
- pip_freeze = subprocess.check_output(["pip3", "freeze"])
- for element in pip_freeze.split():
- lib_and_version = element.decode("utf-8").split("==")
- if len(lib_and_version) == 1: # No explicit version (no ==).
- library, version = lib_and_version[0], "N/A"
- else:
- library, version = lib_and_version
- reqs[library] = {"version": version}
-
rst: Dict[str, Dict[str, Dict[str, str]]] = {
- "python": reqs,
+ "python": {},
"tools": {},
"modules": {},
"other": {},
"group of modules": {},
}
+ # pip3 freeze
+ pip_freeze = subprocess.check_output(["pip3", "freeze"])
+ for element in pip_freeze.split(b"\n"):
+ line = element.decode("utf-8")
+ if line:
+ if " @ " in line:
+ library, version = line.split(" @ ")
+ elif line.startswith("-e "):
+ version, library = line[3:].split("#egg=")
+ else:
+ lib_and_version = line.split("==")
+ if len(lib_and_version) == 1: # No explicit version (no ==).
+ library, version = lib_and_version[0], "N/A"
+ else:
+ library, version = lib_and_version
+ if library.startswith("odoo-addon"):
+ section = "modules"
+ else:
+ section = "python"
+ if version.startswith("file:///"):
+ # In the case of complicated name, the replace call is not enough
+ version_split = version.split(library.replace("-", "_"))
+ if len(version_split) > 1:
+ version = version_split[1][1:].split("-py")[0]
+
+ rst[section][library] = {"version": version}
+
# load the conf file
# test if there is a .hgconf file
hgconf_path = os.path.join(directory, ".hgconf")
@@ -192,4 +196,4 @@
if __name__ == "__main__":
- main()
+ sys.exit(main())
diff --git a/odoo_scripts/config.py b/odoo_scripts/config.py
--- a/odoo_scripts/config.py
+++ b/odoo_scripts/config.py
@@ -5,7 +5,8 @@
import os
import re
import sys
-from collections import defaultdict
+from ast import literal_eval
+from collections import OrderedDict, defaultdict
from glob import glob
from typing import Any, Dict, List, Optional, Union
@@ -148,6 +149,27 @@
)
setattr(self, key, list(set_values))
+ self.python_packages: Dict[str, Dict[str, str]]
+ if toread("python_packages"):
+ self.python_packages = OrderedDict()
+ # add any expanded values
+ for config in self._expanded_configuration.values():
+ if hasattr(config, "python_packages"):
+ for python_package in config.python_packages:
+ self.python_packages[python_package] = config.python_packages[
+ python_package
+ ]
+ for key, value in literal_eval(
+ section.get("python_packages", "{}")
+ ).items():
+ value_with_default = {
+ "mount": ".",
+ "target": os.path.join("odoo", "addons", os.path.basename(key)),
+ "compile": "True",
+ }
+ value_with_default.update(value)
+ self.python_packages[key] = value_with_default
+
self.addons_path = {os.path.dirname(module) for module in self.modules}
"""Set of directory containing modules"""
diff --git a/odoo_scripts/docker_build.py b/odoo_scripts/docker_build.py
--- a/odoo_scripts/docker_build.py
+++ b/odoo_scripts/docker_build.py
@@ -19,9 +19,9 @@
_logger = logging.getLogger(__name__)
-__version__ = "1.3.0"
+__version__ = "1.4.0"
__date__ = "2018-04-04"
-__updated__ = "2023-02-27"
+__updated__ = "2023-03-02"
def add_build_options(parser: argparse.ArgumentParser):
@@ -140,17 +140,28 @@
buildargs = {}
if os.path.exists(".hg"):
+ description = check_output(
+ [
+ "hg",
+ "log",
+ "-r",
+ ".",
+ "-T",
+ "{latesttag}{sub('^-0-.*', '', '-{latesttagdistance}-h{node|short}')}",
+ ]
+ ).decode()
+ buildargs["VERSION"] = description
+ buildargs["SENTRY_RELEASE_ARG"] = description
tags_string = check_output(["hg", "identify", "--tags"]).decode()
- buildargs["VERSION"] = tags_string
- buildargs["SENTRY_RELEASE_ARG"] = tags_string
if tags_string:
tags = tags_string.split()
buildargs["VCS_URL"] = check_output(["hg", "paths", "default"]).decode()
buildargs["VCS_REF"] = check_output(["hg", "identify", "--id"]).decode()
elif os.path.exists(".git"):
+ description = check_output(["git", "describe"]).decode()
+ buildargs["VERSION"] = description
+ buildargs["SENTRY_RELEASE_ARG"] = description
tags_string = check_output(["git", "describe", "--tags"]).decode()
- buildargs["VERSION"] = tags_string
- buildargs["SENTRY_RELEASE_ARG"] = tags_string
if tags_string:
tags = tags_string.split()
buildargs["BUILD_DATE"] = datetime.datetime.now().isoformat()
diff --git a/odoo_scripts/docker_build_clean.py b/odoo_scripts/docker_build_clean.py
--- a/odoo_scripts/docker_build_clean.py
+++ b/odoo_scripts/docker_build_clean.py
@@ -7,28 +7,24 @@
import sys
from typing import List, Optional
+from .docker_build_copy import PYTHON_PACKAGES_DIR
from .list_modules import MODULES_LIST_FILE
from .parsing import apply, basic_parser
_logger = logging.getLogger(__name__)
-__version__ = "2.0.0"
+__version__ = "2.1.0"
__date__ = "2020-06-30"
-__updated__ = "2021-12-10"
+__updated__ = "2023-02-27"
def __parser():
- program_version = __version__
- program_build_date = str(__updated__)
- program_version_message = "%%(prog)s %s (%s)" % (
- program_version,
- program_build_date,
- )
- program_shortdesc = __doc__.split(".", maxsplit=1)[0]
- program_license = """%s
+ program_version_message = f"%(prog)s {__version__} ({__updated__})"
+ program_short_description = __doc__.split(".", maxsplit=1)[0]
+ program_license = f"""{program_short_description}
- Created by Vincent Hatakeyama on %s.
- Copyright 2020, 2021 XCG Consulting. All rights reserved.
+ Created by Vincent Hatakeyama on {__date__}.
+ Copyright 2020, 2021, 2023 XCG Consulting. All rights reserved.
Licensed under the MIT License
@@ -36,10 +32,7 @@
or conditions of any kind, either express or implied.
USAGE
- """ % (
- program_shortdesc,
- str(__date__),
- )
+ """
parser = basic_parser(program_license, program_version_message)
return parser
@@ -58,7 +51,7 @@
"""Clean up after a build"""
if os.path.exists(MODULES_LIST_FILE):
os.remove(MODULES_LIST_FILE)
- for directory in ("odoo_modules", "odoo_setup"):
+ for directory in ("odoo_modules", "odoo_setup", PYTHON_PACKAGES_DIR):
if os.path.exists(directory):
shutil.rmtree(directory)
diff --git a/odoo_scripts/docker_build_copy.py b/odoo_scripts/docker_build_copy.py
--- a/odoo_scripts/docker_build_copy.py
+++ b/odoo_scripts/docker_build_copy.py
@@ -4,7 +4,7 @@
import logging
import os
import sys
-from subprocess import call
+from subprocess import check_call
from typing import List, Optional
from .config import Config
@@ -13,23 +13,20 @@
_logger = logging.getLogger(__name__)
-__version__ = "2.0.0"
+__version__ = "2.1.0"
__date__ = "2020-06-30"
-__updated__ = "2021-12-10"
+__updated__ = "2023-02-28"
+
+PYTHON_PACKAGES_DIR = "python_packages"
def __parser():
- program_version = __version__
- program_build_date = str(__updated__)
- program_version_message = "%%(prog)s %s (%s)" % (
- program_version,
- program_build_date,
- )
+ program_version_message = f"%(prog)s {__version__} ({__updated__})"
program_shortdesc = __doc__.split(".", maxsplit=1)[0]
- program_license = """%s
+ program_license = f"""{program_shortdesc}
- Created by Vincent Hatakeyama on %s.
- Copyright 2020, 2021 XCG Consulting. All rights reserved.
+ Created by Vincent Hatakeyama on {__date__}.
+ Copyright 2020, 2021, 2023 XCG Consulting. All rights reserved.
Licensed under the MIT License
@@ -37,10 +34,7 @@
or conditions of any kind, either express or implied.
USAGE
- """ % (
- program_shortdesc,
- str(__date__),
- )
+ """
parser = basic_parser(program_license, program_version_message)
return parser
@@ -54,13 +48,15 @@
return 0
+def _target_path(path: str) -> str:
+ return path.replace(os.path.sep, "-")
+
+
def copy():
"""Copy modules for a build"""
c = Config()
modules = [os.path.realpath(module) for module in c.modules]
- # copy files
target = "odoo_modules"
- _logger.info("Copying Odoo modules files to %s", target)
if not os.path.exists(target):
os.mkdir(target)
cmd = (
@@ -80,7 +76,83 @@
+ [target]
)
_logger.debug(" ".join(cmd))
- call(cmd)
+ check_call(cmd)
+
+ target = PYTHON_PACKAGES_DIR
+ if not os.path.exists(PYTHON_PACKAGES_DIR):
+ os.mkdir(target)
+ copy_paths = set()
+ packages_to_compile = []
+ requirements = []
+ # Try to include the CVS information in target so that setuptools-scm and
+ # setuptools-odoo work as expected
+ for package, options in c.python_packages.items():
+ if options["compile"] in ("True", "true"):
+ packages_to_compile.append(package)
+ else:
+ cvs_parent = package
+ package_path_in_target = ""
+ while not (
+ os.path.exists(os.path.join(cvs_parent, ".git"))
+ or os.path.exists(os.path.join(cvs_parent, ".hg"))
+ or os.path.exists(os.path.join(cvs_parent, ".hg_archival.txt"))
+ or os.path.exists(os.path.join(cvs_parent, ".git_archival.txt"))
+ ):
+ package_path_in_target = os.path.join(
+ os.path.basename(cvs_parent), package_path_in_target
+ )
+ if os.path.dirname(cvs_parent) == "":
+ # stop at one level in the super project
+ break
+ cvs_parent = os.path.dirname(cvs_parent)
+ copy_paths.add(cvs_parent)
+ requirements.append(
+ os.path.join(target, _target_path(cvs_parent), package_path_in_target)
+ )
+ for copy_path in copy_paths:
+ src, dest = copy_path + os.path.sep, os.path.join(
+ target, _target_path(copy_path)
+ )
+ _logger.info("Copying package %s to %s", src, dest)
+ if not os.path.exists(target):
+ os.mkdir(target)
+ cmd = [
+ "rsync",
+ "--delete",
+ "--include=core",
+ "--copy-links",
+ "--exclude='*.pyc'",
+ "-r",
+ "--times",
+ src,
+ dest,
+ ]
+ _logger.debug(" ".join(cmd))
+ check_call(cmd)
+ # The option to ignore the python verion can be an issue with newer syntax.
+ # In the CI it will work fine, locally, maybe add an option to avoid compiling?
+ if packages_to_compile:
+ cmd = [
+ sys.executable,
+ "-m",
+ "pip",
+ "wheel",
+ "--no-deps",
+ "-w",
+ target,
+ "--ignore-requires-python",
+ ] + packages_to_compile
+ _logger.debug(" ".join(cmd))
+ check_call(cmd)
+ for entry in os.listdir(target):
+ if entry.endswith(".whl"):
+ requirements.append(os.path.join(".", target, entry))
+ # Write requirements
+ # pip needs to be run with pip install -r <target>/requirements for the path in the
+ # file to be valid
+ with open(os.path.join(target, "requirements"), "wt") as f:
+ f.write("\n".join(requirements))
+
# copy setup files to odoo_setup
source = "setup/"
target = "odoo_setup"
@@ -101,7 +173,7 @@
"--times",
] + [source, target]
_logger.debug(" ".join(cmd))
- call(cmd)
+ check_call(cmd)
list_modules()
diff --git a/odoo_scripts/docker_client.py b/odoo_scripts/docker_client.py
--- a/odoo_scripts/docker_client.py
+++ b/odoo_scripts/docker_client.py
@@ -241,8 +241,10 @@
mount_list: List[Mount] = []
mount_dict: Dict[str, str] = {}
if config.odoo_type in (ODOO_7, ODOO_8, ODOO_9, ODOO_10):
+ base_python_path = "/usr/local/lib/python2.7/dist-packages/"
target_path = "/opt/odoo/sources/odoo/addons"
else:
+ base_python_path = os.path.dirname(get_odoo_base_path(config))
target_path = get_odoo_base_path(config) + "/addons"
for module in modules:
@@ -251,6 +253,15 @@
mount_list.append(Mount(full_target_path, full_project_path, "bind"))
mount_dict[full_target_path] = full_project_path
+ _logger.debug("python packages: %s", ",".join(config.python_packages))
+ for python_package in config.python_packages.items():
+ full_target_path = os.path.join(base_python_path, python_package[1]["target"])
+ full_project_path = os.path.realpath(
+ os.path.join(project_path, python_package[0], python_package[1]["mount"])
+ )
+ mount_list.append(Mount(full_target_path, full_project_path, "bind"))
+ mount_dict[full_target_path] = full_project_path
+
return mount_list, mount_dict
@@ -276,6 +287,7 @@
read_only=True,
)
)
+ _logger.info("mounts len: %s", len(mounts))
return mounts
diff --git a/odoo_scripts/update_duplicate_sources.py b/odoo_scripts/update_duplicate_sources.py
--- a/odoo_scripts/update_duplicate_sources.py
+++ b/odoo_scripts/update_duplicate_sources.py
@@ -15,9 +15,9 @@
_logger = logging.getLogger(__name__)
-__version__ = "1.1.0"
+__version__ = "1.2.0"
__date__ = "2020-06-19"
-__updated__ = "2021-10-01"
+__updated__ = "2023-02-27"
def __parser():
@@ -31,7 +31,7 @@
program_license = """%s
Created by Vincent Hatakeyama on %s.
- Copyright 2018, 2020, 2021 XCG Consulting. All rights reserved.
+ Copyright 2018, 2020, 2021, 2023 XCG Consulting. All rights reserved.
Licensed under the MIT License
@@ -135,6 +135,7 @@
+ conf.modules
+ conf.dependencies
+ conf.other_sources
+ + list(conf.python_packages.keys())
+ [duplicate_destination.decode()]
)
_logger.debug(" ".join(cmd))