# HG changeset patch
# User Vincent Hatakeyama <vincent.hatakeyama@xcg-consulting.fr>
# Date 1649161345 -7200
# Tue Apr 05 14:22:25 2022 +0200
# Node ID b165d05bd4f8f1e1506ced6792f281e4e1177929
# Parent a2407b37b97de71f765c6a5ff7669525173fbd12
Support marabunta that is in the images.
Marabunta is not run when using do_tests.
Change how the tests are run with do_tests: modules not to be tested are installed, then modules to test are installed with test enabled. This fix the issues of installation tests that were not run previously.
diff --git a/NEWS.rst b/NEWS.rst
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -2,6 +2,15 @@
History
=======
+16.0.0
+------
+
+Support marabunta that is in the images.
+
+Marabunta is not run when using do_tests.
+
+Change how the tests are run with do_tests: modules not to be tested are installed, then modules to test are installed with test enabled. This fix the issues of installation tests that were not run previously.
+
15.3.1
------
diff --git a/odoo_scripts/config.py b/odoo_scripts/config.py
--- a/odoo_scripts/config.py
+++ b/odoo_scripts/config.py
@@ -168,6 +168,7 @@
"db_password",
"load-language",
"duplicate_repo",
+ "marabunta_migration_file",
):
setattr(self, key_format(key), section.get(key, None))
diff --git a/odoo_scripts/do_tests.py b/odoo_scripts/do_tests.py
--- a/odoo_scripts/do_tests.py
+++ b/odoo_scripts/do_tests.py
@@ -22,24 +22,19 @@
_logger = logging.getLogger(__name__)
-__version__ = "3.0.1"
+__version__ = "4.0.0"
__date__ = "2018-04-13"
-__updated__ = "2022-01-27"
+__updated__ = "2022-03-22"
def main(argv=None): # IGNORE:C0111
"""Parse arguments and docker build"""
- program_version = __version__
- program_build_date = str(__updated__)
- program_version_message = "%%(prog)s %s (%s)" % (
- program_version,
- program_build_date,
- )
- program_shortdesc = __doc__.split(".", maxsplit=1)[0]
- program_license = """%s
+ program_version_message = f"%(prog)s {__version__} ({__updated__})"
+ program_short_description = __doc__.split(".", maxsplit=1)[0]
+ program_license = f"""{program_short_description}
- Created by Vincent Hatakeyama on %s.
- Copyright 2018, 2020 XCG Consulting. All rights reserved.
+ Created by Vincent Hatakeyama on {__date__}.
+ Copyright 2018, 2020, 2022 XCG Consulting. All rights reserved.
Licensed under the MIT License
@@ -47,10 +42,7 @@
or conditions of any kind, either express or implied.
USAGE
-""" % (
- program_shortdesc,
- str(__date__),
- )
+"""
# the script assume it is launched from the super project
project_path = os.path.realpath(".")
@@ -139,7 +131,7 @@
)
parser.add_argument(
"--load-language",
- help="specifies the languages for the translations you want to be" " loaded",
+ help="specifies the languages for the translations you want to be loaded",
default=None,
dest="LOAD_LANGUAGE",
)
@@ -197,6 +189,7 @@
extensions.append("unaccent")
else:
_logger.debug("No sample configuration %s", sample_conf)
+ # our base images have unaccent activated
extensions.append("unaccent")
if not odoo_db_user:
@@ -228,6 +221,9 @@
"--no-flake8",
"--no-isort",
"--no-dev",
+ "--no-marabunta",
+ "--without-demo",
+ "",
]
if odoo_db_user:
args.append("--db_user")
@@ -254,6 +250,30 @@
args.append("--chown")
# also pass build options
args.extend(get_build_options(nmspc))
+
+ module_list = (
+ override_installed_module.split(",")
+ if override_installed_module
+ else config.module_list
+ )
+ module_list_tests = (
+ override_tested_module.split(",")
+ if override_tested_module
+ else config.module_list_tests
+ )
+
+ install_modules = ",".join(
+ module for module in module_list if module not in module_list_tests
+ )
+ tested_non_installed_modules = [
+ module for module in module_list_tests if module not in module_list
+ ]
+ if tested_non_installed_modules:
+ _logger.error(
+ "Modules to tests but not in module list: %s",
+ ", ".join(tested_non_installed_modules),
+ )
+
if recreate_db:
if start_postgresql:
container, stop_method, socket_path = docker_run_postgresql(
@@ -289,23 +309,25 @@
)
connection.commit()
connection.close()
- odoo_connection = connect(
+ with connect(
user=pg_user, database="postgres", host=socket_path, port=5432
- )
- odoo_connection.autocommit = True
- with odoo_connection.cursor() as cursor:
- _logger.debug("Drop database %s", dbname)
- cursor.execute('DROP DATABASE IF EXISTS "%s"' % dbname)
- _logger.debug("Create database %s", dbname)
- cursor.execute('CREATE DATABASE "%s" OWNER %s' % (dbname, odoo_db_user))
- odoo_connection = connect(
+ ) as odoo_connection:
+ odoo_connection.autocommit = True
+ with odoo_connection.cursor() as cursor:
+ _logger.debug("Drop database %s", dbname)
+ cursor.execute('DROP DATABASE IF EXISTS "%s"' % dbname)
+ _logger.debug("Create database %s", dbname)
+ cursor.execute(
+ 'CREATE DATABASE "%s" OWNER %s' % (dbname, odoo_db_user)
+ )
+ with connect(
user=pg_user, database=dbname, host=socket_path, port=5432
- )
- odoo_connection.autocommit = True
- with odoo_connection.cursor() as cursor:
- for extension in extensions:
- _logger.info("Adding extension %s", extension)
- cursor.execute("CREATE EXTENSION %s" % extension)
+ ) as odoo_connection:
+ odoo_connection.autocommit = True
+ with odoo_connection.cursor() as cursor:
+ for extension in extensions:
+ _logger.info("Adding extension %s", extension)
+ cursor.execute("CREATE EXTENSION %s" % extension)
# TODO do we really need to stop the container each time?
# that seems faster
# stop_method()
@@ -348,14 +370,12 @@
if result:
return result
- # TODO start odoo and detect if install fails
+ _logger.info("Installing modules %s in database %s", install_modules, dbname)
+ # install modules that are not to be tested
if nmspc.docker:
install_args = list(args)
- if override_installed_module:
- install_args.append("--install")
- install_args.append(override_installed_module)
- else:
- install_args.append("--install-default")
+ install_args.append("--install")
+ install_args.append(install_modules)
if languages:
install_args.append("--load-language")
install_args.append(languages)
@@ -370,12 +390,10 @@
raise NotImplementedError
# start odoo and detect if test fails
if nmspc.docker:
+ module_list_tests = ",".join(module_list_tests)
test_args = list(args)
- if override_tested_module:
- test_args.append("--test")
- test_args.append(override_tested_module)
- else:
- test_args.append("--test-default")
+ test_args.append("--install-test")
+ test_args.append(module_list_tests)
if test_log_level:
test_args.append("--log-level")
test_args.append(test_log_level)
diff --git a/odoo_scripts/docker_black.py b/odoo_scripts/docker_black.py
--- a/odoo_scripts/docker_black.py
+++ b/odoo_scripts/docker_black.py
@@ -4,6 +4,7 @@
import logging
import os
import sys
+from argparse import ArgumentParser
from .config import Config
from .docker_build import add_build_options, build_local_image, get_build_options
@@ -19,7 +20,7 @@
"""Name of destination variable used for black in parsing"""
-def __parser():
+def __parser() -> ArgumentParser:
program_version_message = f"%(prog)s {__version__} ({__updated__})"
program_short_description = __doc__.split(".", maxsplit=1)[0]
program_license = f"""{program_short_description}
diff --git a/odoo_scripts/docker_dev_start.py b/odoo_scripts/docker_dev_start.py
--- a/odoo_scripts/docker_dev_start.py
+++ b/odoo_scripts/docker_dev_start.py
@@ -7,6 +7,7 @@
import os
import pwd
import sys
+from argparse import ArgumentParser
from configparser import ConfigParser
from subprocess import call
from typing import List
@@ -39,24 +40,19 @@
_logger = logging.getLogger(__name__)
-__version__ = "3.3.1"
+__version__ = "3.4.0"
__date__ = "2017-08-11"
__updated__ = "2022-04-04"
-def main(argv=None): # IGNORE:C0111
- """Parse arguments and launch conversion"""
- program_version = __version__
- program_build_date = str(__updated__)
- program_version_message = "%%(prog)s %s (%s)" % (
- program_version,
- program_build_date,
- )
+def __parser(project_name: str) -> ArgumentParser:
+ """Return a parser for docker_dev_start"""
+ program_version_message = f"%(prog)s {__version__} ({__updated__})"
program_shortdesc = __doc__.split(".", maxsplit=1)[0]
- program_license = """%s
+ program_license = f"""{program_shortdesc}
- Created by Vincent Hatakeyama on %s.
- Copyright 2017, 2018, 2019, 2020, 2021 XCG Consulting. All rights reserved.
+ Created by Vincent Hatakeyama on {__date__}.
+ Copyright 2017, 2018, 2019, 2020, 2021, 2022 XCG Consulting. All rights reserved.
Licensed under the MIT License
@@ -64,19 +60,7 @@
or conditions of any kind, either express or implied.
USAGE
-""" % (
- program_shortdesc,
- str(__date__),
- )
-
- project_path = os.path.realpath(".")
- project_name = os.path.basename(project_path)
- if project_name == "odoo_scripts":
- logging.fatal(
- "You must run this script from the super project"
- " (./odoo_scripts/docker_dev_start.py)"
- )
- return 1
+"""
# TODO add a way to store configuration options in a project file
# Argument parsing
@@ -121,6 +105,14 @@
default=None,
)
group.add_argument(
+ "--install-test",
+ help="Modules to install for tests (will also set --log-level=test)"
+ " [default: %(default)s]\n"
+ "Options --update, --test, --install-test and --test-default"
+ " cannot be used at the same time",
+ default=None,
+ )
+ group.add_argument(
"-t",
"--test",
help="Modules to test (will also set --log-level=test)"
@@ -299,8 +291,49 @@
dest="run_chown",
)
+ marabunta_group = parser.add_mutually_exclusive_group()
+ marabunta_group.add_argument(
+ "--no-marabunta",
+ help="Do not run marabunta",
+ action="store_false",
+ default=True,
+ dest="marabunta",
+ )
+ marabunta_options_group = marabunta_group.add_argument_group()
+ marabunta_options_group.add_argument(
+ "--marabunta-db-user",
+ help="specify a db user for marabunta, this is useful if the odoo user is not "
+ "able to create databases or extensions",
+ )
+ marabunta_options_group.add_argument(
+ "--marabunta-db-password",
+ help="specify a db password for marabunta",
+ )
+ marabunta_options_group.add_argument(
+ "--marabunta-force-version",
+ help="marabunta force version argument (see marabunta documentation)",
+ )
+
+ return parser
+
+
+def main(argv=None): # IGNORE:C0111
+ """Parse arguments and launch conversion"""
+ project_path = os.path.realpath(".")
+ project_name = os.path.basename(project_path)
+
+ # basic detection to avoid errors
+ if project_name == "odoo_scripts":
+ logging.fatal(
+ "You must run this script from the super project"
+ " (./odoo_scripts/docker_dev_start.py)"
+ )
+ return 1
+
# TODO detect that user is member of docker group
+ parser = __parser(project_name)
+
# TODO add a way to add options to docker
# TODO add a way to add arg to odoo
@@ -324,6 +357,10 @@
run_chown = nmspc.run_chown
populate_model = nmspc.populate_model
populate_size = nmspc.populate_size
+ marabunta = nmspc.marabunta
+ marabunta_db_user = nmspc.marabunta_db_user
+ marabunta_db_password = nmspc.marabunta_db_password
+ marabunta_force_version = nmspc.marabunta_force_version
if restore_filename:
if not database:
@@ -370,8 +407,11 @@
if not use_host_network:
options["ports"][8069] = 8069
mounts: List[Mount] = []
+ tested_modules = None
# TODO handle other version of odoo (base image need to be changed too)
- coverage = (nmspc.test or nmspc.test_default) and odoo_type in (
+ coverage = (
+ nmspc.test or nmspc.test_default or nmspc.install_test
+ ) and odoo_type in (
ODOO_11,
ODOO_13,
ODOO_15,
@@ -382,29 +422,34 @@
else ("coverage-start" if coverage else "start")
]
if db_password:
- arg.append("--db_password")
- arg.append(db_password)
+ arg.extend(("--db_password", db_password))
if nmspc.update:
- arg.extend(("-u", nmspc.update))
- arg.append("--i18n-overwrite")
+ arg.extend(("-u", nmspc.update, "--i18n-overwrite"))
installing_or_updating = True
- if nmspc.test or nmspc.test_default:
+ if nmspc.test or nmspc.test_default or nmspc.install_test:
arg.append("--test-enable")
if nmspc.test:
+ tested_modules = nmspc.test
arg.extend(("-u", nmspc.test))
if odoo_type == ODOO_7:
arg.append("--log-level=test")
installing_or_updating = True
- if nmspc.test or nmspc.stop_after_init:
- arg.append("--stop-after-init")
+ if nmspc.install_test:
+ tested_modules = nmspc.install_test
+ arg.extend(("-i", nmspc.install_test))
+ if odoo_type == "odoo7":
+ arg.append("--log-level=test")
+ installing_or_updating = True
if nmspc.test_default:
test_modules = c.module_list_tests
- str_modules = ",".join(test_modules)
- arg.extend(("-u", str_modules))
+ tested_modules = ",".join(test_modules)
+ arg.extend(("-u", tested_modules))
if odoo_type == ODOO_7:
arg.append("--log-level=test")
arg.append("--stop-after-init")
installing_or_updating = True
+ if tested_modules or nmspc.stop_after_init:
+ arg.append("--stop-after-init")
if database:
arg.extend(("-d", database))
if nmspc.install or nmspc.install_default:
@@ -417,7 +462,7 @@
if modules_to_install:
arg.append("-i")
arg.append(",".join(modules_to_install))
- if nmspc.without_demo:
+ if nmspc.without_demo is not None:
arg.append("--without-demo")
arg.append(nmspc.without_demo)
if nmspc.max_cron_threads:
@@ -471,7 +516,7 @@
local_ip = addresses[0]
except ImportError:
_logger.warn(
- "Consider installing python netifaces" " to ease local IP detection"
+ "Consider installing python netifaces to ease local IP detection"
)
if not local_ip:
import socket
@@ -714,12 +759,13 @@
# --- restore ---
# TODO find out why odoo_help would stop us from restoring
if start_postgresql and not odoo_help and restore_filename:
+ _logger.info("Creating database %s for %s", database, user)
+ pg.exec_run(["createdb", "-U", user, database])
+ if start_postgresql and not odoo_help and restore_filename:
restore_basename = os.path.basename(restore_filename)
_logger.info("Copying dump file in docker")
inside_restore_filename = "/tmp/{}".format(restore_basename)
DockerClient.put_file(restore_filename, pg, inside_restore_filename)
- _logger.info("Creating database %s for %s", database, user)
- pg.exec_run(["createdb", "-U", user, database])
_logger.info("Restoring database %s", database)
restore, output = pg.exec_run(
[
@@ -738,10 +784,18 @@
pg.exec_run(["rm", inside_restore_filename])
if not start_postgresql and not odoo_help and restore_filename:
- _logger.info("Creating database %s", database)
- createdb = call(["createdb", "-U", user, database])
- if not createdb:
- return 17
+ with connect(
+ user=user,
+ password=password,
+ database="postgres",
+ host=db_host,
+ port=dbport,
+ ) as connection:
+ connection.autocommit = True
+ _logger.info("Creating database %s", database)
+ with connection.cursor() as cursor:
+ cursor.execute(f'CREATE DATABASE "{database}"')
+ if not start_postgresql and not odoo_help and restore_filename:
_logger.info("Restoring database %s", database)
restore = call(
["pg_restore", "-U", user, "-O", "-d", database, restore_filename]
@@ -874,14 +928,76 @@
# TODO when testing base modules, coverage will not be shown
source_files = ",".join(
[
- "{}/{}".format(target_module_directory, module)
- for module in nmspc.test.split(",")
+ f"{target_module_directory}/{module}"
+ for module in tested_modules.split(",")
]
)
options["environment"]["COVERAGE_RUN_OPTIONS"] = (
"--source=" + source_files + " --omit=*/__manifest__.py --branch"
)
+ if marabunta:
+ marabunta_migration_file = c.marabunta_migration_file
+ # if nothing is set, set the value to marabunta.yaml if there is such a file
+ if not marabunta_migration_file:
+ if os.path.exists("marabunta.yaml"):
+ marabunta_migration_file = "marabunta.yaml"
+ else:
+ marabunta_migration_file = None
+
+ if marabunta_migration_file is not None:
+ marabunta_container_path = "/etc/marabunta.yaml"
+ # add marabunta file to mounts
+ mounts.append(
+ Mount(
+ marabunta_container_path,
+ os.path.join(project_path, marabunta_migration_file),
+ "bind",
+ read_only=True,
+ )
+ )
+ # also mount songs for anthem to avoid having to rebuild the image when they
+ # are changed.
+ if os.path.exists("songs"):
+ for root, dirs, _files in os.walk("songs"):
+ for dir in dirs:
+ # with older odoo this will not be set
+ for python in pythons:
+ mounts.append(
+ Mount(
+ f"/usr/local/lib/{python}/dist-packages/{root}/{dir}",
+ os.path.join(project_path, root, dir),
+ "bind",
+ read_only=True,
+ )
+ )
+
+ options["environment"].update(
+ {
+ "MARABUNTA_MIGRATION_FILE": marabunta_container_path,
+ "MARABUNTA_MODE": "local",
+ "MARABUNTA_ALLOW_SERIE": "on",
+ }
+ )
+ if start_postgresql:
+ options["environment"]["MARABUNTA_DB_USER"] = (
+ marabunta_db_user or "postgres"
+ )
+ elif marabunta_db_user:
+ # only set if provided a value, otherwise the image will get the value from
+ # command line arguments or the configuration file (using the same user os
+ # the one used in odoo)
+ options["environment"]["MARABUNTA_DB_USER"] = marabunta_db_user
+ # marabunta is run inside the container by the start command
+ else:
+ # Only set to avoid problems with image that inherit another image using
+ # marabunta while not using marabunta themselves
+ options["environment"]["MARABUNTA_MIGRATION_FILE"] = ""
+ if marabunta_force_version:
+ options["environment"]["MARABUNTA_FORCE_VERSION"] = marabunta_force_version
+ if marabunta_db_password:
+ options["environment"]["MARABUNTA_DB_PASSWORD"] = marabunta_db_password
+
interactive = True
if interactive:
options["stdin_open"] = True
diff --git a/odoo_scripts/odoo_conf_inject_env_var.py b/odoo_scripts/odoo_conf_inject_env_var.py
--- a/odoo_scripts/odoo_conf_inject_env_var.py
+++ b/odoo_scripts/odoo_conf_inject_env_var.py
@@ -27,10 +27,18 @@
_logger = logging.getLogger(__name__)
+class TargetExistsException(Exception):
+ """Exception when the target already exists"""
+
+ pass
+
+
def inject_env_var(source_path: str, target_path: str, overwrite: bool = False):
- """Inject environment variables into an Odoo configuration file."""
+ """Inject environment variables into an Odoo configuration file.
+ :raises TargetExistsException: when the target already exists
+ """
if os.path.exists(target_path) and not overwrite:
- raise Exception(f"{target_path} already exists")
+ raise TargetExistsException(f"{target_path} already exists")
config_parser = ConfigParser()
config_parser.read(source_path)
with open(target_path, "w", encoding="UTF-8") as file:
diff --git a/tests/test_odoo_conf_inject_env_var.py b/tests/test_odoo_conf_inject_env_var.py
--- a/tests/test_odoo_conf_inject_env_var.py
+++ b/tests/test_odoo_conf_inject_env_var.py
@@ -4,7 +4,11 @@
import unittest
from configparser import ConfigParser
-from odoo_scripts.odoo_conf_inject_env_var import inject_env_var, main
+from odoo_scripts.odoo_conf_inject_env_var import (
+ TargetExistsException,
+ inject_env_var,
+ main,
+)
def clean_up_env():
@@ -88,7 +92,7 @@
new_conf = os.path.join(tmpdirname, "odoo.conf")
with open(new_conf, "w", encoding="UTF-8") as file:
file.write("\n")
- with self.assertRaises(Exception):
+ with self.assertRaises(TargetExistsException):
inject_env_var(self.conf_path, new_conf)
def test_no_options_section(self):