# HG changeset patch
# User Vincent Hatakeyama <vincent.hatakeyama@xcg-consulting.fr>
# Date 1599826184 -7200
# Fri Sep 11 14:09:44 2020 +0200
# Node ID 60350f857f141fd8fc9ae605b638012e8747c0e4
# Parent 9889f48ee32d547a4c1ff0be78fa9553c43fbf2e
✨ update python docker requirements to use the API directly rather than using subprocesses
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -12,7 +12,7 @@
TAG_LATEST: branch/default
DOCKER_IMAGE: xcgd/odoo_scripts
HTML_DOC_SOURCES: doc/_build/html
- SH_SCRIPTS: isort create_archive start
+ SH_SCRIPTS: create_archive start
checkbashisms:
allow_failure: true
diff --git a/NEWS.rst b/NEWS.rst
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -23,6 +23,12 @@
Colored logs.
+Docker calls are now done with the API rather than with subprocesses.
+
+:file:`isort` shell script has been removed, it is now available as a python 3 script :ref:`docker_isort`.
+
+Logging default is now the info level. Use `-q`/`--quiet` to decrease verbosity.
+
4.1
---
diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -73,17 +73,6 @@
zsh script to create a tar.xz file containing all sources.
-isort
------
-
-Run dockerized isort on current directory. This uses a configuration file that is adapted to OCA guidelines for imports.
-
-For help:
-
-.. code-block:: SH
-
- ./isort --help
-
Scripts
=======
@@ -94,6 +83,13 @@
The prerequisites for this module or one of its section can be installed by using pip3 or the package manager; the requirements are defined in ``setup.py``.
As there is no way of indicating sections (it is even handling them incorrectly), do not use ``python3 setup.py install``.
+docker_isort
+------------
+
+Run dockerized isort on current directory. This uses a configuration file that is adapted to OCA guidelines for imports.
+
+This is part of the docker section.
+
conf2reST.py
------------
@@ -223,7 +219,7 @@
Generate completion file for python scripts (from the superproject for the scripts that need to be run from there, and with the required requirements too)::
- for command_name in docker_dev_start docker_build docker_build_copy docker_build_clean do_tests conf2reST import_base_import import_jsonrpc import_sql ; do
+ for command_name in docker_dev_start docker_build docker_build_copy docker_build_clean do_tests conf2reST import_base_import import_jsonrpc import_sql docker_flake8 docker_isort ; do
$command_name --help | ~/src/zsh-completion-generator/help2comp.py $command_name > ~/.local/share/zsh/completion/_$command_name
done
diff --git a/isort b/isort
deleted file mode 100755
--- a/isort
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-/usr/bin/docker run --rm --volume "$PWD:/mnt" -ti -w /mnt xcgd/isort:odoo isort "$@"
diff --git a/odoo_scripts/conf2reST.py b/odoo_scripts/conf2reST.py
--- a/odoo_scripts/conf2reST.py
+++ b/odoo_scripts/conf2reST.py
@@ -1,7 +1,6 @@
#!/usr/bin/env python3
"""Generate a reST file from configuration files.
"""
-import argparse
import configparser
import logging
import os
@@ -10,7 +9,7 @@
import yaml
-from .parsing import add_verbosity_to_parser, logging_from_verbose
+from .parsing import apply, basic_parser
_logger = logging.getLogger(__name__)
@@ -50,15 +49,10 @@
str(__date__),
)
# Argument parsing
- parser = argparse.ArgumentParser(
- description=program_license,
- formatter_class=argparse.RawDescriptionHelpFormatter,
+ parser = basic_parser(
+ description=program_license, version=program_version_message
)
parser.add_argument(
- "-V", "--version", action="version", version=program_version_message
- )
- add_verbosity_to_parser(parser)
- parser.add_argument(
"-d",
"--directory",
help="Source directory [default: %(default)s]",
@@ -67,7 +61,7 @@
parser.add_argument("-o", "--output", help="Output file name")
nmspc = parser.parse_args()
- logging_from_verbose(nmspc)
+ apply(nmspc)
conf2rst(nmspc.directory, nmspc.output)
diff --git a/odoo_scripts/do_tests.py b/odoo_scripts/do_tests.py
--- a/odoo_scripts/do_tests.py
+++ b/odoo_scripts/do_tests.py
@@ -1,37 +1,29 @@
#!/usr/bin/env python3
# vim: set shiftwidth=4 softtabstop=4:
"""Script to run test with local modules
-
-isort:skip_file
"""
# Version 4.2.1
-import argparse
-from configparser import ConfigParser
import logging
import os
+import sys
+from configparser import ConfigParser
from subprocess import call
-import sys
-import docker # apt python3-docker (1.9) or pip3 install docker
-from psycopg2 import connect, OperationalError # apt python3-psycopg2
+from odoo_scripts.docker_postgresql import docker_run_postgresql
+from psycopg2 import OperationalError, connect
-from .docker_dev_start import (
- docker_run_postgresql,
- flake8,
- main as docker_dev_start_main,
-)
from .config import Config
-
-if docker.__version__ > "2.5.0":
- from docker import APIClient as docker_api
-else:
- from docker import Client as docker_api
+from .docker_dev_start import main as docker_dev_start_main
+from .docker_flake8 import flake8
+from .docker_isort import isort
+from .docker_postgresql import POSTGRES_PASSWORD
+from .parsing import apply, basic_parser
_logger = logging.getLogger(__name__)
-__version__ = "1.0.2"
+__version__ = "2.0.0"
__date__ = "2018-04-13"
-__updated__ = "2020-09-02"
+__updated__ = "2020-09-15"
def main(argv=None): # IGNORE:C0111
@@ -60,32 +52,14 @@
str(__date__),
)
- # TODO the script assume it is launched from the parent super project
+ # the script assume it is launched from the super project
project_path = os.path.realpath(".")
project_name = os.path.basename(project_path)
- # XXX is this test still valid?
- if project_name == "odoo_scripts":
- logging.fatal(
- "You must run this script from the super project"
- " (docker_build.py)"
- )
- return 1
# TODO add a way to store configuration options in a project file
# Argument parsing
- parser = argparse.ArgumentParser(
- description=program_license,
- formatter_class=argparse.RawDescriptionHelpFormatter,
- )
- parser.add_argument(
- "-V", "--version", action="version", version=program_version_message
- )
- parser.add_argument(
- "-v",
- "--verbose",
- dest="verbose",
- action="count",
- help="set verbosity level [default: %(default)s]",
+ parser = basic_parser(
+ description=program_license, version=program_version_message
)
docker_group = parser.add_mutually_exclusive_group()
docker_group.add_argument(
@@ -173,13 +147,7 @@
# - db user for creation/remove
nmspc = parser.parse_args(argv)
- verbose = nmspc.verbose
- if not verbose:
- logging.basicConfig(level=logging.WARN)
- if verbose == 1:
- logging.basicConfig(level=logging.INFO)
- if verbose and verbose > 1:
- logging.basicConfig(level=logging.DEBUG)
+ apply(nmspc)
dbname = nmspc.dbname
odoo_db_user = nmspc.db_user
odoo_db_password = nmspc.db_password
@@ -254,9 +222,9 @@
"-d",
dbname,
"--stop-after-init",
- "--max-cron-threads",
- "0",
+ "--max-cron-threads=0",
"--no-flake8",
+ "--no-isort",
"--no-dev",
]
if odoo_db_user:
@@ -269,7 +237,9 @@
args.append("--dbport")
args.append(dbport)
+ # only run flake8 and isort once rather than once per docker_dev_start call
flake8(odoo_type)
+ isort()
if nmspc.log_handler:
for lh in nmspc.log_handler:
args.append("--log-handler")
@@ -281,9 +251,8 @@
args.append(nmspc.LOAD_LANGUAGE)
if recreate_db:
if start_postgresql:
- docker_client = docker_api(base_url="unix://var/run/docker.sock")
- name, stop_method, socket_path = docker_run_postgresql(
- docker_client, project_name, postgresql_version, None
+ container, stop_method, socket_path = docker_run_postgresql(
+ project_name, postgresql_version
)
# system user used in the pg docker image
pg_user = "postgres"
@@ -302,8 +271,10 @@
user=pg_user,
database="postgres",
host=socket_path,
- post=5432,
+ port=5432,
+ password=POSTGRES_PASSWORD,
)
+ _logger.info("Creating role %s", odoo_db_user)
with connection.cursor() as cursor:
# not injection safe but you are on your own machine
# with already full access to db
@@ -333,7 +304,9 @@
for extension in extensions:
_logger.info("Adding extension %s", extension)
cursor.execute("CREATE EXTENSION %s" % extension)
- stop_method()
+ # TODO do we really need to stop the container each time?
+ # that seems faster
+ # stop_method()
else:
# TODO use psycopg2 (see above)
# drop database
diff --git a/odoo_scripts/docker_build.py b/odoo_scripts/docker_build.py
--- a/odoo_scripts/docker_build.py
+++ b/odoo_scripts/docker_build.py
@@ -11,25 +11,17 @@
import sys
from subprocess import call, check_output
-import docker # apt python3-docker (1.9) or pip3 install docker
-
from .config import Config
from .docker_build_clean import clean
from .docker_build_copy import copy
-from .parsing import basic_parser, logging_from_verbose
-
-# TODO change requirements to increase docker version to avoid that
-if docker.__version__ > "2.5.0":
- from docker import APIClient as docker_api
-else:
- from docker import Client as docker_api
-
+from .docker_client import DockerClient
+from .parsing import apply, basic_parser
_logger = logging.getLogger(__name__)
-__version__ = "0.2.0"
+__version__ = "1.0.0"
__date__ = "2018-04-04"
-__updated__ = "2020-06-30"
+__updated__ = "2020-09-11"
def __parser():
@@ -43,7 +35,7 @@
program_license = """%s
Created by Vincent Hatakeyama on %s.
- Copyright 2018, 2020 XCG Consulting. All rights reserved.
+ Copyright 2018, 2019, 2020 XCG Consulting. All rights reserved.
Licensed under the MIT License
@@ -66,7 +58,7 @@
project_path = os.path.realpath(".")
project_name = os.path.basename(project_path)
if project_name == "odoo_scripts":
- logging.fatal("You must run this script from the super project")
+ _logger.fatal("You must run this script from the super project")
return 1
# TODO add a way to store configuration options in a project file
@@ -101,10 +93,8 @@
)
# TODO (maybe) add argument for other build arg
- # TODO detect that user is member of docker group
-
nmspc = parser.parse_args(argv)
- logging_from_verbose(nmspc)
+ apply(nmspc)
ensureconf = nmspc.ensureconf
dev = nmspc.dev
push = nmspc.push
@@ -114,7 +104,7 @@
project = c.image
odoo_type = c.odoo_type
image = "%s/%s:latest" % (registry, project)
- logging.debug("Docker image: %s", image)
+ _logger.debug("Docker image: %s", image)
# TODO ensureconf
if ensureconf:
raise NotImplementedError
@@ -127,23 +117,26 @@
clean()
# XXX needed?
- # sys.exit(0)
-
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
- # TODO docker build
buildargs = dict()
if os.path.exists(".hg"):
- buildargs["REVISION"] = (
- check_output("hg identify -i".split()).split()[0].decode("utf-8")
- )
- buildargs["CREATED"] = datetime.datetime.now().isoformat()
+ tags = check_output(["hg", "identify", "--tags"]).decode()
+ if tags:
+ buildargs["VERSION"] = tags
+ buildargs["VCS_URL"] = check_output(
+ ["hg", "paths", "default"]
+ ).decode()
+ buildargs["VCS_REF"] = check_output(
+ ["hg", "identify", "--id"]
+ ).decode()
+ buildargs["BUILD_DATE"] = datetime.datetime.now().isoformat()
if nmspc.build_arg:
for arg in nmspc.build_arg:
a = arg.split("=")
buildargs[a[0]] = a[1]
- logging.debug("Build args: %s", buildargs)
+ _logger.debug("Build args: %s", buildargs)
dockerfile = "Dockerfile"
if dev:
debug_dockerfile = "Dockerfile.debug"
@@ -172,10 +165,10 @@
dockerfile = debug_dockerfile
# TODO remove temp image
- docker_client = docker_api(base_url="unix://var/run/docker.sock")
+ docker_client = DockerClient.client
pull = not nmspc.no_pull
- logging.debug("Docker Pull %s", pull)
- builder = docker_client.build(
+ _logger.debug("Docker Pull %s", pull)
+ builder = docker_client.api.build(
path=".",
rm=True,
pull=pull,
@@ -183,18 +176,16 @@
tag=image,
dockerfile=dockerfile,
)
- # this is for python docker 1.8-1.9
- # TODO add compatibility with newer python docker
for line in builder:
d = json.loads(line.decode("utf-8"))
if "stream" in d:
- logging.info(d["stream"])
+ for line in d["stream"].rstrip().split("\n"):
+ _logger.info(line)
if "errorDetail" in d:
- logging.fatal(d["errorDetail"])
+ _logger.fatal(d["errorDetail"])
return 1
if dev:
call(["rm", dockerfile])
- # TODO exit if build failed
# TODO docker tag with tags/bookmarks (unused so maybe no need)
# TODO docker push (only when asked for)
diff --git a/odoo_scripts/docker_build_clean.py b/odoo_scripts/docker_build_clean.py
--- a/odoo_scripts/docker_build_clean.py
+++ b/odoo_scripts/docker_build_clean.py
@@ -8,7 +8,7 @@
import sys
from .list_modules import MODULES_LIST_FILE
-from .parsing import basic_parser, logging_from_verbose
+from .parsing import apply, basic_parser
_logger = logging.getLogger(__name__)
@@ -49,7 +49,7 @@
"""
parser = __parser()
nmspc = parser.parse_args(argv)
- logging_from_verbose(nmspc)
+ apply(nmspc)
# TODO check odoo_scripts before deleting anything
clean()
return 0
diff --git a/odoo_scripts/docker_build_copy.py b/odoo_scripts/docker_build_copy.py
--- a/odoo_scripts/docker_build_copy.py
+++ b/odoo_scripts/docker_build_copy.py
@@ -9,13 +9,13 @@
from .config import Config
from .list_modules import list_modules
-from .parsing import basic_parser, logging_from_verbose
+from .parsing import apply, basic_parser
_logger = logging.getLogger(__name__)
-__version__ = "1.0.0"
+__version__ = "1.0.1"
__date__ = "2020-06-30"
-__updated__ = "2020-06-30"
+__updated__ = "2020-09-11"
def __parser():
@@ -50,7 +50,7 @@
"""
parser = __parser()
nmspc = parser.parse_args(argv)
- logging_from_verbose(nmspc)
+ apply(nmspc)
copy()
return 0
diff --git a/odoo_scripts/docker_client.py b/odoo_scripts/docker_client.py
new file mode 100644
--- /dev/null
+++ b/odoo_scripts/docker_client.py
@@ -0,0 +1,54 @@
+import logging
+import os
+import tarfile
+from typing import Dict
+
+import docker
+
+_logger = logging.getLogger(__name__)
+if docker.__version__ < "3.4.0":
+ _logger.warning(
+ "Unexepected python docker version: %s", docker.__version__
+ )
+
+# TODO detect that user is member of docker group
+
+
+class DockerClient:
+ client = docker.from_env()
+
+ @classmethod
+ def create_volume(
+ cls, volume_name: str, extra_labels: Dict[str, str] = None
+ ) -> docker.models.volumes.Volume:
+ """Return the volume passed in parameter, creating it if it does not
+ exists.
+ :param volume_name: name of the volume to create
+ :param extra_labels: extra labels to put on the volume (only on
+ creation)
+ """
+ volumes = cls.client.volumes.list(filters={"name": volume_name})
+ if volumes:
+ _logger.debug("Volume %s already exists", volume_name)
+ return volumes[0]
+ else:
+ _logger.debug("Creating volume %s", volume_name)
+ labels = dict(odoo_scripts="")
+ if extra_labels:
+ labels.update(extra_labels)
+ return cls.client.volumes.create(name=volume_name, labels=labels)
+
+ @staticmethod
+ def put_file(src, container, dst):
+ # copied from https://stackoverflow.com/questions/46390309/how-to-copy-a-file-from-host-to-container-using-docker-py-docker-sdk # noqa: E501
+ os.chdir(os.path.dirname(src))
+ srcname = os.path.basename(src)
+ # TODO put that in a temporary directory
+ tar = tarfile.open(src + ".tar", mode="w")
+ try:
+ tar.add(srcname)
+ finally:
+ tar.close()
+
+ data = open(src + ".tar", "rb").read()
+ container.put_archive(os.path.dirname(dst), data)
diff --git a/odoo_scripts/docker_dev_start.py b/odoo_scripts/docker_dev_start.py
--- a/odoo_scripts/docker_dev_start.py
+++ b/odoo_scripts/docker_dev_start.py
@@ -2,107 +2,36 @@
# vim: set shiftwidth=4 softtabstop=4:
"""Launch a docker image but bind the local directory inside the container
and define the module path automatically.
-
-isort:skip_file
"""
# Version 4.2.1
-import argparse
-import atexit
-from configparser import ConfigParser
import logging
import os
import pwd
+import sys
+from configparser import ConfigParser
from subprocess import call
-import sys
-import tempfile
-import time
-from typing import Callable, Dict, Optional, Tuple
+from typing import List
+
+import dockerpty
+from docker.types import Mount
+from psycopg2 import OperationalError, connect
-import docker # apt python3-docker (1.9) or pip install docker
-from psycopg2 import connect, OperationalError # apt python3-psycopg2
-
+from . import docker_build
from .config import Config
-from . import docker_build
-
-if docker.__version__ > "2.5.0":
- from docker import APIClient as docker_api
-else:
- from docker import Client as docker_api
+from .docker_client import DockerClient
+from .docker_flake8 import flake8
+from .docker_isort import isort
+from .docker_postgresql import POSTGRES_PASSWORD, docker_run_postgresql
+from .docker_py3o import start_py3o
+from .parsing import apply, basic_parser
# TODO auto create list of module
_logger = logging.getLogger(__name__)
-__version__ = "2.0.4"
+__version__ = "3.0.0"
__date__ = "2017-08-11"
-__updated__ = "2020-09-02"
-
-
-def which(program):
- """Return path of program if it exists
-from:
-https://stackoverflow.com/questions/377017/test-if-executable-exists-in-python/377028#377028
- """
-
- def is_exe(fpath):
- return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
-
- fpath, fname = os.path.split(program)
- if fpath:
- if is_exe(program):
- return program
- else:
- for path in os.environ["PATH"].split(os.pathsep):
- exe_file = os.path.join(path, program)
- if is_exe(exe_file):
- return exe_file
-
- return None
-
-
-def flake8(odoo_type):
- """Run flake8
- """
- _logger.info("Running flake8")
- if odoo_type in ("odoo7", "odoo8", "odoo9", "odoo10"):
- image = "xcgd/flake8:2"
- else:
- image = "xcgd/flake8:3"
- call(
- [
- "docker",
- "run",
- "--rm",
- "--volume",
- "{}:/mnt:ro".format(os.environ["PWD"]),
- "--workdir",
- "/mnt",
- image,
- ".",
- ]
- )
-
-
-def isort(docker_client):
- """Run isort
- """
- _logger.info("Pulling isort")
- docker_client.pull(repository="xcgd/isort", tag="odoo")
- _logger.info("Running isort")
- call(
- [
- "docker",
- "run",
- "--rm",
- "--volume",
- "{}:/mnt".format(os.environ["PWD"]),
- "-w",
- "/mnt",
- "xcgd/isort:odoo",
- "isort",
- "-c",
- ]
- )
+__updated__ = "2020-09-15"
def main(argv=None): # IGNORE:C0111
@@ -142,9 +71,8 @@
# TODO add a way to store configuration options in a project file
# Argument parsing
- parser = argparse.ArgumentParser(
- description=program_license,
- formatter_class=argparse.RawDescriptionHelpFormatter,
+ parser = basic_parser(
+ description=program_license, version=program_version_message
)
parser.add_argument(
"--name",
@@ -155,16 +83,6 @@
default=project_name,
)
parser.add_argument(
- "-V", "--version", action="version", version=program_version_message
- )
- parser.add_argument(
- "-v",
- "--verbose",
- dest="verbose",
- action="count",
- help="set verbosity level [default: %(default)s]",
- )
- parser.add_argument(
"--db_user", help="Database user [default: %(default)s]", default=None
)
parser.add_argument(
@@ -371,13 +289,7 @@
nmspc = parser.parse_args(argv)
project_name = nmspc.name
- verbose = nmspc.verbose
- if not verbose:
- logging.basicConfig(level=logging.WARN)
- elif verbose == 1:
- logging.basicConfig(level=logging.INFO)
- else:
- logging.basicConfig(level=logging.DEBUG)
+ apply(nmspc)
db_user = nmspc.db_user
db_password = nmspc.db_password
use_host_network = nmspc.host_network
@@ -427,8 +339,8 @@
_logger.debug("Docker image: %s", image)
# detect if docker image already exists
- docker_client = docker_api(base_url="unix://var/run/docker.sock")
- image_list = docker_client.images(name=image, quiet=True)
+ client = DockerClient.client
+ image_list = client.images.list(name=image)
if not image_list:
_logger.info("Image %s does not exist", image)
else:
@@ -436,20 +348,22 @@
if (not image_list and not nmspc.no_build) or nmspc.force_build:
_logger.info("Building image %s", image)
arguments = ["--dev"]
- if verbose == 1:
- arguments.append("-v")
- elif verbose and verbose > 1:
- arguments.append("-vv")
failed = docker_build.main(arguments)
if failed:
return failed
# options is only used with subprocess call
- options = ["--name", project_name, "--rm", "--tty", "--interactive"]
+ options = {
+ "name": project_name,
+ "hostname": project_name,
+ "tty": True,
+ "extra_hosts": dict(),
+ "ports": dict(),
+ "environment": dict(),
+ }
if not use_host_network:
- options.append("--publish")
- options.append("8069:8069")
- binds = []
+ options["ports"][8069] = 8069
+ mounts: List[Mount] = list()
# TODO handle other version of odoo (base image need to be changed too)
coverage = (nmspc.test or nmspc.test_default) and odoo_type in (
"odoo11",
@@ -512,8 +426,7 @@
# auto detect local ip
if use_host_network:
local_ip = "127.0.0.1"
- options.append("--network")
- options.append("host")
+ options["network_mode"] = "host"
else:
local_ip = None
try:
@@ -567,17 +480,19 @@
if odoo_type in ("odoo11", "odoo12", "odoo13")
else "/opt/odoo/etc/odoo.conf"
)
- binds.append(
- "{}:{}".format(
- os.path.join(project_path, local_conf_path), odoo_conf_file
+ mounts.append(
+ Mount(
+ odoo_conf_file,
+ os.path.join(project_path, local_conf_path),
+ "bind",
)
)
cp_local = ConfigParser()
cp_local.read(local_conf_path)
if not user:
- user = cp_local.get("options", "db_user")
+ user = cp_local.get("options", "db_user", fallback=None)
if not password:
- password = cp_local.get("options", "db_password")
+ password = cp_local.get("options", "db_password", fallback=None)
# doing this avoid having to change the value of redis_host
# used by smile_redis_session_store
redis_host = (
@@ -586,8 +501,7 @@
and cp_local.get("options", "redis_host")
) or None
if redis_host:
- options.append("--add-host")
- options.append("{}:{}".format(redis_host, local_ip))
+ options["extra_hosts"][redis_host] = local_ip
else:
_logger.info("No configuration file at: %s", local_conf_path)
# default values if nothing else
@@ -599,44 +513,43 @@
# data volume handling
base_data_volume_name = "{}_data".format(project_name)
- data_volumes = dict()
if odoo_type == "odoo7":
# this path is in odoo code, the last part is the system user
- data_volumes["_sessions"] = "/tmp/oe-sessions-{}".format("odoo")
+ session_volume_name = "{}{}".format(base_data_volume_name, "_sessions")
+ target = "/tmp/oe-sessions-{}".format("odoo")
+ client.create_volume(session_volume_name, {"mounted in": target})
+ mounts.append(Mount(target, session_volume_name))
# by default odoo store attachments in database but it can be set on
# disk. mount the volume in all cases.
# XXX data is the path set in the database (in the form file:///data)
# it should be read from there, in that case target directory needs
# to include the database name.
# our databases only use file:///data so this should be fine
- data_volumes["_filestore"] = (
+ target = (
"/usr/local/lib/python2.7/dist-packages/"
- "openerp-7.0-py2.7.egg/openerp/{}"
- ).format("data")
+ "openerp-7.0-py2.7.egg/openerp/data"
+ )
+ filestore_volume_name = "{}{}".format(
+ base_data_volume_name, "_filestore"
+ )
+ client.create_volume(filestore_volume_name, {"mounted in": target})
+ mounts.append(target, filestore_volume_name)
else:
data_volume_path = "/mnt/data"
- data_volumes[""] = data_volume_path
- arg.append("--data-dir {}".format(data_volume_path))
+ mounts.append(Mount(data_volume_path, base_data_volume_name))
+ arg.append("--data-dir")
+ arg.append(data_volume_path)
- mount_opts = list()
- for key, val in data_volumes.items():
- data_volume_name = "{}{}".format(base_data_volume_name, key)
- _logger.debug("Using data volume %s", data_volume_name)
- createVolume(docker_client, data_volume_name, {"mounted in": val})
- binds.append("{}:{}".format(data_volume_name, val))
- mount_opts.append("--mount")
- mount_opts.append(
- "source={},target=/mnt/{}".format(data_volume_name, key)
- )
# make sure the permission in the volumes are correct
# TODO replace by something cleaner if possible
- chown_cmd = (
- ["docker", "run", "--rm"]
- + mount_opts
- + ["--entrypoint", "/bin/chown", image, "odoo", "--recursive", "/mnt"]
+ _logger.info("chown")
+ client.containers.run(
+ image,
+ command=["odoo", "--recursive", "/mnt"],
+ remove=True,
+ entrypoint="/bin/chown",
+ mounts=mounts,
)
- _logger.debug("Calling %s", " ".join(chown_cmd))
- call(chown_cmd)
# avoid the duplication of unbind volumes with all addons
if odoo_type == "odoo7":
@@ -664,35 +577,33 @@
volume_name = "{}_{}".format(
project_name, extra_volume.replace("/", "_")
)
- createVolume(
- docker_client,
+ DockerClient.create_volume(
volume_name,
{
"mounted in": extra_volume,
"comment": "only used to avoid creating too many volumes",
},
)
- binds.append("{}:{}".format(volume_name, extra_volume))
+ mounts.append(Mount(extra_volume, volume_name))
if start_py3o_stack:
- start_py3o(docker_client)
- options.append("--add-host")
- options.append("py3o:{}".format(local_ip))
- options.append("--add-host")
- options.append("py3o-fusion-server:{}".format(local_ip))
+ start_py3o()
+ # provide a couple of handy dns name for py3o
+ options["extra_hosts"]["py3o"] = local_ip
+ options["extra_hosts"]["py3o-fusion-server"] = local_ip
if start_postgresql and not odoo_help:
# Use socket rather than export port (avoid having to find a free port
# number)
- name, stop_postgresql, socket_path = docker_run_postgresql(
- docker_client, project_name, postgresql_version, host_pg_port=None
+ pg, stop_postgresql, socket_path = docker_run_postgresql(
+ project_name, postgresql_version, host_pg_port=None
)
- binds.append("{}:/var/run/postgresql".format(socket_path))
+ mounts.append(Mount("/var/run/postgresql", socket_path, "bind"))
# Check that connection can be done, try to create user if asked to
# TODO: handle the case where the database is still starting up
# TODO: (and remove the sleep)
- _logger.info("Testing postgres connection")
+ _logger.info("Testing database connection on base postgres")
try:
if start_postgresql:
connect(
@@ -725,6 +636,7 @@
database="postgres",
host=socket_path,
port=5432,
+ password=POSTGRES_PASSWORD,
)
else:
loginname = pwd.getpwuid(os.getuid())[0]
@@ -751,37 +663,26 @@
if start_postgresql and not odoo_help and restore_filename:
restore_basename = os.path.basename(restore_filename)
_logger.info("Copying dump file in docker")
- call(
+ inside_restore_filename = "/tmp/{}".format(restore_basename)
+ DockerClient.put_file(restore_filename, pg, inside_restore_filename)
+ _logger.info("Creating database %s for %s", database, user)
+ pg.exec_run(["createdb", "-U", user, database])
+ _logger.info("Restoring database %s", database)
+ restore, output = pg.exec_run(
[
- "docker",
- "cp",
- restore_filename,
- "{}:/tmp/{}".format(name, restore_basename),
- ]
- )
- _logger.info("Creating database")
- call(["docker", "exec", name, "createdb", "-U", user, database])
- _logger.info("Restoring database %s", database)
- restore = call(
- [
- "docker",
- "exec",
- name,
"pg_restore",
"-U",
user,
- "-O",
+ "--no-owner",
"-d",
database,
- "/tmp/{}".format(restore_basename),
+ inside_restore_filename,
]
)
if not restore:
return 15
_logger.info("Removing dump file in docker")
- call(
- ["docker", "exec", name, "rm", "/tmp/{}".format(restore_basename)]
- )
+ pg.exec_run(["rm", inside_restore_filename])
if not start_postgresql and not odoo_help and restore_filename:
_logger.info("Creating database %s", database)
@@ -802,46 +703,32 @@
_logger.info(
"Copying SQL script %s in docker", pre_sql_script_basename
)
- call(
- [
- "docker",
- "cp",
- pre_sql_script,
- "{}:/tmp/{}".format(name, pre_sql_script_basename),
- ]
- )
+ inside_pre_sql_script = "/tmp/{}".format(pre_sql_script_basename)
+ client.put_file(pre_sql_script, pg, inside_pre_sql_script)
_logger.info(
"Running SQL script %s in %s",
pre_sql_script_basename,
database,
)
- script = call(
+ exit_code, output = pg.exec_run(
[
- "docker",
- "exec",
- name,
"psql",
"-U",
user,
"-d",
database,
"-f",
- "/tmp/{}".format(pre_sql_script_basename),
+ inside_pre_sql_script,
]
)
- if script > 0:
- _logger.fatal("Error when running script (%s)", script)
+ if exit_code > 0:
+ _logger.fatal(
+ "Error when running SQL script %s", pre_sql_script
+ )
return 16
- _logger.info("Removing SQL script %s in docker")
- call(
- [
- "docker",
- "exec",
- name,
- "rm",
- "/tmp/{}".format(pre_sql_script_basename),
- ]
- )
+ _logger.info("Removing SQL script %s in docker", pre_sql_script)
+ # remove script
+ pg.exec_run(["rm", inside_pre_sql_script])
else:
_logger.info(
"Running SQL script %s in %s", pre_sql_script, database
@@ -850,18 +737,17 @@
["psql", "-U", user, "-d", database, "-f", pre_sql_script]
)
if script > 0:
- _logger.fatal("Error when running script (%s)", script)
+ _logger.fatal("Error when running SQL script %s", script)
return 16
# volume magic
target_module_directory = "/mnt/addons"
for module in modules:
- binds.append(
- "%s:%s/%s"
- % (
+ mounts.append(
+ Mount(
+ "%s/%s" % (target_module_directory, os.path.basename(module)),
os.path.realpath(os.path.join(project_path, module)),
- target_module_directory,
- os.path.basename(module),
+ "bind",
)
)
all_addons_dir = [target_module_directory]
@@ -875,7 +761,7 @@
flake8(odoo_type)
if run_isort:
- isort(docker_client)
+ isort()
if dev:
if odoo_type in ("odoo10", "odoo11", "odoo13"):
@@ -902,29 +788,39 @@
else:
pythons = ("python3.6",)
for python in pythons:
- binds.append(
- "%s/odoo:/usr/local/lib/%s/dist-packages/odoo"
- % (odoo_sources, python)
+ mounts.append(
+ Mount(
+ "/usr/local/lib/%s/dist-packages/odoo" % python,
+ os.path.join(odoo_sources, "odoo"),
+ "bind",
+ )
)
- binds.append(
- "%s/addons:/usr/local/lib/%s/dist-packages/odoo/addons"
- % (odoo_sources, python)
+ mounts.append(
+ Mount(
+ "/usr/local/lib/%s/dist-packages/odoo/addons" % python,
+ os.path.join(odoo_sources, "addons"),
+ "bind",
+ )
)
- binds.append(
- "%s/odoo/addons/base:/usr/local/lib/%s/dist-packages/"
- "odoo/addons/base" % (odoo_sources, python)
+ mounts.append(
+ Mount(
+ "/usr/local/lib/%s/dist-packages/odoo/addons/base"
+ % python,
+ os.path.join(odoo_sources, "odoo", "addons", "base"),
+ "bind",
+ )
)
elif odoo_type in ("odoo7", "odoo8"):
- binds.append("%s/odoo:/opt/odoo/sources/odoo" % odoo_sources)
+ mounts.append(
+ Mount("/opt/odoo/sources/odoo", odoo_sources, "bind")
+ )
else:
raise Exception(
"Unexpected odoo_type when binding sources: %s" % odoo_type
)
- # use call to allow usage of pdb
- for bind in binds:
- options.append("--volume")
- options.append(bind)
+ # avoid chowning at each run, faster but can cause trouble
+ options["environment"]["ODOO_CHOWN"] = "false"
if coverage:
if nmspc.test_default:
@@ -937,174 +833,29 @@
for module in nmspc.test.split(",")
]
)
- options.extend(
- ("--env", "COVERAGE_RUN_OPTIONS=--source=" + source_files)
+ options["environment"]["COVERAGE_RUN_OPTIONS"] = (
+ "--source=" + source_files
)
- cmd = ["docker", "run"]
- cmd.extend(options)
- cmd.append(image)
- cmd.extend(arg)
- _logger.debug(" ".join(cmd))
- result = call(cmd)
- return result
-
-
-def createVolume(
- docker_client, volume_name: str, extra_labels: Dict[str, str] = None
-):
- """Return the volume passed in parameter, creating it if it does not exists
- :param docker_client: docker api object
- :param volume_name: name of the volume to create
- :param extra_labels: extra labels to put on the volume (only on creation)
- """
- volumes = docker_client.volumes(filters={"name": volume_name})
- if volumes["Volumes"]:
- _logger.debug("Volume %s already exist", volume_name)
- return volumes["Volumes"][0]
- else:
- _logger.debug("Creating volume %s", volume_name)
- labels = dict(odoo_scripts="", docker_dev_start=__version__)
- if extra_labels:
- labels.update(extra_labels)
- return docker_client.create_volume(name=volume_name, labels=labels)
-
-
-def isRunning(docker_client, container_id):
- """Return true if the container is still running
- """
- return len(
- docker_client.containers(filters={"id": container_id}, quiet=True)
- )
-
-
-def docker_run_postgresql(
- docker_client,
- project_name: str,
- postgresql_version: str,
- host_pg_port=None,
- stop_at_exit: bool = True,
-) -> Tuple[Optional[str], Callable, Optional[str]]:
- """
-
- :param docker_client:
- :param project_name:
- :param postgresql_version:
- :param host_pg_port: if None, put the socket in a temp directory,
- otherwise publish the port
- :param stop_at_exit: True to stop the container at exit
- :return: name of the container, method to call to stop it, socket directory
- if any
- """
- pg_repository = "postgres"
- version = "{}-alpine".format(postgresql_version)
- pg_image = "{}:{}".format(pg_repository, version)
- name = "postgresql-{}-{}".format(postgresql_version, project_name)
- # docker_client.pull(repository=pg_repository, tag=version)
- pg_data_volume_name = "postgresql_{}-{}".format(
- postgresql_version, project_name
- )
- path: str = None
-
- # volume = createVolume(docker_client, pg_data_volume_name)
- binds = ["{}:/var/lib/postgresql/data".format(pg_data_volume_name)]
- port_bindings = {}
- if host_pg_port:
- port_bindings[5432] = host_pg_port
- else:
- path = tempfile.TemporaryDirectory(
- prefix="odoo_scripts_postgres_socket-"
- ).name
- binds.append("{}:/var/run/postgresql".format(path))
- host_config = docker_client.create_host_config(
- binds=binds, port_bindings=port_bindings
+ interactive = True
+ if interactive:
+ options["stdin_open"] = True
+ odoo_container = client.containers.create(
+ image, command=arg, mounts=mounts, **options
)
- if any(
- "/{}".format(name) in container["Names"]
- for container in docker_client.containers()
- ):
- _logger.debug("Postgresql Container already running")
- return name, False, None
- _logger.debug("Creating postgresql container")
- pg = docker_client.create_container(
- image=pg_image, host_config=host_config, name=name
- )
-
- _logger.debug("Starting postgresql container")
- docker_client.start(pg.get("Id"))
-
- def stop_postgresql():
- # TODO test if still exists
- try:
- _logger.info("Stopping postgresql")
- docker_client.stop(pg.get("Id"))
- _logger.info("Removing container postgresql")
- docker_client.remove_container(pg.get("Id"))
- except docker.errors.NotFound:
- _logger.info("Postgresql already stopped")
- # crash because docker or pg changes the owner of the directory
- # temp_directory.cleanup()
-
- if stop_at_exit:
- atexit.register(stop_postgresql)
- _logger.debug("Waiting for postgres to start")
- # give pg the time to start up
- time.sleep(5)
- return name, stop_postgresql, path
-
-
-def find_container(docker_client, name):
- """Return container object from its name
- """
- for container in docker_client.containers():
- if "/{}".format(name) in container["Names"]:
- return container
-
-
-def remove_and_stop(docker_client, name):
- # TODO handle the case when the container has been created, not running
- tostop = find_container(docker_client, name)
- docker_client.stop(tostop.get("Id"))
- docker_client.remove_container(tostop.get("Id"))
-
-
-def start_py3o(docker_client, host_fusion_port=8765, stop_at_exit=True):
- fusion_repository = "xcgd/py3o"
- fusion_version = "1.0.0"
- fusion_name = "py3o_fusion"
- fusion_image = "{}:{}".format(fusion_repository, fusion_version)
- try:
- docker_client.pull(repository=fusion_repository, tag=fusion_version)
- except Exception as e:
- _logger.warning("Exception when trying to pull: %s", e)
- if any(
- "/{}".format(fusion_name) in container["Names"]
- for container in docker_client.containers()
- ):
- _logger.debug("%s Container already running", fusion_name)
- remove_and_stop(docker_client, fusion_name)
- # TODO handle --host-network option
- port_bindings = {8765: host_fusion_port}
- host_config = docker_client.create_host_config(
- binds=[], port_bindings=port_bindings
- )
- _logger.debug("Starting %s container", fusion_name)
- fusion = docker_client.create_container(
- image=fusion_image, host_config=host_config, name=fusion_name
- )
- _logger.debug("Starting %s container", fusion_name)
- docker_client.start(container=fusion.get("Id"))
-
- def stop_py3o():
- # TODO test if still exists
- _logger.info("Stopping fusion")
- docker_client.stop(fusion.get("Id"))
- _logger.info("Removing containers")
- docker_client.remove_container(fusion.get("Id"))
-
- if stop_at_exit:
- atexit.register(stop_py3o)
- return fusion_name, stop_py3o
+ _logger.info("Starting Odoo")
+ if interactive:
+ dockerpty.start(client.api, odoo_container.id)
+ # already stopped but this is the easiest way to get the return code
+ result_dict = odoo_container.wait()
+ result = result_dict["StatusCode"]
+ else:
+ # XXX untested
+ # that might need a way to pass any signal to the container
+ result = odoo_container.start()
+ # only remove after getting the return code
+ odoo_container.remove()
+ return result
if __name__ == "__main__":
diff --git a/odoo_scripts/docker_flake8.py b/odoo_scripts/docker_flake8.py
new file mode 100644
--- /dev/null
+++ b/odoo_scripts/docker_flake8.py
@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+# vim: set shiftwidth=4 softtabstop=4:
+"""Run flake8 in a docker
+"""
+import atexit
+import logging
+import os
+import sys
+
+from .config import Config
+from .docker_client import DockerClient
+from .parsing import apply, basic_parser
+
+__version__ = "1.0.0"
+__date__ = "2020-09-10"
+__updated__ = "2020-09-10"
+
+_logger = logging.getLogger(__name__)
+
+
+def __parser():
+ program_version = __version__
+ program_build_date = str(__updated__)
+ program_version_message = "%%(prog)s %s (%s)" % (
+ program_version,
+ program_build_date,
+ )
+ program_shortdesc = __doc__.split(".")[0]
+ program_license = """%s
+
+ Created by Vincent Hatakeyama on %s.
+ Copyright 2020 XCG Consulting. All rights reserved.
+
+ Licensed under the MIT License
+
+ Distributed on an "AS IS" basis without warranties
+ or conditions of any kind, either express or implied.
+
+ USAGE
+ """ % (
+ program_shortdesc,
+ str(__date__),
+ )
+ parser = basic_parser(program_license, program_version_message)
+ return parser
+
+
+def main(argv=None):
+ """Copy modules for a build, callable version that parses arguments
+ """
+ parser = __parser()
+ nmspc = parser.parse_args(argv)
+ apply(nmspc)
+ c = Config()
+ odoo_type = c.odoo_type
+ return flake8(odoo_type)
+
+
+def flake8(odoo_type):
+ """Run flake8
+ """
+ client = DockerClient.client
+ repository = "xcgd/flake8"
+ if odoo_type in ("odoo7", "odoo8", "odoo9", "odoo10"):
+ tag = "2"
+ _logger.info("Running flake8 for Python 2")
+ else:
+ tag = "3"
+ _logger.info("Running flake8 for Python 3")
+ client.images.pull(repository, tag)
+ image = "{}:{}".format(repository, tag)
+ container = None
+
+ # XXX When pressing ctrl-c during the container creating, the container
+ # is not removed as there is no return of the value of the Container
+ # object. There is several ways to do that:
+ # - give the container a unique name we know, and use that to clean
+ # - maybe use some kind of time frame to identify the container
+ # - warn the user of a potential container leak by using signals
+
+ def stop_remove():
+ """Stop then remove the container."""
+ if container:
+ if container.status == "running":
+ container.stop()
+ container.wait()
+ container.remove()
+
+ atexit.register(stop_remove)
+
+ # split create and start so in case of SIGTERM, container has fewer chances
+ # of being None
+ container = client.containers.create(
+ image,
+ command=["."],
+ volumes={os.environ["PWD"]: {"bind": "/mnt", "mode": "ro"}},
+ working_dir="/mnt",
+ detach=True,
+ )
+ container.start()
+ result = container.wait()
+ atexit.unregister(stop_remove)
+ atexit.register(container.remove)
+ print(container.logs().decode().rstrip())
+ if result.get("Error", None):
+ _logger.warning(result.decode())
+ container.remove(v=True)
+ atexit.unregister(container.remove)
+ return result["StatusCode"]
+
+
+if __name__ == "__main__":
+ return_code = main(sys.argv[1:])
+ if return_code:
+ exit(return_code)
diff --git a/odoo_scripts/docker_isort.py b/odoo_scripts/docker_isort.py
new file mode 100644
--- /dev/null
+++ b/odoo_scripts/docker_isort.py
@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+# vim: set shiftwidth=4 softtabstop=4:
+"""Run isort in a docker
+"""
+import atexit
+import logging
+import os
+import sys
+
+import docker
+
+from .docker_client import DockerClient
+from .parsing import apply, basic_parser
+
+__version__ = "1.0.0"
+__date__ = "2020-09-10"
+__updated__ = "2020-09-10"
+
+_logger = logging.getLogger(__name__)
+
+
+def __parser():
+ program_version = __version__
+ program_build_date = str(__updated__)
+ program_version_message = "%%(prog)s %s (%s)" % (
+ program_version,
+ program_build_date,
+ )
+ program_shortdesc = __doc__.split(".")[0]
+ program_license = """%s
+
+ Created by Vincent Hatakeyama on %s.
+ Copyright 2020 XCG Consulting. All rights reserved.
+
+ Licensed under the MIT License
+
+ Distributed on an "AS IS" basis without warranties
+ or conditions of any kind, either express or implied.
+
+ USAGE
+ """ % (
+ program_shortdesc,
+ str(__date__),
+ )
+ parser = basic_parser(program_license, program_version_message)
+ return parser
+
+
+def main(argv=None):
+ """Copy modules for a build, callable version that parses arguments
+ """
+ parser = __parser()
+ nmspc = parser.parse_args(argv)
+ apply(nmspc)
+ isort()
+ return 0
+
+
+def isort(pull=True):
+ """Run isort
+ """
+ client = DockerClient.client
+ if pull:
+ _logger.info("Pulling isort")
+ # TODO find a better way to avoid error when pulling without a
+ # connection
+ try:
+ client.images.pull(repository="xcgd/isort", tag="odoo")
+ except docker.errors.APIError:
+ _logger.warning(
+ "No connection to server, using existing version, if any"
+ )
+ _logger.info("Running isort for Odoo")
+ container = None
+
+ # XXX When pressing ctrl-c during the container creating, the container
+ # is not removed as there is no return of the value of the Container
+ # object. There is several ways to do that:
+ # - give the container a unique name we know, and use that to clean
+ # - maybe use some kind of time frame to identify the container
+ # - warn the user of a potential container leak by using signals
+
+ def stop_remove():
+ """Stop then remove the container."""
+ if container:
+ if container.status == "running":
+ container.stop()
+ container.wait()
+ container.remove()
+
+ atexit.register(stop_remove)
+
+ # split create and start so in case of SIGTERM, container has fewer chances
+ # of being None
+ container = client.containers.create(
+ "xcgd/isort:odoo",
+ command=["isort", "--check"],
+ volumes={os.environ["PWD"]: {"bind": "/mnt", "mode": "ro"}},
+ working_dir="/mnt",
+ detach=True,
+ )
+ container.start()
+ result = container.wait()
+ atexit.unregister(stop_remove)
+ atexit.register(container.remove)
+ print(container.logs().decode().rstrip())
+ if result.get("Error", None):
+ _logger.warning(result.decode())
+ container.remove(v=True)
+ atexit.unregister(container.remove)
+ return result["StatusCode"]
+
+
+if __name__ == "__main__":
+ return_code = main(sys.argv[1:])
+ if return_code:
+ exit(return_code)
diff --git a/odoo_scripts/docker_postgresql.py b/odoo_scripts/docker_postgresql.py
new file mode 100644
--- /dev/null
+++ b/odoo_scripts/docker_postgresql.py
@@ -0,0 +1,104 @@
+# vim: set shiftwidth=4 softtabstop=4:
+import atexit
+import logging
+import tempfile
+import time
+from typing import Callable, Tuple
+
+import docker
+
+from .docker_client import DockerClient
+
+_logger = logging.getLogger(__name__)
+
+POSTGRES_PASSWORD = "this-could-be-anything"
+
+
+def docker_run_postgresql(
+ project_name: str,
+ postgresql_version: str,
+ host_pg_port=None,
+ stop_at_exit: bool = True,
+) -> Tuple[docker.models.containers.Container, Callable, str]:
+ """
+
+ :param project_name:
+ :param postgresql_version:
+ :param host_pg_port: if None, put the socket in a temp directory,
+ otherwise publish the port
+ :param stop_at_exit: True to stop the container at exit
+ :return: container, method to call to stop it, socket directory
+ """
+ docker_client = DockerClient.client
+ pg_repository = "postgres"
+ version = "{}-alpine".format(postgresql_version)
+ pg_image = "{}:{}".format(pg_repository, version)
+ name = "postgresql-{}-{}".format(postgresql_version, project_name)
+ # need to pull otherwise the containers.create might fail
+ docker_client.images.pull(repository=pg_repository, tag=version)
+ pg_data_volume_name = "postgresql_{}-{}".format(
+ postgresql_version, project_name
+ )
+ path: str = None
+ env = dict()
+ env["POSTGRES_PASSWORD"] = POSTGRES_PASSWORD
+
+ def stop_postgresql(pg_container: docker.models.containers.Container):
+ # TODO test if still exists
+ # maybe by looking at pg.status
+ # TODO factorized with other methods that start dockers
+ try:
+ _logger.info("Stopping postgresql")
+ # Need to stop too?
+ pg_container.stop()
+ pg_container.wait()
+ _logger.info("Removing container postgresql")
+ pg_container.remove()
+ except docker.errors.NotFound:
+ _logger.info("Postgresql already stopped")
+ # crash because docker or pg changes the owner of the directory
+ # temp_directory.cleanup()
+
+ for container in docker_client.containers.list():
+ if name == container.name:
+ _logger.info("Postgresql Container already running")
+ source = None
+ for mount_dict in container.attrs["Mounts"]:
+ if mount_dict["Destination"] == "/var/run/postgresql":
+ source = mount_dict["Source"]
+ return container, lambda: stop_postgresql(container), source
+ volumes = dict()
+
+ volumes[pg_data_volume_name] = {
+ "bind": "/var/lib/postgresql/data",
+ "mode": "rw",
+ }
+ port_bindings = dict()
+ if host_pg_port:
+ port_bindings[5432] = host_pg_port
+ else:
+ path = tempfile.TemporaryDirectory(
+ prefix="odoo_scripts_postgres_socket-"
+ ).name
+ volumes[path] = {"bind": "/var/run/postgresql", "mode": "rw"}
+ _logger.debug("Creating postgresql container")
+ pg = docker_client.containers.create(
+ pg_image,
+ volumes=volumes,
+ ports=port_bindings,
+ name=name,
+ environment=env,
+ )
+
+ _logger.debug("Starting postgresql container")
+ pg.start()
+
+ def stop_pg():
+ return stop_postgresql(pg)
+
+ if stop_at_exit:
+ atexit.register(stop_pg)
+ _logger.debug("Waiting for postgres to start")
+ # give pg the time to start up
+ time.sleep(5)
+ return pg, stop_pg, path
diff --git a/odoo_scripts/docker_py3o.py b/odoo_scripts/docker_py3o.py
new file mode 100644
--- /dev/null
+++ b/odoo_scripts/docker_py3o.py
@@ -0,0 +1,66 @@
+# vim: set shiftwidth=4 softtabstop=4:
+"""Py3o methods
+"""
+import atexit
+import logging
+
+from .docker_client import DockerClient
+
+_logger = logging.getLogger(__name__)
+
+
+# TODO move to docker_client directly
+def find_container(name):
+ """Return container object from its name
+ """
+ for container in DockerClient.client.containers():
+ if name == container.names:
+ return container
+
+
+def remove_and_stop(docker_client, name):
+ # TODO handle the case when the container has been created, not running
+ tostop = find_container(name)
+ docker_client.stop(tostop.get("Id"))
+ docker_client.remove_container(tostop.get("Id"))
+
+
+def start_py3o(host_fusion_port=8765, stop_at_exit=True):
+ """start a py3o container"""
+ docker_client = DockerClient.client
+ fusion_repository = "xcgd/py3o"
+ fusion_version = "1.0.0"
+ fusion_name = "py3o_fusion"
+ fusion_image = "{}:{}".format(fusion_repository, fusion_version)
+ try:
+ docker_client.pull(repository=fusion_repository, tag=fusion_version)
+ except Exception as e:
+ _logger.warning("Exception when trying to pull: %s", e)
+ if any(
+ "/{}".format(fusion_name) in container["Names"]
+ for container in docker_client.containers()
+ ):
+ _logger.debug("%s Container already running", fusion_name)
+ remove_and_stop(docker_client, fusion_name)
+ # TODO handle --host-network option
+ port_bindings = {8765: host_fusion_port}
+ host_config = docker_client.create_host_config(
+ binds=[], port_bindings=port_bindings
+ )
+ _logger.debug("Starting %s container", fusion_name)
+ fusion = docker_client.create_container(
+ image=fusion_image, host_config=host_config, name=fusion_name
+ )
+ _logger.debug("Starting %s container", fusion_name)
+ docker_client.start(container=fusion.get("Id"))
+
+ def stop_py3o():
+ # TODO test if still exists
+ _logger.info("Stopping fusion")
+ docker_client.stop(fusion.get("Id"))
+ _logger.info("Removing containers")
+ docker_client.remove_container(fusion.get("Id"))
+
+ if stop_at_exit:
+ atexit.register(stop_py3o)
+ return fusion_name, stop_py3o
diff --git a/odoo_scripts/import_base_import.py b/odoo_scripts/import_base_import.py
--- a/odoo_scripts/import_base_import.py
+++ b/odoo_scripts/import_base_import.py
@@ -16,7 +16,7 @@
add_importing_file_parsing,
extract_model_lang_from_parsed,
)
-from .parsing import logging_from_verbose
+from .parsing import apply
_logger = logging.getLogger(__name__)
@@ -159,7 +159,7 @@
add_importing_file_parsing(parser)
nmspc = parser.parse_args(argv)
- logging_from_verbose(nmspc)
+ apply(nmspc)
return import_with_base_import(
login=nmspc.login,
diff --git a/odoo_scripts/import_jsonrpc.py b/odoo_scripts/import_jsonrpc.py
--- a/odoo_scripts/import_jsonrpc.py
+++ b/odoo_scripts/import_jsonrpc.py
@@ -17,13 +17,13 @@
extract_model_lang_from_parsed,
)
from .odoo import odoo_connect_parser, odoo_login
-from .parsing import logging_from_verbose
+from .parsing import apply
_logger = logging.getLogger(__name__)
-__version__ = "1.0.1"
+__version__ = "1.0.2"
__date__ = "2020-02-17"
-__updated__ = "2020-02-26"
+__updated__ = "2020-09-11"
def import_with_jsonrpc(
@@ -314,7 +314,7 @@
)
nmspc = parser.parse_args(argv)
- logging_from_verbose(nmspc)
+ apply(nmspc)
extra_context = dict()
if nmspc.context:
diff --git a/odoo_scripts/import_sql.py b/odoo_scripts/import_sql.py
--- a/odoo_scripts/import_sql.py
+++ b/odoo_scripts/import_sql.py
@@ -212,6 +212,7 @@
delimiter=nmspc.delimiter,
)
conn.close()
+ return 0
if __name__ == "__main__":
diff --git a/odoo_scripts/list_modules.py b/odoo_scripts/list_modules.py
--- a/odoo_scripts/list_modules.py
+++ b/odoo_scripts/list_modules.py
@@ -9,15 +9,15 @@
import sys
from .config import Config
-from .parsing import basic_parser, logging_from_verbose
+from .parsing import apply, basic_parser
MODULES_LIST_FILE = "odoo_modules_list"
_logger = logging.getLogger(__name__)
-__version__ = "1.0.2"
+__version__ = "1.0.3"
__date__ = "2020-02-26"
-__updated__ = "2020-06-18"
+__updated__ = "2020-09-11"
def main(argv=None): # IGNORE:C0111
@@ -53,7 +53,7 @@
# Argument parsing
parser = basic_parser(program_license, program_version_message)
nmspc = parser.parse_args()
- logging_from_verbose(nmspc)
+ apply(nmspc)
list_modules()
diff --git a/odoo_scripts/parsing.py b/odoo_scripts/parsing.py
--- a/odoo_scripts/parsing.py
+++ b/odoo_scripts/parsing.py
@@ -8,22 +8,36 @@
def add_verbosity_to_parser(parser: argparse.ArgumentParser):
- parser.add_argument(
+ verbosity_group = parser.add_mutually_exclusive_group()
+ verbosity_group.add_argument(
"-v",
"--verbose",
dest="verbose",
action="count",
- help="set verbosity level [default: %(default)s]",
+ help="Increase verbosity level [default: %(default)s]",
+ default=0,
+ )
+ verbosity_group.add_argument(
+ "-q",
+ "--quiet",
+ dest="quiet",
+ action="count",
+ help="Decreate verbosity level [default: %(default)s]",
+ default=0,
)
def logging_from_verbose(namespace: argparse.Namespace):
- verbose = namespace.verbose
- if not verbose:
- level = logging.WARN
- elif verbose == 1:
+ verbosity = namespace.verbose - namespace.quiet
+ if verbosity == 0:
level = logging.INFO
- else:
+ elif verbosity == -1:
+ level = logging.WARNING
+ elif verbosity == -2:
+ level = logging.ERROR
+ elif verbosity == -3:
+ level = logging.CRITICAL
+ elif verbosity > 0:
level = logging.DEBUG
coloredlogs.install(
level, fmt="%(asctime)s %(levelname)8s %(name)s %(message)s"
diff --git a/odoo_scripts/update_duplicate_sources.py b/odoo_scripts/update_duplicate_sources.py
--- a/odoo_scripts/update_duplicate_sources.py
+++ b/odoo_scripts/update_duplicate_sources.py
@@ -11,7 +11,7 @@
import hglib
from .config import Config
-from .parsing import basic_parser, logging_from_verbose
+from .parsing import apply, basic_parser
_logger = logging.getLogger(__name__)
@@ -52,7 +52,7 @@
"""
parser = __parser()
nmspc = parser.parse_args(argv)
- logging_from_verbose(nmspc)
+ apply(nmspc)
# XXX probably needs an option to indicate path to use, defaulting to
# default
_update_duplicate_sources()
diff --git a/odoo_scripts/which.py b/odoo_scripts/which.py
new file mode 100644
--- /dev/null
+++ b/odoo_scripts/which.py
@@ -0,0 +1,27 @@
+# vim: set shiftwidth=4 softtabstop=4:
+"""Function to help find a program path
+"""
+import os
+from typing import AnyStr, Optional
+
+
+def which(program: AnyStr) -> Optional[AnyStr]:
+ """Return path of program if it exists
+from:
+https://stackoverflow.com/questions/377017/test-if-executable-exists-in-python/377028#377028
+ """
+
+ def is_exe(fpath):
+ return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+ fpath, fname = os.path.split(program)
+ if fpath:
+ if is_exe(program):
+ return program
+ else:
+ for path in os.environ["PATH"].split(os.pathsep):
+ exe_file = os.path.join(path, program)
+ if is_exe(exe_file):
+ return exe_file
+
+ return None
diff --git a/requirements b/requirements
--- a/requirements
+++ b/requirements
@@ -5,6 +5,7 @@
PyYAML
coloredlogs
docker
+dockerpty
psycopg2
mercurial >=5.2
python-hglib
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -21,7 +21,7 @@
extras_require={
# used to require PyYAML but it is used in import scripts too
"conf2reST": [],
- "docker": ["docker"],
+ "docker": ["docker >=3.4", "dockerpty"],
"import_sql": ["psycopg2"],
"source_control": [
# Only mercurial 5.2 support Python 3.5+
@@ -39,6 +39,8 @@
"docker_build=odoo_scripts.docker_build:main [docker]",
"docker_build_clean=odoo_scripts.docker_build_clean:main",
"docker_build_copy=odoo_scripts.docker_build_copy:main",
+ "docker_flake8=odoo_scripts.docker_flake8:main [docker]",
+ "docker_isort=odoo_scripts.docker_isort:main [docker]",
"conf2reST=odoo_scripts.conf2reST:main [conf2reST]",
"list_modules=odoo_scripts.list_modules:main",
"update_duplicate_sources="