update python docker requirements to use the API directly rather than using subprocesses

.gitlab-ci.yml

@@ -12,7 +12,7 @@
   TAG_LATEST: branch/default
   DOCKER_IMAGE: xcgd/odoo_scripts
   HTML_DOC_SOURCES: doc/_build/html
-  SH_SCRIPTS: isort create_archive start
+  SH_SCRIPTS: create_archive start
 
 checkbashisms:
   allow_failure: true

NEWS.rst

@@ -23,6 +23,12 @@
 
 Colored logs.
 
+Docker calls are now done with the API rather than with subprocesses.
+
+:file:`isort` shell script has been removed, it is now available as a python 3 script :ref:`docker_isort`.
+
+Logging default is now the info level. Use `-q`/`--quiet` to decrease verbosity.
+
 4.1
 ---
 

README.rst

@@ -73,17 +73,6 @@
 
 zsh script to create a tar.xz file containing all sources.
 
-isort
------
-
-Run dockerized isort on current directory. This uses a configuration file that is adapted to OCA guidelines for imports.
-
-For help:
-
-.. code-block:: SH
-
-    ./isort --help
-
 Scripts
 =======
 
@@ -94,6 +83,13 @@
 The prerequisites for this module or one of its section can be installed by using pip3 or the package manager; the requirements are defined in ``setup.py``.
 As there is no way of indicating sections (it is even handling them incorrectly), do not use ``python3 setup.py install``.
 
+docker_isort
+------------
+
+Run dockerized isort on current directory. This uses a configuration file that is adapted to OCA guidelines for imports.
+
+This is part of the docker section.
+
 conf2reST.py
 ------------
 
@@ -223,7 +219,7 @@
 
 Generate completion file for python scripts (from the superproject for the scripts that need to be run from there, and with the required requirements too)::
 
-  for command_name in docker_dev_start docker_build docker_build_copy docker_build_clean do_tests conf2reST import_base_import import_jsonrpc import_sql ; do
+  for command_name in docker_dev_start docker_build docker_build_copy docker_build_clean do_tests conf2reST import_base_import import_jsonrpc import_sql docker_flake8 docker_isort ; do
     $command_name --help | ~/src/zsh-completion-generator/help2comp.py $command_name > ~/.local/share/zsh/completion/_$command_name
   done
 

isort

-#!/bin/sh
-/usr/bin/docker run --rm --volume "$PWD:/mnt" -ti -w /mnt xcgd/isort:odoo isort "$@"

odoo_scripts/conf2reST.py

 #!/usr/bin/env python3
 """Generate a reST file from configuration files.
 """
-import argparse
 import configparser
 import logging
 import os
@@ -10,7 +9,7 @@
 
 import yaml
 
-from .parsing import add_verbosity_to_parser, logging_from_verbose
+from .parsing import apply, basic_parser
 
 _logger = logging.getLogger(__name__)
 
@@ -50,8 +49,7 @@
         str(__date__),
     )
     # Argument parsing
-    parser = argparse.ArgumentParser(
-        description=program_license,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
+    parser = basic_parser(
+        description=program_license, version=program_version_message
     )
     parser.add_argument(
@@ -56,9 +54,5 @@
     )
     parser.add_argument(
-        "-V", "--version", action="version", version=program_version_message
-    )
-    add_verbosity_to_parser(parser)
-    parser.add_argument(
         "-d",
         "--directory",
         help="Source directory [default: %(default)s]",
@@ -67,7 +61,7 @@
     parser.add_argument("-o", "--output", help="Output file name")
 
     nmspc = parser.parse_args()
-    logging_from_verbose(nmspc)
+    apply(nmspc)
     conf2rst(nmspc.directory, nmspc.output)
 
 

odoo_scripts/do_tests.py

 #!/usr/bin/env python3
 # vim: set shiftwidth=4 softtabstop=4:
 """Script to run test with local modules
-
-isort:skip_file
 """
 # Version 4.2.1
@@ -6,6 +4,4 @@
 """
 # Version 4.2.1
-import argparse
-from configparser import ConfigParser
 import logging
 import os
@@ -10,3 +6,5 @@
 import logging
 import os
+import sys
+from configparser import ConfigParser
 from subprocess import call
@@ -12,3 +10,2 @@
 from subprocess import call
-import sys
 
@@ -14,4 +11,4 @@
 
-import docker  # apt python3-docker (1.9) or pip3 install docker
-from psycopg2 import connect, OperationalError  # apt python3-psycopg2
+from odoo_scripts.docker_postgresql import docker_run_postgresql
+from psycopg2 import OperationalError, connect
 
@@ -17,7 +14,2 @@
 
-from .docker_dev_start import (
-    docker_run_postgresql,
-    flake8,
-    main as docker_dev_start_main,
-)
 from .config import Config
@@ -23,9 +15,9 @@
 from .config import Config
-
-if docker.__version__ > "2.5.0":
-    from docker import APIClient as docker_api
-else:
-    from docker import Client as docker_api
+from .docker_dev_start import main as docker_dev_start_main
+from .docker_flake8 import flake8
+from .docker_isort import isort
+from .docker_postgresql import POSTGRES_PASSWORD
+from .parsing import apply, basic_parser
 
 _logger = logging.getLogger(__name__)
 
@@ -29,5 +21,5 @@
 
 _logger = logging.getLogger(__name__)
 
-__version__ = "1.0.2"
+__version__ = "2.0.0"
 __date__ = "2018-04-13"
@@ -33,5 +25,5 @@
 __date__ = "2018-04-13"
-__updated__ = "2020-09-02"
+__updated__ = "2020-09-15"
 
 
 def main(argv=None):  # IGNORE:C0111
@@ -60,6 +52,6 @@
         str(__date__),
     )
 
-    # TODO the script assume it is launched from the parent super project
+    # the script assume it is launched from the super project
     project_path = os.path.realpath(".")
     project_name = os.path.basename(project_path)
@@ -64,12 +56,5 @@
     project_path = os.path.realpath(".")
     project_name = os.path.basename(project_path)
-    # XXX is this test still valid?
-    if project_name == "odoo_scripts":
-        logging.fatal(
-            "You must run this script from the super project"
-            " (docker_build.py)"
-        )
-        return 1
 
     # TODO add a way to store configuration options in a project file
     # Argument parsing
@@ -73,19 +58,8 @@
 
     # TODO add a way to store configuration options in a project file
     # Argument parsing
-    parser = argparse.ArgumentParser(
-        description=program_license,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    parser.add_argument(
-        "-V", "--version", action="version", version=program_version_message
-    )
-    parser.add_argument(
-        "-v",
-        "--verbose",
-        dest="verbose",
-        action="count",
-        help="set verbosity level [default: %(default)s]",
+    parser = basic_parser(
+        description=program_license, version=program_version_message
     )
     docker_group = parser.add_mutually_exclusive_group()
     docker_group.add_argument(
@@ -173,13 +147,7 @@
     # - db user for creation/remove
 
     nmspc = parser.parse_args(argv)
-    verbose = nmspc.verbose
-    if not verbose:
-        logging.basicConfig(level=logging.WARN)
-    if verbose == 1:
-        logging.basicConfig(level=logging.INFO)
-    if verbose and verbose > 1:
-        logging.basicConfig(level=logging.DEBUG)
+    apply(nmspc)
     dbname = nmspc.dbname
     odoo_db_user = nmspc.db_user
     odoo_db_password = nmspc.db_password
@@ -254,6 +222,5 @@
             "-d",
             dbname,
             "--stop-after-init",
-            "--max-cron-threads",
-            "0",
+            "--max-cron-threads=0",
             "--no-flake8",
@@ -259,4 +226,5 @@
             "--no-flake8",
+            "--no-isort",
             "--no-dev",
         ]
         if odoo_db_user:
@@ -269,4 +237,5 @@
             args.append("--dbport")
             args.append(dbport)
 
+    # only run flake8 and isort once rather than once per docker_dev_start call
     flake8(odoo_type)
@@ -272,4 +241,5 @@
     flake8(odoo_type)
+    isort()
     if nmspc.log_handler:
         for lh in nmspc.log_handler:
             args.append("--log-handler")
@@ -281,9 +251,8 @@
         args.append(nmspc.LOAD_LANGUAGE)
     if recreate_db:
         if start_postgresql:
-            docker_client = docker_api(base_url="unix://var/run/docker.sock")
-            name, stop_method, socket_path = docker_run_postgresql(
-                docker_client, project_name, postgresql_version, None
+            container, stop_method, socket_path = docker_run_postgresql(
+                project_name, postgresql_version
             )
             # system user used in the pg docker image
             pg_user = "postgres"
@@ -302,5 +271,6 @@
                     user=pg_user,
                     database="postgres",
                     host=socket_path,
-                    post=5432,
+                    port=5432,
+                    password=POSTGRES_PASSWORD,
                 )
@@ -306,4 +276,5 @@
                 )
+                _logger.info("Creating role %s", odoo_db_user)
                 with connection.cursor() as cursor:
                     # not injection safe but you are on your own machine
                     # with already full access to db
@@ -333,7 +304,9 @@
                 for extension in extensions:
                     _logger.info("Adding extension %s", extension)
                     cursor.execute("CREATE EXTENSION %s" % extension)
-            stop_method()
+            # TODO do we really need to stop the container each time?
+            #  that seems faster
+            # stop_method()
         else:
             # TODO use psycopg2 (see above)
             # drop database

odoo_scripts/docker_build.py

@@ -11,8 +11,6 @@
 import sys
 from subprocess import call, check_output
 
-import docker  # apt python3-docker (1.9) or pip3 install docker
-
 from .config import Config
 from .docker_build_clean import clean
 from .docker_build_copy import copy
@@ -16,14 +14,8 @@
 from .config import Config
 from .docker_build_clean import clean
 from .docker_build_copy import copy
-from .parsing import basic_parser, logging_from_verbose
-
-# TODO change requirements to increase docker version to avoid that
-if docker.__version__ > "2.5.0":
-    from docker import APIClient as docker_api
-else:
-    from docker import Client as docker_api
-
+from .docker_client import DockerClient
+from .parsing import apply, basic_parser
 
 _logger = logging.getLogger(__name__)
 
@@ -27,5 +19,5 @@
 
 _logger = logging.getLogger(__name__)
 
-__version__ = "0.2.0"
+__version__ = "1.0.0"
 __date__ = "2018-04-04"
@@ -31,5 +23,5 @@
 __date__ = "2018-04-04"
-__updated__ = "2020-06-30"
+__updated__ = "2020-09-11"
 
 
 def __parser():
@@ -43,7 +35,7 @@
     program_license = """%s
 
       Created by Vincent Hatakeyama on %s.
-      Copyright 2018, 2020 XCG Consulting. All rights reserved.
+      Copyright 2018, 2019, 2020 XCG Consulting. All rights reserved.
 
       Licensed under the MIT License
 
@@ -66,7 +58,7 @@
     project_path = os.path.realpath(".")
     project_name = os.path.basename(project_path)
     if project_name == "odoo_scripts":
-        logging.fatal("You must run this script from the super project")
+        _logger.fatal("You must run this script from the super project")
         return 1
 
     # TODO add a way to store configuration options in a project file
@@ -101,6 +93,4 @@
     )
     # TODO (maybe) add argument for other build arg
 
-    # TODO detect that user is member of docker group
-
     nmspc = parser.parse_args(argv)
@@ -106,5 +96,5 @@
     nmspc = parser.parse_args(argv)
-    logging_from_verbose(nmspc)
+    apply(nmspc)
     ensureconf = nmspc.ensureconf
     dev = nmspc.dev
     push = nmspc.push
@@ -114,7 +104,7 @@
     project = c.image
     odoo_type = c.odoo_type
     image = "%s/%s:latest" % (registry, project)
-    logging.debug("Docker image: %s", image)
+    _logger.debug("Docker image: %s", image)
     # TODO ensureconf
     if ensureconf:
         raise NotImplementedError
@@ -127,8 +117,6 @@
         clean()
         # XXX needed?
 
-    #    sys.exit(0)
-
     signal.signal(signal.SIGINT, signal_handler)
     signal.signal(signal.SIGTERM, signal_handler)
 
@@ -132,6 +120,5 @@
     signal.signal(signal.SIGINT, signal_handler)
     signal.signal(signal.SIGTERM, signal_handler)
 
-    # TODO docker build
     buildargs = dict()
     if os.path.exists(".hg"):
@@ -136,10 +123,16 @@
     buildargs = dict()
     if os.path.exists(".hg"):
-        buildargs["REVISION"] = (
-            check_output("hg identify -i".split()).split()[0].decode("utf-8")
-        )
-    buildargs["CREATED"] = datetime.datetime.now().isoformat()
+        tags = check_output(["hg", "identify", "--tags"]).decode()
+        if tags:
+            buildargs["VERSION"] = tags
+        buildargs["VCS_URL"] = check_output(
+            ["hg", "paths", "default"]
+        ).decode()
+        buildargs["VCS_REF"] = check_output(
+            ["hg", "identify", "--id"]
+        ).decode()
+    buildargs["BUILD_DATE"] = datetime.datetime.now().isoformat()
     if nmspc.build_arg:
         for arg in nmspc.build_arg:
             a = arg.split("=")
             buildargs[a[0]] = a[1]
@@ -142,8 +135,8 @@
     if nmspc.build_arg:
         for arg in nmspc.build_arg:
             a = arg.split("=")
             buildargs[a[0]] = a[1]
-    logging.debug("Build args: %s", buildargs)
+    _logger.debug("Build args: %s", buildargs)
     dockerfile = "Dockerfile"
     if dev:
         debug_dockerfile = "Dockerfile.debug"
@@ -172,5 +165,5 @@
         dockerfile = debug_dockerfile
         # TODO remove temp image
 
-    docker_client = docker_api(base_url="unix://var/run/docker.sock")
+    docker_client = DockerClient.client
     pull = not nmspc.no_pull
@@ -176,6 +169,6 @@
     pull = not nmspc.no_pull
-    logging.debug("Docker Pull %s", pull)
-    builder = docker_client.build(
+    _logger.debug("Docker Pull %s", pull)
+    builder = docker_client.api.build(
         path=".",
         rm=True,
         pull=pull,
@@ -183,8 +176,6 @@
         tag=image,
         dockerfile=dockerfile,
     )
-    # this is for python docker 1.8-1.9
-    # TODO add compatibility with newer python docker
     for line in builder:
         d = json.loads(line.decode("utf-8"))
         if "stream" in d:
@@ -188,5 +179,6 @@
     for line in builder:
         d = json.loads(line.decode("utf-8"))
         if "stream" in d:
-            logging.info(d["stream"])
+            for line in d["stream"].rstrip().split("\n"):
+                _logger.info(line)
         if "errorDetail" in d:
@@ -192,5 +184,5 @@
         if "errorDetail" in d:
-            logging.fatal(d["errorDetail"])
+            _logger.fatal(d["errorDetail"])
             return 1
     if dev:
         call(["rm", dockerfile])
@@ -194,7 +186,6 @@
             return 1
     if dev:
         call(["rm", dockerfile])
-    # TODO exit if build failed
 
     # TODO docker tag with tags/bookmarks (unused so maybe no need)
     # TODO docker push (only when asked for)

odoo_scripts/docker_build_clean.py

@@ -8,7 +8,7 @@
 import sys
 
 from .list_modules import MODULES_LIST_FILE
-from .parsing import basic_parser, logging_from_verbose
+from .parsing import apply, basic_parser
 
 _logger = logging.getLogger(__name__)
 
@@ -49,7 +49,7 @@
     """
     parser = __parser()
     nmspc = parser.parse_args(argv)
-    logging_from_verbose(nmspc)
+    apply(nmspc)
     # TODO check odoo_scripts before deleting anything
     clean()
     return 0

odoo_scripts/docker_build_copy.py

@@ -9,7 +9,7 @@
 
 from .config import Config
 from .list_modules import list_modules
-from .parsing import basic_parser, logging_from_verbose
+from .parsing import apply, basic_parser
 
 _logger = logging.getLogger(__name__)
 
@@ -13,5 +13,5 @@
 
 _logger = logging.getLogger(__name__)
 
-__version__ = "1.0.0"
+__version__ = "1.0.1"
 __date__ = "2020-06-30"
@@ -17,5 +17,5 @@
 __date__ = "2020-06-30"
-__updated__ = "2020-06-30"
+__updated__ = "2020-09-11"
 
 
 def __parser():
@@ -50,7 +50,7 @@
     """
     parser = __parser()
     nmspc = parser.parse_args(argv)
-    logging_from_verbose(nmspc)
+    apply(nmspc)
     copy()
     return 0
 

odoo_scripts/docker_client.py

+import logging
+import os
+import tarfile
+from typing import Dict
+
+import docker
+
+_logger = logging.getLogger(__name__)
+if docker.__version__ < "3.4.0":
+    _logger.warning(
+        "Unexepected python docker version: %s", docker.__version__
+    )
+
+# TODO detect that user is member of docker group
+
+
+class DockerClient:
+    client = docker.from_env()
+
+    @classmethod
+    def create_volume(
+        cls, volume_name: str, extra_labels: Dict[str, str] = None
+    ) -> docker.models.volumes.Volume:
+        """Return the volume passed in parameter, creating it if it does not
+        exists.
+        :param volume_name: name of the volume to create
+        :param extra_labels: extra labels to put on the volume (only on
+        creation)
+        """
+        volumes = cls.client.volumes.list(filters={"name": volume_name})
+        if volumes:
+            _logger.debug("Volume %s already exists", volume_name)
+            return volumes[0]
+        else:
+            _logger.debug("Creating volume %s", volume_name)
+            labels = dict(odoo_scripts="")
+            if extra_labels:
+                labels.update(extra_labels)
+            return cls.client.volumes.create(name=volume_name, labels=labels)
+
+    @staticmethod
+    def put_file(src, container, dst):
+        # copied from https://stackoverflow.com/questions/46390309/how-to-copy-a-file-from-host-to-container-using-docker-py-docker-sdk # noqa: E501
+        os.chdir(os.path.dirname(src))
+        srcname = os.path.basename(src)
+        # TODO put that in a temporary directory
+        tar = tarfile.open(src + ".tar", mode="w")
+        try:
+            tar.add(srcname)
+        finally:
+            tar.close()
+
+        data = open(src + ".tar", "rb").read()
+        container.put_archive(os.path.dirname(dst), data)

odoo_scripts/docker_flake8.py

+#!/usr/bin/env python3
+# vim: set shiftwidth=4 softtabstop=4:
+"""Run flake8 in a docker
+"""
+import atexit
+import logging
+import os
+import sys
+
+from .config import Config
+from .docker_client import DockerClient
+from .parsing import apply, basic_parser
+
+__version__ = "1.0.0"
+__date__ = "2020-09-10"
+__updated__ = "2020-09-10"
+
+_logger = logging.getLogger(__name__)
+
+
+def __parser():
+    program_version = __version__
+    program_build_date = str(__updated__)
+    program_version_message = "%%(prog)s %s (%s)" % (
+        program_version,
+        program_build_date,
+    )
+    program_shortdesc = __doc__.split(".")[0]
+    program_license = """%s
+
+      Created by Vincent Hatakeyama on %s.
+      Copyright 2020 XCG Consulting. All rights reserved.
+
+      Licensed under the MIT License
+
+      Distributed on an "AS IS" basis without warranties
+      or conditions of any kind, either express or implied.
+
+    USAGE
+    """ % (
+        program_shortdesc,
+        str(__date__),
+    )
+    parser = basic_parser(program_license, program_version_message)
+    return parser
+
+
+def main(argv=None):
+    """Copy modules for a build, callable version that parses arguments
+    """
+    parser = __parser()
+    nmspc = parser.parse_args(argv)
+    apply(nmspc)
+    c = Config()
+    odoo_type = c.odoo_type
+    return flake8(odoo_type)
+
+
+def flake8(odoo_type):
+    """Run flake8
+    """
+    client = DockerClient.client
+    repository = "xcgd/flake8"
+    if odoo_type in ("odoo7", "odoo8", "odoo9", "odoo10"):
+        tag = "2"
+        _logger.info("Running flake8 for Python 2")
+    else:
+        tag = "3"
+        _logger.info("Running flake8 for Python 3")
+    client.images.pull(repository, tag)
+    image = "{}:{}".format(repository, tag)
+    container = None
+
+    # XXX When pressing ctrl-c during the container creating, the container
+    #  is not removed as there is no return of the value of the Container
+    #  object. There is several ways to do that:
+    #  - give the container a unique name we know, and use that to clean
+    #  - maybe use some kind of time frame to identify the container
+    #  - warn the user of a potential container leak by using signals
+
+    def stop_remove():
+        """Stop then remove the container."""
+        if container:
+            if container.status == "running":
+                container.stop()
+                container.wait()
+            container.remove()
+
+    atexit.register(stop_remove)
+
+    # split create and start so in case of SIGTERM, container has fewer chances
+    # of being None
+    container = client.containers.create(
+        image,
+        command=["."],
+        volumes={os.environ["PWD"]: {"bind": "/mnt", "mode": "ro"}},
+        working_dir="/mnt",
+        detach=True,
+    )
+    container.start()
+    result = container.wait()
+    atexit.unregister(stop_remove)
+    atexit.register(container.remove)
+    print(container.logs().decode().rstrip())
+    if result.get("Error", None):
+        _logger.warning(result.decode())
+    container.remove(v=True)
+    atexit.unregister(container.remove)
+    return result["StatusCode"]
+
+
+if __name__ == "__main__":
+    return_code = main(sys.argv[1:])
+    if return_code:
+        exit(return_code)

odoo_scripts/docker_isort.py

+#!/usr/bin/env python3
+# vim: set shiftwidth=4 softtabstop=4:
+"""Run isort in a docker
+"""
+import atexit
+import logging
+import os
+import sys
+
+import docker
+
+from .docker_client import DockerClient
+from .parsing import apply, basic_parser
+
+__version__ = "1.0.0"
+__date__ = "2020-09-10"
+__updated__ = "2020-09-10"
+
+_logger = logging.getLogger(__name__)
+
+
+def __parser():
+    program_version = __version__
+    program_build_date = str(__updated__)
+    program_version_message = "%%(prog)s %s (%s)" % (
+        program_version,
+        program_build_date,
+    )
+    program_shortdesc = __doc__.split(".")[0]
+    program_license = """%s
+
+      Created by Vincent Hatakeyama on %s.
+      Copyright 2020 XCG Consulting. All rights reserved.
+
+      Licensed under the MIT License
+
+      Distributed on an "AS IS" basis without warranties
+      or conditions of any kind, either express or implied.
+
+    USAGE
+    """ % (
+        program_shortdesc,
+        str(__date__),
+    )
+    parser = basic_parser(program_license, program_version_message)
+    return parser
+
+
+def main(argv=None):
+    """Copy modules for a build, callable version that parses arguments
+    """
+    parser = __parser()
+    nmspc = parser.parse_args(argv)
+    apply(nmspc)
+    isort()
+    return 0
+
+
+def isort(pull=True):
+    """Run isort
+    """
+    client = DockerClient.client
+    if pull:
+        _logger.info("Pulling isort")
+        # TODO find a better way to avoid error when pulling without a
+        #  connection
+        try:
+            client.images.pull(repository="xcgd/isort", tag="odoo")
+        except docker.errors.APIError:
+            _logger.warning(
+                "No connection to server, using existing version, if any"
+            )
+    _logger.info("Running isort for Odoo")
+    container = None
+
+    # XXX When pressing ctrl-c during the container creating, the container
+    #  is not removed as there is no return of the value of the Container
+    #  object. There is several ways to do that:
+    #  - give the container a unique name we know, and use that to clean
+    #  - maybe use some kind of time frame to identify the container
+    #  - warn the user of a potential container leak by using signals
+
+    def stop_remove():
+        """Stop then remove the container."""
+        if container:
+            if container.status == "running":
+                container.stop()
+                container.wait()
+            container.remove()
+
+    atexit.register(stop_remove)
+
+    # split create and start so in case of SIGTERM, container has fewer chances
+    # of being None
+    container = client.containers.create(
+        "xcgd/isort:odoo",
+        command=["isort", "--check"],
+        volumes={os.environ["PWD"]: {"bind": "/mnt", "mode": "ro"}},
+        working_dir="/mnt",
+        detach=True,
+    )
+    container.start()
+    result = container.wait()
+    atexit.unregister(stop_remove)
+    atexit.register(container.remove)
+    print(container.logs().decode().rstrip())
+    if result.get("Error", None):
+        _logger.warning(result.decode())
+    container.remove(v=True)
+    atexit.unregister(container.remove)
+    return result["StatusCode"]
+
+
+if __name__ == "__main__":
+    return_code = main(sys.argv[1:])
+    if return_code:
+        exit(return_code)

odoo_scripts/docker_postgresql.py

+# vim: set shiftwidth=4 softtabstop=4:
+import atexit
+import logging
+import tempfile
+import time
+from typing import Callable, Tuple
+
+import docker
+
+from .docker_client import DockerClient
+
+_logger = logging.getLogger(__name__)
+
+POSTGRES_PASSWORD = "this-could-be-anything"
+
+
+def docker_run_postgresql(
+    project_name: str,
+    postgresql_version: str,
+    host_pg_port=None,
+    stop_at_exit: bool = True,
+) -> Tuple[docker.models.containers.Container, Callable, str]:
+    """
+
+    :param project_name:
+    :param postgresql_version:
+    :param host_pg_port: if None, put the socket in a temp directory,
+     otherwise publish the port
+    :param stop_at_exit: True to stop the container at exit
+    :return: container, method to call to stop it, socket directory
+    """
+    docker_client = DockerClient.client
+    pg_repository = "postgres"
+    version = "{}-alpine".format(postgresql_version)
+    pg_image = "{}:{}".format(pg_repository, version)
+    name = "postgresql-{}-{}".format(postgresql_version, project_name)
+    # need to pull otherwise the containers.create might fail
+    docker_client.images.pull(repository=pg_repository, tag=version)
+    pg_data_volume_name = "postgresql_{}-{}".format(
+        postgresql_version, project_name
+    )
+    path: str = None
+    env = dict()
+    env["POSTGRES_PASSWORD"] = POSTGRES_PASSWORD
+
+    def stop_postgresql(pg_container: docker.models.containers.Container):
+        # TODO test if still exists
+        #  maybe by looking at pg.status
+        # TODO factorized with other methods that start dockers
+        try:
+            _logger.info("Stopping postgresql")
+            # Need to stop too?
+            pg_container.stop()
+            pg_container.wait()
+            _logger.info("Removing container postgresql")
+            pg_container.remove()
+        except docker.errors.NotFound:
+            _logger.info("Postgresql already stopped")
+        # crash because docker or pg changes the owner of the directory
+        # temp_directory.cleanup()
+
+    for container in docker_client.containers.list():
+        if name == container.name:
+            _logger.info("Postgresql Container already running")
+            source = None
+            for mount_dict in container.attrs["Mounts"]:
+                if mount_dict["Destination"] == "/var/run/postgresql":
+                    source = mount_dict["Source"]
+            return container, lambda: stop_postgresql(container), source
+    volumes = dict()
+
+    volumes[pg_data_volume_name] = {
+        "bind": "/var/lib/postgresql/data",
+        "mode": "rw",
+    }
+    port_bindings = dict()
+    if host_pg_port:
+        port_bindings[5432] = host_pg_port
+    else:
+        path = tempfile.TemporaryDirectory(
+            prefix="odoo_scripts_postgres_socket-"
+        ).name
+        volumes[path] = {"bind": "/var/run/postgresql", "mode": "rw"}
+    _logger.debug("Creating postgresql container")
+    pg = docker_client.containers.create(
+        pg_image,
+        volumes=volumes,
+        ports=port_bindings,
+        name=name,
+        environment=env,
+    )
+
+    _logger.debug("Starting postgresql container")
+    pg.start()
+
+    def stop_pg():
+        return stop_postgresql(pg)
+
+    if stop_at_exit:
+        atexit.register(stop_pg)
+    _logger.debug("Waiting for postgres to start")
+    # give pg the time to start up
+    time.sleep(5)
+    return pg, stop_pg, path

odoo_scripts/docker_py3o.py

+# vim: set shiftwidth=4 softtabstop=4:
+"""Py3o methods
+"""
+import atexit
+import logging
+
+from .docker_client import DockerClient
+
+_logger = logging.getLogger(__name__)
+
+
+# TODO move to docker_client directly
+def find_container(name):
+    """Return container object from its name
+    """
+    for container in DockerClient.client.containers():
+        if name == container.names:
+            return container
+
+
+def remove_and_stop(docker_client, name):
+    # TODO handle the case when the container has been created, not running
+    tostop = find_container(name)
+    docker_client.stop(tostop.get("Id"))
+    docker_client.remove_container(tostop.get("Id"))
+
+
+def start_py3o(host_fusion_port=8765, stop_at_exit=True):
+    """start a py3o container"""
+    docker_client = DockerClient.client
+    fusion_repository = "xcgd/py3o"
+    fusion_version = "1.0.0"
+    fusion_name = "py3o_fusion"
+    fusion_image = "{}:{}".format(fusion_repository, fusion_version)
+    try:
+        docker_client.pull(repository=fusion_repository, tag=fusion_version)
+    except Exception as e:
+        _logger.warning("Exception when trying to pull: %s", e)
+    if any(
+        "/{}".format(fusion_name) in container["Names"]
+        for container in docker_client.containers()
+    ):
+        _logger.debug("%s Container already running", fusion_name)
+        remove_and_stop(docker_client, fusion_name)
+    # TODO handle --host-network option
+    port_bindings = {8765: host_fusion_port}
+    host_config = docker_client.create_host_config(
+        binds=[], port_bindings=port_bindings
+    )
+    _logger.debug("Starting %s container", fusion_name)
+    fusion = docker_client.create_container(
+        image=fusion_image, host_config=host_config, name=fusion_name
+    )
+    _logger.debug("Starting %s container", fusion_name)
+    docker_client.start(container=fusion.get("Id"))
+
+    def stop_py3o():
+        # TODO test if still exists
+        _logger.info("Stopping fusion")
+        docker_client.stop(fusion.get("Id"))
+        _logger.info("Removing containers")
+        docker_client.remove_container(fusion.get("Id"))
+
+    if stop_at_exit:
+        atexit.register(stop_py3o)
+    return fusion_name, stop_py3o

odoo_scripts/import_base_import.py

@@ -16,7 +16,7 @@
     add_importing_file_parsing,
     extract_model_lang_from_parsed,
 )
-from .parsing import logging_from_verbose
+from .parsing import apply
 
 _logger = logging.getLogger(__name__)
 
@@ -159,7 +159,7 @@
     add_importing_file_parsing(parser)
 
     nmspc = parser.parse_args(argv)
-    logging_from_verbose(nmspc)
+    apply(nmspc)
 
     return import_with_base_import(
         login=nmspc.login,

odoo_scripts/import_jsonrpc.py

@@ -17,7 +17,7 @@
     extract_model_lang_from_parsed,
 )
 from .odoo import odoo_connect_parser, odoo_login
-from .parsing import logging_from_verbose
+from .parsing import apply
 
 _logger = logging.getLogger(__name__)
 
@@ -21,5 +21,5 @@
 
 _logger = logging.getLogger(__name__)
 
-__version__ = "1.0.1"
+__version__ = "1.0.2"
 __date__ = "2020-02-17"
@@ -25,5 +25,5 @@
 __date__ = "2020-02-17"
-__updated__ = "2020-02-26"
+__updated__ = "2020-09-11"
 
 
 def import_with_jsonrpc(
@@ -314,7 +314,7 @@
     )
 
     nmspc = parser.parse_args(argv)
-    logging_from_verbose(nmspc)
+    apply(nmspc)
 
     extra_context = dict()
     if nmspc.context:

odoo_scripts/import_sql.py

@@ -212,6 +212,7 @@
         delimiter=nmspc.delimiter,
     )
     conn.close()
+    return 0
 
 
 if __name__ == "__main__":

odoo_scripts/list_modules.py

@@ -9,9 +9,9 @@
 import sys
 
 from .config import Config
-from .parsing import basic_parser, logging_from_verbose
+from .parsing import apply, basic_parser
 
 MODULES_LIST_FILE = "odoo_modules_list"
 
 _logger = logging.getLogger(__name__)
 
@@ -13,7 +13,7 @@
 
 MODULES_LIST_FILE = "odoo_modules_list"
 
 _logger = logging.getLogger(__name__)
 
-__version__ = "1.0.2"
+__version__ = "1.0.3"
 __date__ = "2020-02-26"
@@ -19,5 +19,5 @@
 __date__ = "2020-02-26"
-__updated__ = "2020-06-18"
+__updated__ = "2020-09-11"
 
 
 def main(argv=None):  # IGNORE:C0111
@@ -53,7 +53,7 @@
     # Argument parsing
     parser = basic_parser(program_license, program_version_message)
     nmspc = parser.parse_args()
-    logging_from_verbose(nmspc)
+    apply(nmspc)
     list_modules()
 
 

odoo_scripts/parsing.py

@@ -8,8 +8,9 @@
 
 
 def add_verbosity_to_parser(parser: argparse.ArgumentParser):
-    parser.add_argument(
+    verbosity_group = parser.add_mutually_exclusive_group()
+    verbosity_group.add_argument(
         "-v",
         "--verbose",
         dest="verbose",
         action="count",
@@ -12,9 +13,18 @@
         "-v",
         "--verbose",
         dest="verbose",
         action="count",
-        help="set verbosity level [default: %(default)s]",
+        help="Increase verbosity level [default: %(default)s]",
+        default=0,
+    )
+    verbosity_group.add_argument(
+        "-q",
+        "--quiet",
+        dest="quiet",
+        action="count",
+        help="Decreate verbosity level [default: %(default)s]",
+        default=0,
     )
 
 
 def logging_from_verbose(namespace: argparse.Namespace):
@@ -17,9 +27,7 @@
     )
 
 
 def logging_from_verbose(namespace: argparse.Namespace):
-    verbose = namespace.verbose
-    if not verbose:
-        level = logging.WARN
-    elif verbose == 1:
+    verbosity = namespace.verbose - namespace.quiet
+    if verbosity == 0:
         level = logging.INFO
@@ -25,5 +33,11 @@
         level = logging.INFO
-    else:
+    elif verbosity == -1:
+        level = logging.WARNING
+    elif verbosity == -2:
+        level = logging.ERROR
+    elif verbosity == -3:
+        level = logging.CRITICAL
+    elif verbosity > 0:
         level = logging.DEBUG
     coloredlogs.install(
         level, fmt="%(asctime)s %(levelname)8s %(name)s %(message)s"