# HG changeset patch
# User Vincent Hatakeyama <vincent.hatakeyama@xcg-consulting.fr>
# Date 1698762595 -3600
# Tue Oct 31 15:29:55 2023 +0100
# Node ID 73e4d97e8d2312501020b8b5cc4e84c0fef4c852
# Parent 675704728b79365b8a9a53ff35af5d91f35facab
✨ Replace docker-py by python_on_whales
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -53,7 +53,7 @@
image: $TEMP_IMAGE
script:
- python3 -m pip install mypy types-PyYAML types-psycopg2 types-python-dateutil
- types-requests build twine
+ types-requests build twine ".[docker]"
- mypy odoo_scripts tests
rules:
- if: $CI_COMMIT_TAG == null
diff --git a/NEWS.rst b/NEWS.rst
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -2,6 +2,13 @@
History
=======
+21.0.0
+------
+
+Use python_on_whales rather than Docker-py to use buildkit when building images.
+
+https://github.com/docker/docker-py/issues/2230
+
20.14.0
-------
diff --git a/odoo_scripts/docker_black.py b/odoo_scripts/docker_black.py
--- a/odoo_scripts/docker_black.py
+++ b/odoo_scripts/docker_black.py
@@ -9,12 +9,12 @@
from .config import ODOO_11, Config
from .docker_build import add_build_options, build_local_image, get_build_options
-from .docker_client import DockerClient
+from .docker_client import DockerClient, get_volumes
from .parsing import apply, basic_parser
-__version__ = "2.1.0"
+__version__ = "2.2.0"
__date__ = "2022-03-03"
-__updated__ = "2023-03-31"
+__updated__ = "2023-10-31"
_logger = logging.getLogger(__name__)
_BLACK_DEST = "black"
@@ -68,21 +68,7 @@
build_local_image(build_options)
pwd = os.environ["PWD"]
- path = os.path.abspath(os.path.join(pwd, directory))
- volumes = {
- pwd: {
- "bind": pwd,
- "mode": "rw" if write else "ro",
- }
- }
- # also bind any symbolic link, but only for the given directory or its parent.
- # all symbolic link could be scanned for but that might take too much time
- for potential_link in (path, os.path.dirname(path)):
- if os.path.islink(potential_link):
- volumes[potential_link] = {
- "bind": os.path.realpath(potential_link),
- "mode": "rw" if write else "ro",
- }
+ volumes, path = get_volumes(pwd, directory, write)
command = ["black", "."]
# The line was in a configuration file but not in modules in that Odoo 11 modules
@@ -105,8 +91,7 @@
"user": os.getuid(),
"command": ["-c", ";".join(command)],
"volumes": volumes,
- "working_dir": path,
- "tty": True,
+ "workdir": path,
},
False,
)
diff --git a/odoo_scripts/docker_build.py b/odoo_scripts/docker_build.py
--- a/odoo_scripts/docker_build.py
+++ b/odoo_scripts/docker_build.py
@@ -3,7 +3,6 @@
"""
import argparse
import datetime
-import json
import logging
import os
import signal
@@ -11,19 +10,21 @@
from subprocess import call, check_output
from typing import List, Optional
+from python_on_whales import docker
+
from .config import ODOO_7, ODOO_8, ODOO_10, ODOO_11, ODOO_13, ODOO_14, ODOO_15, Config
from .docker_build_clean import clean
from .docker_build_copy import Configuration as CopyConfiguration
from .docker_build_copy import add_build_copy_options, copy, get_build_copy_options
-from .docker_client import DockerClient
+from .docker_client import capture_docker_exit_code
from .parsing import apply, basic_parser
from .storage import get_orus_api_token
_logger = logging.getLogger(__name__)
-__version__ = "1.5.0"
+__version__ = "1.6.0"
__date__ = "2018-04-04"
-__updated__ = "2023-10-17"
+__updated__ = "2023-10-31"
def add_build_options(parser: argparse.ArgumentParser):
@@ -237,46 +238,26 @@
dockerfile = debug_dockerfile
# TODO remove temp image
- docker_client = DockerClient.client
pull = not nmspc.no_pull
_logger.debug("Docker Pull %s", pull)
- builder = docker_client.api.build(
- path=".",
- rm=True,
- pull=pull,
- buildargs=buildargs,
- tag=repository,
- dockerfile=dockerfile,
- )
- for line in builder:
- d = json.loads(line.decode("utf-8"))
- if "stream" in d:
- for line in d["stream"].rstrip().split("\n"):
- _logger.info(line)
- if "errorDetail" in d:
- _logger.fatal(d["errorDetail"])
- return 1
+ full_tags = [repository] if nmspc.local else [f"{repository}:{tag}" for tag in tags]
+ with capture_docker_exit_code() as exit_code:
+ docker.image.build(
+ context_path=".",
+ pull=pull,
+ build_args=buildargs,
+ tags=full_tags,
+ file=dockerfile,
+ )
if dev:
call(["rm", dockerfile])
- # retag
- for tag in tags:
- if tag != "tip":
- _logger.info("Docker with tag %s", tag)
- docker_client.images.get(repository).tag(repository, tag=tag)
- if push:
+ if push and not exit_code:
_logger.info("Docker push %s", repository)
- for line in docker_client.images.push(repository, stream=True, decode=True):
- _logger.debug(line)
- for tag in tags:
- _logger.info("Docker push %s:%s", repository, tag)
- for line in docker_client.images.push(
- repository, tag=tag, stream=True, decode=True
- ):
- _logger.debug(line)
+ docker.push(full_tags)
# XXX call cleanup more intelligently
signal_handler(0, None)
- return 0
+ return int(exit_code)
def build_local_image(build_options: List[str], force: bool = False) -> int:
@@ -287,7 +268,7 @@
c = Config()
image = c.local_image_name
- image_list = DockerClient.client.images.list(name=image)
+ image_list = docker.image.list(image)
if not image_list:
_logger.info("Image %s does not exist", image)
else:
diff --git a/odoo_scripts/docker_build_copy.py b/odoo_scripts/docker_build_copy.py
--- a/odoo_scripts/docker_build_copy.py
+++ b/odoo_scripts/docker_build_copy.py
@@ -441,8 +441,7 @@
# TODO changer pour prendre la version du python du super
# projet. Probablement pas renseigné où que ce soit.
return [(package_name, tag.decode("UTF-8"))]
- else:
- _logger.debug("No tag found")
+ _logger.debug("No tag found")
else:
return [
(
diff --git a/odoo_scripts/docker_build_doc.py b/odoo_scripts/docker_build_doc.py
--- a/odoo_scripts/docker_build_doc.py
+++ b/odoo_scripts/docker_build_doc.py
@@ -88,7 +88,7 @@
"user": os.getuid(),
"command": ["html"],
"volumes": volumes,
- "working_dir": path,
+ "workdir": path,
},
False,
)
diff --git a/odoo_scripts/docker_client.py b/odoo_scripts/docker_client.py
--- a/odoo_scripts/docker_client.py
+++ b/odoo_scripts/docker_client.py
@@ -1,17 +1,16 @@
"""Class that encapsulate the docker client and provide common methods"""
import atexit
+import functools
import logging
import os
import re
-import tarfile
+import sys
from functools import partial
from multiprocessing import Process
-from typing import Any, Dict, List, Mapping, Optional, Pattern, Tuple
+from typing import Any, Callable, Dict, List, Mapping, Optional, Pattern, Tuple, Union
-import docker # type: ignore[import]
-import dockerpty # type: ignore[import]
-from docker.models.containers import Container # type: ignore[import]
-from docker.types import Mount # type: ignore[import]
+from python_on_whales import Container, Volume, docker
+from python_on_whales.exceptions import DockerException
from .config import ODOO_7, ODOO_8, ODOO_9, ODOO_10, Configuration
@@ -19,16 +18,27 @@
# TODO detect that user is member of docker group
+Mount = List[str]
+
+
+def mount(
+ target: str, source: str, mount_type: Optional[str] = None, readonly: bool = False
+) -> Mount:
+ result = [f"target={target}", f"source={source}"]
+ if mount_type is not None:
+ result.append(f"type={mount_type}")
+ if readonly:
+ result.append("readonly")
+ return result
+
class DockerClient:
"""Provide additional abstraction and extra methods of the docker client API"""
- client = docker.from_env()
-
@classmethod
def create_volume(
cls, volume_name: str, extra_labels: Optional[Dict[str, str]] = None
- ) -> Tuple[docker.models.volumes.Volume, bool]:
+ ) -> Tuple[Volume, bool]:
"""Return the volume passed in parameter, creating it if it does not exist.
:param volume_name: name of the volume to create
:param extra_labels: extra labels to put on the volume (only on
@@ -36,7 +46,7 @@
:return: a tuple, the first one is the volume, the second is a boolean
indicating that the volume has just been created (true if created)
"""
- volumes = cls.client.volumes.list(filters={"name": volume_name})
+ volumes = docker.volume.list(filters={"name": volume_name})
if volumes:
_logger.debug("Volume %s already exists", volume_name)
return volumes[0], False
@@ -44,28 +54,12 @@
labels = dict(odoo_scripts="")
if extra_labels:
labels.update(extra_labels)
- return cls.client.volumes.create(name=volume_name, labels=labels), True
-
- @staticmethod
- def put_file(src, container, dst):
- """put a file into a container"""
- # copied from https://stackoverflow.com/questions/46390309/how-to-copy-a-file-from-host-to-container-using-docker-py-docker-sdk # noqa: E501 # pylint: disable=line-too-long
- os.chdir(os.path.dirname(src))
- srcname = os.path.basename(src)
- # TODO put that in a temporary directory
- tar = tarfile.open(src + ".tar", mode="w")
- try:
- tar.add(srcname)
- finally:
- tar.close()
-
- data = open(src + ".tar", "rb").read()
- container.put_archive(os.path.dirname(dst), data)
+ return docker.volume.create(volume_name, labels=labels), True
@classmethod
def exec(
cls,
- container: docker.models.containers.Container,
+ container: Container,
command: List[str],
user: str,
interactive: bool = False,
@@ -77,24 +71,48 @@
:param user: User to execute command as.
:param interactive: True if the container needs to be interactive
"""
- if interactive:
- dockerpty.exec_command(cls.client.api, container.id, command)
- # it would be nice to return the exec command return code, but
- # there is no way to do that apparently
- result = 0
- else:
- run_result = container.exec_run(command, user=user)
- result = run_result.exit_code
- print(run_result.output.decode().rstrip())
- return result
+ with capture_docker_exit_code() as exit_code:
+ container.execute(command, user=user, interactive=interactive, tty=True)
+ return int(exit_code)
@staticmethod
- def _replace_then_print(replaces: List[Tuple[Pattern[str], str]], log_line: str):
+ def _replace_then_print(
+ replaces: List[Tuple[Pattern[str], str]], log_line: str, file=sys.stdout
+ ):
# apply all the replacement of the path inside the image to the real path
for pattern, value in replaces:
log_line = pattern.sub(value, log_line)
# print to play nicely with pycharm that will linkify the result
- print(log_line)
+ print(log_line, file=file)
+
+ @classmethod
+ def launch(
+ cls,
+ repository: str,
+ tag: str,
+ run_kwargs: Dict[str, Any],
+ pull: bool = True,
+ display_log: bool = True,
+ ) -> Container:
+ container, stop_remove_registered = cls._create(
+ repository, tag, run_kwargs, pull
+ )
+ _logger.debug("Starting container %s", container.name)
+ container.start()
+ if display_log:
+ process = Process(target=DockerClient.print_log, args=(container,))
+ process.start()
+
+ def stop_remove_wait_process():
+ stop_remove(container)
+ if process and process.is_alive():
+ _logger.info("Waiting for log process to finish")
+ process.join()
+
+ atexit.register(stop_remove_wait_process)
+ atexit.unregister(stop_remove_registered)
+
+ return container
@classmethod
def run(
@@ -104,14 +122,41 @@
run_kwargs: Dict[str, Any],
pull: bool = True,
target_source_dict: Optional[Mapping[str, str]] = None,
- display_log: bool = True,
- ):
- """Run a container with the provided args"""
+ ) -> int:
+ container, stop_remove_registered = cls._create(
+ repository, tag, run_kwargs, pull
+ )
+ _logger.debug("Starting container %s", container.name)
+ container.start()
+
+ atexit.unregister(stop_remove_registered)
+ remove_registered = functools.partial(remove, container)
+ atexit.register(remove_registered)
+
+ # TODO semble trop tard
replaces: List[Tuple[Pattern[str], str]] = (
[(re.compile(key), value) for key, value in target_source_dict.items()]
if target_source_dict
else []
)
+ cls.print_log(container, replaces)
+
+ _logger.debug("Waiting for container %s", container.name)
+ exit_code = docker.wait(container)
+ _logger.debug("Removing container %s", container.name)
+ container.remove(volumes=True)
+ atexit.unregister(remove_registered)
+ return exit_code
+
+ @classmethod
+ def _create(
+ cls,
+ repository: str,
+ tag: str,
+ run_kwargs: Dict[str, Any],
+ pull: bool = True,
+ ) -> Tuple[Container, Callable]:
+ """Run a container with the provided args"""
image = f"{repository}:{tag}"
@@ -119,13 +164,13 @@
_logger.debug("Pulling %s", image)
# TODO find a better way to avoid error when pulling without a connection
try:
- cls.client.images.pull(repository, tag)
- except docker.errors.APIError:
+ docker.image.pull(image)
+ except DockerException:
_logger.warning(
- "No connection to server, using existing version, if any"
+ "Issue when pulling, using existing version, if any", exc_info=True
)
- container = None
+ container: Union[None, Container] = None
# XXX When pressing ctrl-c during the container creating, the container
# is not removed as there is no return of the value of the Container
@@ -134,77 +179,15 @@
# - maybe use some kind of time frame to identify the container
# - warn the user of a potential container leak by using signals
- def remove():
- """Remove the container"""
- if container:
- _logger.debug("Removing container %s", container.name)
- container.remove()
-
- def stop_remove():
- """Stop then remove the container."""
- if container:
- _logger.debug("Stopping container %s", container.name)
- container.stop()
- _logger.debug("Waiting for container %s", container.name)
- container.wait()
- remove()
-
- atexit.register(stop_remove)
+ stop_remove_registered = functools.partial(stop_remove, container)
+ atexit.register(stop_remove_registered)
# split create and start so in case of SIGTERM, container has fewer chances
# of being None
- kwargs = dict(run_kwargs, detach=True)
_logger.debug("Create container with image %s", image)
- container = cls.client.containers.create(image, **kwargs)
+ container = docker.container.create(image, **run_kwargs)
_logger.debug("Created container %s", container.name)
- _logger.debug("Starting container %s", container.name)
- container.start()
- if run_kwargs.get("detach", False):
- if display_log:
- process = Process(target=DockerClient.print_log, args=(container,))
- process.start()
-
- def stop_remove_wait_process():
- stop_remove()
- if process and process.is_alive():
- _logger.info("Waiting for log process to finish")
- process.join()
-
- atexit.register(stop_remove_wait_process)
- atexit.unregister(stop_remove)
-
- return container
-
- atexit.unregister(stop_remove)
- atexit.register(remove)
-
- if kwargs.get("tty", False):
- line = b""
- for log in container.logs(stream=True, follow=True):
- # if there is a tty, send byte by byte, so add them together before
- # printing
- # works correctly because prettier sends \r\n on each line
- if log == b"\r":
- cls._replace_then_print(replaces, line.decode())
- line = b""
- elif log == b"\n":
- pass
- else:
- line += log
- if line:
- cls._replace_then_print(replaces, line.decode())
- else:
- cls.print_log(container, replaces)
-
- _logger.debug("Waiting for container %s", container.name)
- result = container.wait()
-
- if result.get("Error", None):
- _logger.warning(result.decode())
- _logger.debug("Removing container %s", container.name)
- container.remove(v=True)
- atexit.unregister(remove)
- return result["StatusCode"]
+ return container, stop_remove_registered
@classmethod
def print_log(
@@ -215,22 +198,25 @@
"""Print the logs of a container, eventually replacing some of its output."""
if replaces is None:
replaces = []
- for log in container.logs(stream=True, follow=True):
- # \r is needed when using a process that also outputs the logs
- log = log.decode().rstrip() + "\r"
- cls._replace_then_print(replaces, log)
+ logs = docker.container.logs(container, follow=True, stream=True)
+ if logs is not None and not isinstance(logs, str):
+ for channel, log in logs:
+ # \r is needed when using a process that also outputs the logs
+ log_str = log.decode().rstrip() + "\r"
+ cls._replace_then_print(replaces, log_str, getattr(sys, channel))
def get_odoo_base_path(config: Configuration) -> str:
"""Return value of env ODOO_BASE_PATH in the local image name."""
- image = DockerClient.client.images.list(config.local_image_name)
+ image = docker.image.list(config.local_image_name)
if not image:
raise Exception(
"Image %s is not built, needed to be inspected", config.local_image_name
)
- for env in image[0].attrs["Config"]["Env"]:
- if env.startswith("ODOO_BASE_PATH="):
- return env[15:]
+ if isinstance(image[0].config.env, list):
+ for env in image[0].config.env:
+ if env.startswith("ODOO_BASE_PATH="):
+ return env[15:]
raise Exception("Missing ODOO_BASE_PATH in the image")
@@ -251,7 +237,7 @@
for module in modules:
full_target_path = os.path.join(target_path, os.path.basename(module))
full_project_path = os.path.realpath(os.path.join(project_path, module))
- mount_list.append(Mount(full_target_path, full_project_path, "bind"))
+ mount_list.append(mount(full_target_path, full_project_path, "bind"))
mount_dict[full_target_path] = full_project_path
_logger.debug("python packages: %s", ",".join(config.python_packages))
@@ -261,7 +247,7 @@
full_project_path = os.path.realpath(
os.path.join(project_path, python_package[0], dir_[0])
)
- mount_list.append(Mount(full_target_path, full_project_path, "bind"))
+ mount_list.append(mount(full_target_path, full_project_path, "bind"))
mount_dict[full_target_path] = full_project_path
return mount_list, mount_dict
@@ -281,12 +267,12 @@
for root, dirs, _files in os.walk("songs"):
for directory in dirs:
mounts.append(
- Mount(
+ mount(
os.path.dirname(get_odoo_base_path(config))
+ f"/{root}/{directory}",
os.path.join(project_path, root, directory),
"bind",
- read_only=True,
+ True,
)
)
_logger.debug("mounts len: %s", len(mounts))
@@ -298,7 +284,7 @@
mounts: List[Mount] = []
full_source_path = os.path.realpath(source_path)
if config.odoo_type in (ODOO_7, ODOO_8):
- mounts.append(Mount("/opt/odoo/sources/odoo", source_path, "bind"))
+ mounts.append(mount("/opt/odoo/sources/odoo", source_path, "bind"))
else:
install_path = (
"/usr/local/lib/python2.7/dist-packages/odoo"
@@ -308,7 +294,7 @@
for entry in os.listdir(os.path.join(full_source_path, "odoo")):
if entry != "addons":
mounts.append(
- Mount(
+ mount(
install_path + "/" + entry,
os.path.join(full_source_path, "odoo", entry),
"bind",
@@ -317,7 +303,7 @@
for entry in os.listdir(os.path.join(full_source_path, "odoo", "addons")):
# for directory in dirs:
mounts.append(
- Mount(
+ mount(
install_path + "/addons/" + entry,
os.path.join(full_source_path, "odoo", "addons", entry),
"bind",
@@ -326,7 +312,7 @@
# iterate on all the modules, avoids
for entry in os.listdir(os.path.join(full_source_path, "addons")):
mounts.append(
- Mount(
+ mount(
install_path + "/addons/" + entry,
os.path.join(full_source_path, "addons", entry),
"bind",
@@ -340,7 +326,6 @@
def __init__(
self,
- docker_client: docker.client.DockerClient,
name: str,
repository: str,
tag: str,
@@ -352,7 +337,6 @@
force_pull: bool = False,
):
"""
- :param docker_client: instance of docker client
:param name: name for the container
:param repository: repository to use for the image
:param tag: repository tag to use for the image
@@ -360,7 +344,6 @@
:param stream: if not attached, indicate to stream output or not
"""
super().__init__()
- self.docker_cli = docker_client
self.name = name
self.repository: str = repository
self.tag = tag
@@ -371,11 +354,9 @@
self.stream = stream
self.stop_at_exit = stop_at_exit
# Retrieve the container if it already exists
- self._container: Optional[docker.models.containers.Container]
- try:
- self._container = self.docker_cli.containers.get(self.name)
- except docker.errors.NotFound:
- self._container = None
+ self._container: Optional[Container] = None
+ if docker.container.exists(self.name):
+ self._container = docker.container.list(filters={"name": self.name})[0]
self._log_process: Optional[Process] = None
self.stop_remove_if_exists = stop_remove_if_exists
self.force_pull: bool = force_pull
@@ -398,7 +379,7 @@
self._start()
if self.stop_at_exit:
atexit.register(partial(self.stop_and_remove, False))
- if self.attach:
+ if self.attach and self._container is not None:
DockerClient.print_log(self._container)
elif self.stream:
process = Process(target=DockerClient.print_log, args=(self._container,))
@@ -415,7 +396,7 @@
)
def _start(self) -> None:
- self._container = self.docker_cli.containers.create(**self._create_param())
+ self._container = docker.container.create(**self._create_param())
_logger.info("Starting container %s", self.name)
self._container.start()
@@ -424,7 +405,7 @@
try:
self.stop(expect_running) # call stop func
self.remove() # call remove func
- except docker.errors.NotFound as exception:
+ except DockerException as exception:
_logger.info("Stop and remove container, got: %s", exception)
raise exception
@@ -437,8 +418,8 @@
elif self._container is not None:
_logger.info("Stopping %s container", self.name)
self._container.stop()
- self._container.wait()
- except docker.errors.NotFound as exception:
+ docker.container.wait(self._container)
+ except DockerException as exception:
_logger.error("Stopping container %s, got: %s", self.name, exception)
raise
self.join_log_process()
@@ -466,9 +447,9 @@
def pull_image(self):
"""Pulls image if needed or forced."""
- if self.force_pull or not self.docker_cli.images.list(self.base_image):
+ if self.force_pull or not docker.image.list(self.base_image):
_logger.info("Pulling %s", self.base_image)
- self.docker_cli.images.pull(repository=self.repository, tag=self.tag)
+ docker.image.pull(repository=self.repository, tag=self.tag)
@property
def is_running(self) -> bool:
@@ -477,8 +458,7 @@
"""
if self._container is None:
return False
- result = self.docker_cli.containers.list(filters={"id": self._container.id})
- return result and result[0].status == "running"
+ return bool(self._container.state.running)
@property
def is_exited(self) -> bool:
@@ -488,25 +468,86 @@
"""
if self._container is None:
return True
- result = self.docker_cli.containers.list(filters={"id": self._container.id})
- return result and result[0].status == "exited"
+ return self._container.state.status in [None, "exited"]
def get_volumes(base_dir, directory: str, write: bool = False):
"""return volumes to bind"""
path = os.path.abspath(os.path.join(base_dir, directory))
- volumes = {
- base_dir: {
- "bind": base_dir,
- "mode": "rw" if write else "ro",
- }
- }
+ volumes = [
+ (
+ base_dir,
+ base_dir,
+ "rw" if write else "ro",
+ )
+ ]
# also bind any symbolic link, but only for the given directory or its parent.
# all symbolic link could be scanned for but that might take too much time
for potential_link in (path, os.path.dirname(path)):
if os.path.islink(potential_link):
- volumes[potential_link] = {
- "bind": os.path.realpath(potential_link),
- "mode": "rw" if write else "ro",
- }
+ volumes.append(
+ (
+ potential_link,
+ os.path.realpath(potential_link),
+ "rw" if write else "ro",
+ )
+ )
return volumes, path
+
+
+def remove(container: Union[None, Container]):
+ """Remove the container"""
+ if container:
+ _logger.debug("Removing container %s", container.name)
+ container.remove()
+
+
+def stop_remove(container: Union[None, Container]):
+ """Stop then remove the container."""
+ if container:
+ _logger.debug("Stopping container %s", container.name)
+ container.stop()
+ _logger.debug("Waiting for container %s", container.name)
+ docker.wait(container)
+ remove(container)
+
+
+class _MutableInt:
+ """Class with an int.
+ Used in context manager to provide exit status."""
+
+ def __init__(self):
+ self.value = 0
+
+ def set(self, value: int):
+ """Update the value"""
+ self.value = value
+
+ def __int__(self) -> int:
+ return self.value
+
+ def __bool__(self):
+ return bool(self.value)
+
+
+class capture_docker_exit_code:
+ """Context manager to get error code when using docker::run"""
+
+ def __init__(self, print_exc: bool = True):
+ self.exit_code = _MutableInt()
+ self.print_exc = print_exc
+
+ def __enter__(self):
+ return self.exit_code
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ if exc_type == DockerException:
+ self.exit_code = exc_val.return_code
+ if self.print_exc:
+ if exc_val.stdout:
+ print(exc_val.stdout)
+ if exc_val.stderr:
+ print(exc_val.stderr, file=sys.stderr)
+ # Need to return a true value to ignore the error
+ return True
+ return False
diff --git a/odoo_scripts/docker_dev_start.py b/odoo_scripts/docker_dev_start.py
--- a/odoo_scripts/docker_dev_start.py
+++ b/odoo_scripts/docker_dev_start.py
@@ -15,11 +15,9 @@
from subprocess import call, check_output
from typing import Any, Dict, List, Optional
-import dockerpty # type: ignore[import]
-from docker.errors import APIError # type: ignore[import]
-from docker.types import Mount # type: ignore[import]
from psycopg2 import OperationalError, connect
from psycopg2.errors import DuplicateDatabase, DuplicateObject
+from python_on_whales import docker
from .config import (
ODOO_7,
@@ -37,8 +35,11 @@
from .docker_build import add_build_options, build_local_image, get_build_options
from .docker_client import (
DockerClient,
+ Mount,
anthem_mounts,
+ capture_docker_exit_code,
modules_mount,
+ mount,
odoo_source_mounts,
)
from .docker_flake8 import apply_flake8, parser_add_flake8_group
@@ -54,9 +55,9 @@
_logger = logging.getLogger(__name__)
-__version__ = "3.14.0"
+__version__ = "3.15.0"
__date__ = "2017-08-11"
-__updated__ = "2023-07-21"
+__updated__ = "2023-10-31"
def __parser(project_name: str) -> ArgumentParser:
@@ -421,7 +422,6 @@
type=int,
)
- # workers options
parser.add_argument(
"--activate-s3-storage",
help="Activate S3 storage using the provided bucket as the main one [default: "
@@ -536,7 +536,6 @@
_logger.debug("Docker image: %s", image)
# detect if docker image already exists
- client = DockerClient.client
if not nmspc.no_build:
failed = build_local_image(get_build_options(nmspc), nmspc.force_build)
if failed:
@@ -545,13 +544,12 @@
# options is only used with subprocess call
options: Dict[str, Any] = {
"name": project_name,
- "tty": True,
- "extra_hosts": {},
- "ports": {},
- "environment": {},
+ "add_hosts": {},
+ "publish": [],
+ "envs": {},
}
if activate_s3_storage:
- options["environment"].update(
+ options["envs"].update(
{
"s3AttachmentStorageActivate": "true",
"s3AttachmentStorageServerBucket": activate_s3_storage,
@@ -563,7 +561,7 @@
# TODO detect that from configuration or force it
odoo_run_port = 8069
if not gevent and not use_host_network:
- options["ports"][exposed_port] = odoo_run_port
+ options["publish"].append((exposed_port, odoo_run_port))
mounts: List[Mount] = []
"""Mounts for Odoo docker"""
tested_modules: str = ""
@@ -656,7 +654,7 @@
# auto detect local ip
if use_host_network:
local_ip = "127.0.0.1"
- options["network_mode"] = "host"
+ options["networks"] = "host"
else:
local_ip = None
try:
@@ -696,10 +694,10 @@
else:
odoo_conf_file = "/etc/opt/odoo/odoo.conf"
# change ODOO_RC for version<ODOO_16
- options["environment"]["ODOO_RC"] = odoo_conf_file
+ options["envs"]["ODOO_RC"] = odoo_conf_file
mounts.append(
- Mount(
+ mount(
odoo_conf_file,
os.path.join(project_path, local_conf_path),
"bind",
@@ -719,7 +717,7 @@
and cp_local.get("options", "redis_host")
) or None
if redis_host:
- options["extra_hosts"][redis_host] = local_ip
+ options["add_hosts"][redis_host] = local_ip
else:
_logger.info("No configuration file at: %s", local_conf_path)
# default values if nothing else
@@ -746,8 +744,8 @@
# this path is in odoo code, the last part is the system user
session_volume_name = "{}{}".format(base_data_volume_name, "_sessions")
target = "/tmp/oe-sessions-{}".format("odoo")
- client.create_volume(session_volume_name, {"mounted in": target})
- mounts.append(Mount(target, session_volume_name))
+ docker.volume.create(session_volume_name, labels={"mount-location": target})
+ mounts.append(mount(target, session_volume_name))
# by default odoo store attachments in database but it can be set on
# disk. mount the volume in all cases.
# XXX data is the path set in the database (in the form file:///data)
@@ -759,15 +757,15 @@
"openerp-7.0-py2.7.egg/openerp/data"
)
filestore_volume_name = "{}_filestore".format(base_data_volume_name)
- client.create_volume(filestore_volume_name, {"mounted in": target})
- mounts.append(Mount(target, filestore_volume_name))
+ docker.volume.create(filestore_volume_name, labels={"mount-location": target})
+ mounts.append(mount(target, filestore_volume_name))
else:
data_volume_path = "/var/lib/data"
- mounts.append(Mount(data_volume_path, base_data_volume_name))
+ mounts.append(mount(data_volume_path, base_data_volume_name))
_, created = DockerClient.create_volume(
base_data_volume_name,
{
- "mounted in": data_volume_path,
+ "mount-location": data_volume_path,
},
)
if created:
@@ -777,52 +775,13 @@
if run_chown:
chown_directories.add(data_volume_path)
- # avoid the duplication of unbind volumes with all addons
- if odoo_type == ODOO_7:
- extra_volumes = [
- "/opt/odoo/additional_addons",
- "/opt/odoo/var",
- # not used but declared as a mount point in Dockerfile :-(
- "/opt/odoo/data",
- ]
- elif odoo_type == ODOO_8:
- extra_volumes = [
- "/opt/odoo/additional_addons",
- "/opt/odoo/var",
- "/opt/odoo/data",
- ]
- elif odoo_type == ODOO_10:
- extra_volumes = [
- "/opt/odoo/additional_addons",
- "/opt/odoo/var",
- "/opt/odoo/data",
- ]
- elif odoo_type in (ODOO_11, ODOO_12, ODOO_13, ODOO_14):
- extra_volumes = ["/var/lib/odoo"]
- else:
- extra_volumes = ["/var/opt/odoo"]
- for extra_volume in extra_volumes:
- volume_name = "{}_{}".format(project_name, extra_volume.replace("/", "_"))
- _volume, created = DockerClient.create_volume(
- volume_name,
- {
- "mounted in": extra_volume,
- "comment": "only used to avoid creating too many volumes",
- },
- )
- mounts.append(Mount(extra_volume, volume_name))
- if created:
- chown_directories.add(extra_volume)
-
# make sure the permission in the volumes are correct
if run_chown:
- for extra_volume in extra_volumes:
- chown_directories.add(extra_volume)
chown_directories.add("/mnt")
if chown_directories:
_logger.info("chown container’s %s to Odoo user", ", ".join(chown_directories))
- client.containers.run(
+ docker.run(
image,
command=["odoo", "--recursive"] + list(chown_directories),
remove=True,
@@ -834,8 +793,8 @@
if start_py3o_stack:
start_py3o()
# provide a couple of handy dns name for py3o
- options["extra_hosts"]["py3o"] = local_ip
- options["extra_hosts"]["py3o-fusion-server"] = local_ip
+ options["add_hosts"]["py3o"] = local_ip
+ options["add_hosts"]["py3o-fusion-server"] = local_ip
# Database connection
@@ -864,7 +823,8 @@
pg, _stop_postgresql, socket_path = docker_run_postgresql(
project_name, postgresql_version, host_pg_port=None
)
- mounts.append(Mount(db_host, socket_path, "bind"))
+ if socket_path is not None:
+ mounts.append(mount(db_host, socket_path, "bind"))
_logger.info("Testing database connection on base postgres")
try:
if start_postgresql:
@@ -875,7 +835,7 @@
host=socket_path,
port=dbport,
)
- _logger.info("Connection successful")
+ _logger.info("DB connection successful")
else:
# psql uses PGHOST if provided and a socket otherwise
if db_host:
@@ -890,7 +850,7 @@
db_host = local_ip
if db_host.startswith("/"):
# mount the path inside the docker
- mounts.append(Mount(db_host, db_host, "bind"))
+ mounts.append(mount(db_host, db_host, "bind"))
connect(
user=user,
password=password,
@@ -908,7 +868,7 @@
connection = connect(
user="postgres",
database="postgres",
- host=db_host,
+ host=socket_path,
port=5432,
password=POSTGRES_PASSWORD,
)
@@ -917,6 +877,7 @@
# use socket to create user
# TODO test if removing more arguments works
connection = connect(user=loginname, database="postgres")
+ _logger.info("User created")
with connection.cursor() as cursor:
# not injection safe but you are on your own machine
# with already full access to db
@@ -958,30 +919,32 @@
# TODO find out why odoo_help would stop us from restoring
if start_postgresql and not odoo_help and (restore_filename or create_database):
_logger.info("Creating database %s for %s", database, user)
- pg.exec_run(["createdb", "-U", user, database])
+ pg.execute(["createdb", "-U", user, database])
if start_postgresql and not odoo_help and restore_filename:
restore_basename = os.path.basename(restore_filename)
_logger.info("Copying dump file in docker")
inside_restore_filename = "/tmp/{}".format(restore_basename)
- DockerClient.put_file(restore_filename, pg, inside_restore_filename)
+ docker.copy(restore_filename, (pg, inside_restore_filename))
_logger.info("Restoring database %s", database)
- restore, _output = pg.exec_run(
- [
- "pg_restore",
- "-U",
- user,
- "--no-owner",
- "--no-acl",
- "--job=4",
- "--dbname",
- database,
- inside_restore_filename,
- ]
- )
- if not restore:
+ with capture_docker_exit_code() as exit_code:
+ pg.execute(
+ [
+ "pg_restore",
+ "-U",
+ user,
+ "--no-owner",
+ "--no-acl",
+ "--job=4",
+ "--dbname",
+ database,
+ inside_restore_filename,
+ ]
+ )
+ if int(exit_code):
+ _logger.fatal("Error when restoring database")
return 15
_logger.info("Removing dump file in docker")
- pg.exec_run(["rm", inside_restore_filename])
+ pg.execute(["rm", inside_restore_filename])
if not start_postgresql and not odoo_help and (restore_filename or create_database):
connection = connect(
@@ -1028,29 +991,30 @@
pre_sql_script_basename = os.path.basename(pre_sql_script)
_logger.info("Copying SQL script %s in docker", pre_sql_script_basename)
inside_pre_sql_script = "/tmp/{}".format(pre_sql_script_basename)
- client.put_file(pre_sql_script, pg, inside_pre_sql_script)
+ docker.copy(pre_sql_script, (pg, inside_pre_sql_script))
_logger.info(
"Running SQL script %s in %s",
pre_sql_script_basename,
database,
)
- exit_code, _output = pg.exec_run(
- [
- "psql",
- "-U",
- user,
- "-d",
- database,
- "-f",
- inside_pre_sql_script,
- ]
- )
- if exit_code > 0:
+ with capture_docker_exit_code() as exit_code:
+ pg.execute(
+ [
+ "psql",
+ "-U",
+ user,
+ "-d",
+ database,
+ "-f",
+ inside_pre_sql_script,
+ ]
+ )
+ if int(exit_code) > 0:
_logger.fatal("Error when running SQL script %s", pre_sql_script)
return 16
_logger.info("Removing SQL script %s in docker", pre_sql_script)
# remove script
- pg.exec_run(["rm", inside_pre_sql_script])
+ pg.execute(["rm", inside_pre_sql_script])
else:
_logger.info("Running SQL script %s on %s", pre_sql_script, database)
env = {
@@ -1124,7 +1088,6 @@
)
return 81
redner_docker = DockerRednerd(
- DockerClient.client,
project_name,
config.rednerd_repository,
db_user=redner_db_user,
@@ -1141,11 +1104,11 @@
)
redner_docker.start()
# only useful when using port rather than socket. Not used at the moment
- options["extra_hosts"]["redner"] = local_ip
+ options["add_hosts"]["redner"] = local_ip
# bind the socket dir in Odoo
if hasattr(redner_docker, "socket_dir"):
mounts.append(
- Mount(redner_docker.socket_dir, redner_docker.socket_dir, type="bind")
+ mount(redner_docker.socket_dir, redner_docker.socket_dir, "bind")
)
# --- Rednerd change config parameters in Odoo database ---
@@ -1177,13 +1140,11 @@
redis_docker: Optional[DockerRedis] = None
if start_redis:
# TODO stop at exit à vrai sinon s’éteint pas?
- redis_docker = DockerRedis(DockerClient.client, project_name, attach=False)
+ redis_docker = DockerRedis(project_name, attach=False)
redis_docker.start()
# bind the socket dir in Odoo
- mounts.append(
- Mount(redis_docker.socket_dir, redis_docker.socket_dir, type="bind")
- )
- options["environment"].update(
+ mounts.append(mount(redis_docker.socket_dir, redis_docker.socket_dir, "bind"))
+ options["envs"].update(
# Variables used by flanker.
REDIS_PORT="0",
REDIS_DB="0",
@@ -1231,7 +1192,7 @@
if force_python_dev_mode or (
config.pythondevmode and not force_no_python_dev_mode
):
- options["environment"]["PYTHONDEVMODE"] = 1
+ options["envs"]["PYTHONDEVMODE"] = 1
if test_tags:
arg.extend(("--test-tags", test_tags))
@@ -1257,9 +1218,9 @@
if os.path.exists(".git"):
project_version = check_output(["git", "describe"]).decode()
- options["environment"]["VERSION_PACKAGE"] = ""
- options["environment"]["SENTRY_RELEASE"] = project_version
- options["environment"]["VERSION"] = project_version
+ options["envs"]["VERSION_PACKAGE"] = ""
+ options["envs"]["SENTRY_RELEASE"] = project_version
+ options["envs"]["VERSION"] = project_version
if odoo_help:
arg.append("--help")
@@ -1271,13 +1232,13 @@
# avoid chowning at each run, faster but can cause trouble
# newer odoo images does not use this environment variable anymore
- options["environment"]["ODOO_CHOWN"] = "false"
+ options["envs"]["ODOO_CHOWN"] = "false"
if coverage:
source_files = ",".join(
[f"odoo.addons.{module}" for module in tested_modules.split(",")]
)
- options["environment"]["COVERAGE_RUN_OPTIONS"] = (
+ options["envs"]["COVERAGE_RUN_OPTIONS"] = (
"--source=" + source_files + " --omit=*/__manifest__.py --branch"
)
@@ -1294,16 +1255,16 @@
marabunta_container_path = "/etc/marabunta.yaml"
# add marabunta file to mounts
mounts.append(
- Mount(
+ mount(
marabunta_container_path,
os.path.join(project_path, marabunta_migration_file),
"bind",
- read_only=True,
+ True,
)
)
mounts.extend(anthem_mounts(config))
- options["environment"].update(
+ options["envs"].update(
{
"MARABUNTA_MIGRATION_FILE": marabunta_container_path,
"MARABUNTA_MODE": "local",
@@ -1311,29 +1272,27 @@
}
)
if start_postgresql:
- options["environment"]["MARABUNTA_DB_USER"] = (
- marabunta_db_user or "postgres"
- )
+ # TODO Ã tester, je pense que postgres ne se connecte pas
+ options["envs"]["MARABUNTA_DB_USER"] = marabunta_db_user or "postgres"
elif marabunta_db_user:
# only set if provided a value, otherwise the image will get the value from
# command line arguments or the configuration file (using the same user os
# the one used in odoo)
- options["environment"]["MARABUNTA_DB_USER"] = marabunta_db_user
+ options["envs"]["MARABUNTA_DB_USER"] = marabunta_db_user
# marabunta is run inside the container by the start command
else:
# Only set to avoid problems with image that inherit another image using
# marabunta while not using marabunta themselves
- options["environment"]["MARABUNTA_MIGRATION_FILE"] = ""
+ options["envs"]["MARABUNTA_MIGRATION_FILE"] = ""
if marabunta_force_version:
- options["environment"]["MARABUNTA_FORCE_VERSION"] = marabunta_force_version
+ options["envs"]["MARABUNTA_FORCE_VERSION"] = marabunta_force_version
if marabunta_db_password:
- options["environment"]["MARABUNTA_DB_PASSWORD"] = marabunta_db_password
+ options["envs"]["MARABUNTA_DB_PASSWORD"] = marabunta_db_password
if gevent:
gevent_port = find_free_port()
run_kwargs: Dict[str, Any] = {
"command": ["caddy", "run", "--config", "/etc/caddy/Caddyfile"],
- "detach": True,
"name": f"{project_name}_caddy",
"mounts": [],
}
@@ -1343,7 +1302,7 @@
# change the port odoo runs on
# TODO hopefully does not return 8069/odoo_run_port
odoo_run_port = find_free_port()
- options["environment"]["MARABUNTA_WEB_PORT"] = str(odoo_run_port)
+ options["envs"]["MARABUNTA_WEB_PORT"] = str(odoo_run_port)
else:
caddy_port = "2019"
# TODO pre odoo 13: check if http-port should be changed to xmlrpc-port
@@ -1394,19 +1353,12 @@
atexit.register(cleanup_caddyfile)
run_kwargs["mounts"].append(
- Mount("/etc/caddy/Caddyfile", caddyfile_tmp.name, type="bind")
+ mount("/etc/caddy/Caddyfile", caddyfile_tmp.name, "bind")
)
if use_host_network:
- run_kwargs["network_mode"] = "host"
+ run_kwargs["networks"] = "host"
else:
- try:
- client.networks.create(
- name=project_name, driver="bridge", check_duplicate=True
- )
- except APIError:
- # TODO handle that more correctly
- # probably already exists
- pass
+ docker.network.create(name=project_name, driver="bridge")
run_kwargs.update(
{
"network": project_name,
@@ -1423,34 +1375,24 @@
# so maybe start it after Odoo started? (only happens if odoo windows are
# opened)
# TODO also error when Odoo is powered off
- DockerClient.run("caddy", "latest", run_kwargs)
+ DockerClient.launch("caddy", "latest", run_kwargs)
- interactive = True
- if interactive:
- options["stdin_open"] = True
-
- odoo_container = client.containers.create(
- image, command=arg, mounts=mounts, **options
- )
_logger.info("Starting Odoo")
- if interactive:
- dockerpty.start(client.api, odoo_container.id)
- # already stopped but this is the easiest way to get the return code
- result_dict = odoo_container.wait()
- result = result_dict["StatusCode"]
- else:
- # XXX untested
- # that might need a way to pass any signal to the container
- # probably prevent the caddy start bellow
- result = odoo_container.start()
-
- # only remove after getting the return code
- odoo_container.remove()
+ with capture_docker_exit_code() as exit_code:
+ docker.container.run(
+ image,
+ command=arg,
+ mounts=mounts,
+ **options,
+ tty=True,
+ interactive=True,
+ remove=True,
+ )
# Stop redis (python does not exit as the log process is still running)
if redis_docker:
_logger.info("Stopping redis")
redis_docker.stop()
- return result
+ return int(exit_code)
def find_free_port():
diff --git a/odoo_scripts/docker_flake8.py b/odoo_scripts/docker_flake8.py
--- a/odoo_scripts/docker_flake8.py
+++ b/odoo_scripts/docker_flake8.py
@@ -100,7 +100,7 @@
"entrypoint": "flake8",
"command": ["/mnt"],
"volumes": {pwd: {"bind": "/mnt", "mode": "ro"}},
- "working_dir": "/mnt",
+ "workdir": "/mnt",
},
pull,
{"/mnt": pwd},
diff --git a/odoo_scripts/docker_isort.py b/odoo_scripts/docker_isort.py
--- a/odoo_scripts/docker_isort.py
+++ b/odoo_scripts/docker_isort.py
@@ -179,7 +179,7 @@
"user": os.getuid(),
"command": command,
"volumes": volumes,
- "working_dir": path,
+ "workdir": path,
},
pull,
)
diff --git a/odoo_scripts/docker_postgresql.py b/odoo_scripts/docker_postgresql.py
--- a/odoo_scripts/docker_postgresql.py
+++ b/odoo_scripts/docker_postgresql.py
@@ -11,17 +11,18 @@
from functools import partial
from typing import Callable, List, Optional, Tuple
-import docker # type: ignore[import]
+from python_on_whales import Container, docker
+from python_on_whales.exceptions import NoSuchContainer
from .config import Config
-from .docker_client import DockerClient
+from .docker_client import DockerClient, Mount, mount
from .parsing import apply, basic_parser
_logger = logging.getLogger(__name__)
-__version__ = "1.0.1"
+__version__ = "1.1.0"
__date__ = "2020-11-09"
-__updated__ = "2022-12-16"
+__updated__ = "2023-10-31"
POSTGRES_PASSWORD = "this-could-be-anything"
PSQL = "psql"
@@ -33,9 +34,9 @@
def docker_run_postgresql(
project_name: str,
postgresql_version: str,
- host_pg_port=None,
+ host_pg_port: Optional[int] = None,
stop_at_exit: bool = True,
-) -> Tuple[docker.models.containers.Container, Callable, Optional[str]]:
+) -> Tuple[Container, Callable, Optional[str]]:
"""
:param project_name:
@@ -45,60 +46,63 @@
:param stop_at_exit: True to stop the container at exit
:return: container, method to call to stop it, socket directory
"""
- docker_client = DockerClient.client
pg_repository = "postgres"
version = "{}-alpine".format(postgresql_version)
pg_image = "{}:{}".format(pg_repository, version)
name = "postgresql-{}-{}".format(postgresql_version, project_name)
# need to pull otherwise the containers.create might fail
- docker_client.images.pull(repository=pg_repository, tag=version)
+ docker.image.pull(f"{pg_repository}:{version}")
pg_data_volume_name = "postgresql_{}-{}".format(postgresql_version, project_name)
path: Optional[str] = None
env = {"POSTGRES_PASSWORD": POSTGRES_PASSWORD}
- def stop_postgresql(pg_container: docker.models.containers.Container):
+ def stop_postgresql(pg_container: Container):
# TODO test if still exists
# maybe by looking at pg.status
# TODO factorized with other methods that start dockers
try:
_logger.info("Stopping postgresql %s", pg_container.name)
pg_container.stop()
- pg_container.wait()
+ docker.wait(pg_container)
_logger.info("Removing postgresql container %s", pg_container.name)
pg_container.remove()
- except docker.errors.NotFound:
+ except NoSuchContainer:
_logger.info("Postgresql container %s already stopped", pg_container.name)
# crash because docker or pg changes the owner of the directory
# temp_directory.cleanup()
- for container in docker_client.containers.list():
+ for container in docker.container.list():
if name == container.name:
_logger.info("Postgresql Container already running")
- for mount_dict in container.attrs["Mounts"]:
- if mount_dict["Destination"] == "/var/run/postgresql":
- path = str(mount_dict["Source"])
+ for mount_ in container.mounts:
+ if mount_.destination == "/var/run/postgresql":
+ path = str(mount_.source)
return container, partial(stop_postgresql, container), path
- volumes = {
- pg_data_volume_name: {
- "bind": "/var/lib/postgresql/data",
- "mode": "rw",
- }
- }
+ mounts: List[Mount] = [
+ mount(
+ "/var/lib/postgresql/data",
+ pg_data_volume_name,
+ "volume",
+ )
+ ]
- port_bindings = {}
- if host_pg_port:
- port_bindings[5432] = host_pg_port
+ port_bindings: List[
+ Tuple[str | int, str | int] | Tuple[str | int, str | int, str]
+ ] = []
+ if isinstance(host_pg_port, int):
+ port_bindings.append((5432, host_pg_port))
else:
path = tempfile.TemporaryDirectory(prefix="odoo_scripts_postgres_socket-").name
- volumes[path] = {"bind": "/var/run/postgresql", "mode": "rw"}
+ os.makedirs(path)
+ mounts.append(mount("/var/run/postgresql", path, "bind"))
_logger.debug("Creating postgresql container")
- pg = docker_client.containers.create(
+ pg = docker.container.create(
pg_image,
- volumes=volumes,
- ports=port_bindings,
+ mounts=mounts,
+ publish=port_bindings,
name=name,
- environment=env,
+ envs=env,
)
_logger.info("Starting postgresql container %s", pg.name)
diff --git a/odoo_scripts/docker_prettier.py b/odoo_scripts/docker_prettier.py
--- a/odoo_scripts/docker_prettier.py
+++ b/odoo_scripts/docker_prettier.py
@@ -97,8 +97,7 @@
"user": os.getuid(),
"command": command,
"volumes": volumes,
- "working_dir": path,
- "tty": True,
+ "workdir": path,
},
pull=False,
)
diff --git a/odoo_scripts/docker_py3o.py b/odoo_scripts/docker_py3o.py
--- a/odoo_scripts/docker_py3o.py
+++ b/odoo_scripts/docker_py3o.py
@@ -3,9 +3,9 @@
import atexit
import logging
-from docker.errors import APIError # type: ignore[import]
+from python_on_whales import docker
-from .docker_client import DockerClient
+from .docker_client import capture_docker_exit_code
_logger = logging.getLogger(__name__)
@@ -13,9 +13,8 @@
# TODO move to docker_client directly
def find_container(name):
"""Return container object from its name"""
- for container in DockerClient.client.containers():
- if name == container.names:
- return container
+ for container in docker.container.list(filters={"name": name}):
+ return container
return None
@@ -28,25 +27,22 @@
def start_py3o(host_fusion_port=8765, stop_at_exit=True):
"""start a py3o container"""
- docker_client = DockerClient.client
fusion_repository = "xcgd/py3o"
fusion_version = "1.0.0"
fusion_name = "py3o_fusion"
fusion_image = "{}:{}".format(fusion_repository, fusion_version)
- try:
+ with capture_docker_exit_code() as exit_code:
_logger.debug("Pulling %s", fusion_repository)
- docker_client.images.pull(repository=fusion_repository, tag=fusion_version)
- except APIError as e:
- _logger.warning("Exception when trying to pull: %s", e)
- if any(
- fusion_name == container.name for container in docker_client.containers.list()
- ):
+ docker.images.pull(f"{fusion_repository}:{fusion_version}")
+ if exit_code:
+ _logger.warning("Error when pulling")
+ if docker.containers.list(filter={"name": fusion_name}):
_logger.debug("%s Container already running", fusion_name)
- remove_and_stop(docker_client, fusion_name)
+ remove_and_stop(docker, fusion_name)
# TODO handle --host-network option
port_bindings = {8765: host_fusion_port}
_logger.debug("Starting %s container", fusion_name)
- fusion = docker_client.containers.create(
+ fusion = docker.containers.create(
image=fusion_image, ports=port_bindings, name=fusion_name
)
_logger.debug("Starting %s container", fusion_name)
diff --git a/odoo_scripts/docker_pylint.py b/odoo_scripts/docker_pylint.py
--- a/odoo_scripts/docker_pylint.py
+++ b/odoo_scripts/docker_pylint.py
@@ -6,11 +6,9 @@
import sys
from typing import Dict, List, Optional
-from docker.types import Mount # type: ignore[import]
-
from .config import Config
from .docker_build import add_build_options, build_local_image, get_build_options
-from .docker_client import DockerClient, anthem_mounts, modules_mount
+from .docker_client import DockerClient, Mount, anthem_mounts, modules_mount, mount
from .parsing import apply, basic_parser
__version__ = "1.2.0"
@@ -96,7 +94,7 @@
cache_volume_name = "pylint-cache"
DockerClient.create_volume(cache_volume_name)
cache_target_path = "/var/cache/pylint"
- mounts.append(Mount(cache_target_path, cache_volume_name))
+ mounts.append(mount(cache_target_path, cache_volume_name))
environment["PYLINTHOME"] = cache_target_path
mounts.extend(anthem_mounts(config))
@@ -113,7 +111,7 @@
# needed to write in the cache
"user": "root",
"mounts": mounts,
- "environment": environment,
+ "envs": environment,
},
False,
target_source_dict,
diff --git a/odoo_scripts/docker_redis.py b/odoo_scripts/docker_redis.py
--- a/odoo_scripts/docker_redis.py
+++ b/odoo_scripts/docker_redis.py
@@ -9,14 +9,12 @@
import time
from typing import List, Optional
-from docker.types import Mount # type: ignore[import]
-
-from .docker_client import DockerClient, DockerService
+from .docker_client import DockerClient, DockerService, mount
from .parsing import apply, basic_parser
-__version__ = "1.0.0"
+__version__ = "1.1.0"
__date__ = "2023-03-20"
-__updated__ = "2023-03-22"
+__updated__ = "2023-10-31"
_logger = logging.getLogger(__name__)
@@ -26,13 +24,11 @@
def __init__(
self,
- docker_client,
project_name: str,
attach: bool = True,
stop_at_exit: bool = True,
):
super().__init__(
- docker_client,
f"redis_{project_name}",
"redis",
"latest",
@@ -49,14 +45,14 @@
result = super()._create_param()
target = "/data"
result["mounts"] = [
- Mount(
+ mount(
target,
DockerClient.create_volume(
- f"{self.name}{target.replace('/', '_')}", {"mounted in": target}
+ f"{self.name}{target.replace('/', '_')}", {"mount-location": target}
)[0].name,
"volume",
),
- Mount("/var/run/redis", self.socket_dir, "bind"),
+ mount("/var/run/redis", self.socket_dir, "bind"),
]
result["command"] = [
"redis-server",
@@ -76,9 +72,14 @@
# socket exists, but it needs to be done inside the container due to permission
# issue)
time.sleep(0.01)
- DockerClient.exec(
- self._container, ["chmod", "777", "/var/run/redis/redis.sock"], "redis"
- )
+ if self._container is not None:
+ DockerClient.exec(
+ self._container, ["chmod", "777", "/var/run/redis/redis.sock"], "redis"
+ )
+ else:
+ # Could happen if container did not start properly
+ _logger.warning("Redis container is not present")
+ return False
return result
def _start(self) -> None:
@@ -126,7 +127,7 @@
parser = __parser()
namespace = parser.parse_args(argv)
apply(namespace)
- docker_redis = DockerRedis(DockerClient.client, namespace.name)
+ docker_redis = DockerRedis(namespace.name)
docker_redis.start()
return 0
diff --git a/odoo_scripts/docker_rednerd.py b/odoo_scripts/docker_rednerd.py
--- a/odoo_scripts/docker_rednerd.py
+++ b/odoo_scripts/docker_rednerd.py
@@ -10,23 +10,22 @@
import tempfile
import time
from multiprocessing import Process
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
from urllib.parse import quote
-import docker # type: ignore[import]
import requests
import requests_unixsocket # type: ignore[import]
-from docker.types import Mount # type: ignore[import]
+from python_on_whales import docker
from .config import Config
-from .docker_client import DockerClient, DockerService
+from .docker_client import DockerClient, DockerService, Mount, mount
from .parsing import apply, basic_parser
_logger = logging.getLogger(__name__)
-__version__ = "1.1.3"
+__version__ = "1.2.0"
__date__ = "2021-01-14"
-__updated__ = "2023-03-16"
+__updated__ = "2023-10-31"
MIGRATE = "migrate"
ADMIN_PASSWORD = "admin-password"
@@ -40,7 +39,6 @@
def __init__(
self,
- docker_client: docker.client.DockerClient,
project_name: str,
repository: str,
db_user: str,
@@ -60,7 +58,6 @@
log_level: str = "info",
):
"""
- :param docker_client: instance of docker client
:param project_name: name of the project, used for some volume and container
:param repository: repository to use for the redner image
:param db_user: redner database user
@@ -75,7 +72,6 @@
"""
for i, argument in enumerate(
(
- docker_client,
project_name,
repository,
db_user,
@@ -86,7 +82,6 @@
assert argument is not None, "Missing required argument %s" % i
# db_password can be none, in that case expect that any password would work
super().__init__(
- docker_client,
f"rednerd_{project_name}",
repository,
version,
@@ -129,7 +124,7 @@
ports: Dict[str, str] = {}
volumes: Dict[str, Dict[str, str]] = {}
mounts: List[Mount] = []
- create_param = dict(
+ create_param: Dict[str, Any] = dict(
name=self.name,
ports=ports,
environment=self.environment,
@@ -138,7 +133,7 @@
network_mode="host",
)
if self.db_is_socket_path:
- mounts.append(Mount(self.db_host, self.db_host, "bind"))
+ mounts.append(mount(self.db_host, self.db_host, "bind"))
# migrate
_logger.info("migrate")
DockerClient.run(
@@ -175,11 +170,12 @@
ports[f"{self.redner_port}/tcp"] = self.redner_port
_logger.info("Using port %s", self.redner_port)
if hasattr(self, "socket_dir"):
+ # TODO change to use mounts
volumes[self.socket_dir] = {
"bind": self.socket_dir,
"mode": "rw",
}
- self._container = self.docker_cli.containers.create(**create_param)
+ self._container = docker.container.create(**create_param)
_logger.info("Starting container %s", self.name)
_logger.debug(create_param)
self._container.start()
@@ -401,7 +397,6 @@
return 81
# Instantiate docker rednerd
docker_rednerd = DockerRednerd(
- DockerClient.client,
project_name,
rednerd_repository,
db_user=db_user,
diff --git a/pyproject.toml b/pyproject.toml
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -35,9 +35,7 @@
]
conf2reST = []
docker = [
- "docker >=3.4",
- "docker <6.0.0; python_version<'3.7'",
- "dockerpty",
+ "python-on-whales",
# used in docker_rednerd
"requests_unixsocket",
"requests",