# HG changeset patch
# User Vincent Hatakeyama <vincent.hatakeyama@xcg-consulting.fr>
# Date 1679502616 -3600
# Wed Mar 22 17:30:16 2023 +0100
# Node ID c78b0972b6f24b85fab58d41bd301975d6ecd078
# Parent 1ad60f46050d62e0377b74b074aa36cef5c5db54
✨ add redis star. add start redis option and make it usable for flanker and session_redis
diff --git a/NEWS.rst b/NEWS.rst
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -2,6 +2,11 @@
History
=======
+20.1.0
+------
+
+docker_dev_start: Add ability to run a redis container.
+
20.0.1
------
diff --git a/odoo_scripts/docker_client.py b/odoo_scripts/docker_client.py
--- a/odoo_scripts/docker_client.py
+++ b/odoo_scripts/docker_client.py
@@ -4,6 +4,7 @@
import os
import re
import tarfile
+from functools import partial
from multiprocessing import Process
from typing import Any, Dict, List, Mapping, Optional, Pattern, Tuple
@@ -331,3 +332,157 @@
)
)
return mounts
+
+
+class DockerService:
+ """Represent a service in a docker and allows common methods to manipulate it."""
+
+ def __init__(
+ self,
+ docker_client: docker.client.DockerClient,
+ name: str,
+ repository: str,
+ tag: str,
+ stop_remove_if_exists: bool,
+ immediate_pull: bool = False,
+ attach: bool = True,
+ stream: bool = True,
+ stop_at_exit: bool = True,
+ force_pull: bool = False,
+ ):
+ """
+ :param docker_client: instance of docker client
+ :param name: name for the container
+ :param repository: repository to use for the image
+ :param tag: repository tag to use for the image
+ :param immediate_pull: pull now (rather than do it later, like during start)
+ :param stream: if not attached, indicate to stream output or not
+ """
+ super().__init__()
+ self.docker_cli = docker_client
+ self.name = name
+ self.repository: str = repository
+ self.tag = tag
+ self.base_image = f"{self.repository}:{self.tag}"
+ if immediate_pull:
+ self.pull_image()
+ self.attach = attach
+ self.stream = stream
+ self.stop_at_exit = stop_at_exit
+ # Retrieve the container if it already exists
+ self._container: Optional[docker.models.containers.Container]
+ try:
+ self._container = self.docker_cli.containers.get(self.name)
+ except docker.errors.NotFound:
+ self._container = None
+ self._log_process: Optional[Process] = None
+ self.stop_remove_if_exists = stop_remove_if_exists
+ self.force_pull: bool = force_pull
+
+ def start(self) -> bool:
+ """Start the container"""
+ # TODO avoid pull if already done with immediate pull
+ self.pull_image()
+ if self._container is not None:
+ # TODO should it crash?
+ _logger.info("Container %s already exist", self.name)
+ if self.stop_remove_if_exists:
+ # stop and remove
+ self.stop_and_remove()
+ elif not self.is_running:
+ _logger.info("Container %s is not running", self.name)
+ self.remove()
+ else:
+ return True
+ self._start()
+ if self.stop_at_exit:
+ atexit.register(partial(self.stop_and_remove, False))
+ if self.attach:
+ DockerClient.print_log(self._container)
+ elif self.stream:
+ process = Process(target=DockerClient.print_log, args=(self._container,))
+ process.start()
+ self._log_process = process
+
+ return True
+
+ def _create_param(self) -> dict:
+ return dict(
+ name=self.name,
+ network_mode="host",
+ image=self.base_image,
+ )
+
+ def _start(self) -> None:
+ self._container = self.docker_cli.containers.create(**self._create_param())
+ _logger.info("Starting container %s", self.name)
+ self._container.start()
+
+ def stop_and_remove(self, expect_running: bool = True) -> None:
+ """Stop and remove the container"""
+ try:
+ self.stop(expect_running) # call stop func
+ self.remove() # call remove func
+ except docker.errors.NotFound as exception:
+ _logger.info("Stop and remove container, got: %s", exception)
+ raise exception
+
+ def stop(self, expect_running: bool = True) -> None:
+ """Stop service container."""
+ try:
+ if not self.is_running:
+ if expect_running:
+ _logger.warning("Container %s is not running", self.name)
+ elif self._container is not None:
+ _logger.info("Stopping %s container", self.name)
+ self._container.stop()
+ self._container.wait()
+ except docker.errors.NotFound as exception:
+ _logger.error("Stopping container %s, got: %s", self.name, exception)
+ raise
+ self.join_log_process()
+
+ def stop_log_process(self):
+ if self._log_process and self._log_process.is_alive():
+ self._log_process.terminate()
+
+ def join_log_process(self):
+ if self._log_process and self._log_process.is_alive():
+ _logger.info("Waiting for log process to finish")
+ self._log_process.join()
+
+ def remove(self):
+ """Remove the service container created"""
+ try:
+ _logger.info("Removing %s container", self.name)
+ self._container.remove()
+ except docker.errors.NotFound as exception:
+ _logger.error("Removing container %s, got: %s", self.name, exception)
+ raise exception
+
+ def pull_image(self):
+ """Pulls image if needed or forced."""
+ if self.force_pull or not self.docker_cli.images.list(self.base_image):
+ _logger.info("Pulling %s", self.base_image)
+ self.docker_cli.images.pull(repository=self.repository, tag=self.tag)
+
+ @property
+ def is_running(self) -> bool:
+ """Verify the container status
+ :return: true if the status is running
+ """
+ if self._container is None:
+ return False
+ result = self.docker_cli.containers.list(filters={"id": self._container.id})
+ return result and result[0].status == "running"
+
+ @property
+ def is_exited(self) -> bool:
+ """
+ Verify the rednerd container status
+ :return: Boolean if the status is exited
+ """
+ if self._container is None:
+ return True
+ result = self.docker_cli.containers.list(filters={"id": self._container.id})
+ return result and result[0].status == "exited"
diff --git a/odoo_scripts/docker_dev_start.py b/odoo_scripts/docker_dev_start.py
--- a/odoo_scripts/docker_dev_start.py
+++ b/odoo_scripts/docker_dev_start.py
@@ -46,6 +46,7 @@
from .docker_postgresql import POSTGRES_PASSWORD, docker_run_postgresql
from .docker_py3o import start_py3o
from .docker_pylint import apply_pylint, parser_add_pylint_group
+from .docker_redis import DockerRedis
from .docker_rednerd import DockerRednerd
from .parsing import apply, basic_parser
@@ -53,9 +54,9 @@
_logger = logging.getLogger(__name__)
-__version__ = "3.10.0"
+__version__ = "3.11.0"
__date__ = "2017-08-11"
-__updated__ = "2023-02-27"
+__updated__ = "2023-03-22"
def __parser(project_name: str) -> ArgumentParser:
@@ -336,6 +337,23 @@
"%(default)s]",
default="redner-odoo-password",
)
+ redis_group = parser.add_mutually_exclusive_group()
+ redis_group.add_argument(
+ "--start-redis",
+ help="start redis docker [default: %(default)s]",
+ action="store_const",
+ default=None,
+ const=1,
+ dest="start_redis",
+ )
+ redis_group.add_argument(
+ "--no-start-redis",
+ help="do not start redis docker [default: %(default)s]",
+ action="store_const",
+ default=None,
+ const=-1,
+ dest="start_redis",
+ )
parser.add_argument(
"--odoo-sources",
@@ -495,6 +513,9 @@
else:
start_rednerd_stack = config.start_rednerd
_logger.debug("No command line start rednerd %d", start_rednerd_stack)
+ if nmspc.start_redis:
+ start_redis = nmspc.start_redis == 1
+ # TODO ajouter l’option de démarrer redis dans Configuration
_logger.debug("Docker image: %s", image)
# detect if docker image already exists
@@ -714,7 +735,7 @@
"/usr/local/lib/python2.7/dist-packages/"
"openerp-7.0-py2.7.egg/openerp/data"
)
- filestore_volume_name = "{}{}".format(base_data_volume_name, "_filestore")
+ filestore_volume_name = "{}_filestore".format(base_data_volume_name)
client.create_volume(filestore_volume_name, {"mounted in": target})
mounts.append(Mount(target, filestore_volume_name))
else:
@@ -1105,6 +1126,28 @@
cursor, redner_user, redner_docker.api_key, server_url
)
+ # --- Redis start ---
+
+ redis_docker: Optional[DockerRedis] = None
+ if start_redis:
+ # TODO stop at exit à vrai sinon s’éteint pas?
+ redis_docker = DockerRedis(DockerClient.client, project_name, attach=False)
+ redis_docker.start()
+ # bind the socket dir in Odoo
+ mounts.append(
+ Mount(redis_docker.socket_dir, redis_docker.socket_dir, type="bind")
+ )
+ options["environment"].update(
+ # Variables used by flanker.
+ REDIS_PORT="0",
+ REDIS_DB="0",
+ # Rely on a flanker patch to work.
+ REDIS_UNIX_SOCKET_PATH=redis_docker.socket_path,
+ # Variables used by session_redis
+ ODOO_SESSION_REDIS="1",
+ ODOO_SESSION_REDIS_URL=f"unix://{redis_docker.socket_path}?db=1",
+ )
+
# volume magic
mount_list, _ = modules_mount(config)
if mount_list:
@@ -1331,6 +1374,10 @@
# only remove after getting the return code
odoo_container.remove()
+ # Stop redis (python does not exit as the log process is still running)
+ if redis_docker:
+ _logger.info("Stopping redis")
+ redis_docker.stop()
return result
diff --git a/odoo_scripts/docker_redis.py b/odoo_scripts/docker_redis.py
new file mode 100644
--- /dev/null
+++ b/odoo_scripts/docker_redis.py
@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""Run a redis docker with volume based on project name.
+"""
+import argparse
+import logging
+import os
+import sys
+import tempfile
+import time
+from typing import List, Optional
+
+from docker.types import Mount # type: ignore[import]
+
+from .docker_client import DockerClient, DockerService
+from .parsing import apply, basic_parser
+
+__version__ = "1.0.0"
+__date__ = "2023-03-20"
+__updated__ = "2023-03-22"
+
+_logger = logging.getLogger(__name__)
+
+
+class DockerRedis(DockerService):
+ """Redis service"""
+
+ def __init__(
+ self,
+ docker_client,
+ project_name: str,
+ attach: bool = True,
+ stop_at_exit: bool = True,
+ ):
+ super().__init__(
+ docker_client,
+ f"redis_{project_name}",
+ "redis",
+ "latest",
+ True,
+ attach=attach,
+ stop_at_exit=stop_at_exit,
+ )
+ self.socket_dir = tempfile.mkdtemp(prefix="odoo_scripts_redis_socket-")
+ """Redis socket’s directory."""
+ os.chmod(self.socket_dir, 0o777)
+ self.socket_path = os.path.join(self.socket_dir, "redis.sock")
+
+ def _create_param(self) -> dict:
+ result = super()._create_param()
+ target = "/data"
+ result["mounts"] = [
+ Mount(
+ target,
+ DockerClient.create_volume(
+ f"{self.name}{target.replace('/', '_')}", {"mounted in": target}
+ )[0].name,
+ "volume",
+ ),
+ Mount("/var/run/redis", self.socket_dir, "bind"),
+ ]
+ result["command"] = [
+ "redis-server",
+ "--save",
+ "--unixsocket",
+ "/var/run/redis/redis.sock",
+ "--port",
+ "0",
+ ]
+ return result
+
+ def start(self) -> bool:
+ result = super().start()
+ # by default the socket does not have permissions that will make it easy to use
+ # (by odoo for example). User is 999, so that does not help much either.
+ # Let redis start (alternative would be to run a command that check that the
+ # socket exists, but it needs to be done inside the container due to permission
+ # issue)
+ time.sleep(0.01)
+ DockerClient.exec(
+ self._container, ["chmod", "777", "/var/run/redis/redis.sock"], "redis"
+ )
+ return result
+
+ def _start(self) -> None:
+ super()._start()
+ if os.path.exists("/proc/sys/vm/overcommit_memory"):
+ with open("/proc/sys/vm/overcommit_memory", "r") as f:
+ data = f.read()
+ if data == "0\n":
+ _logger.info(
+ "The warning in redis about memory overcommit applies to the "
+ "host system"
+ )
+
+ def __del__(self):
+ os.rmdir(self.socket_dir)
+
+
+def __parser() -> argparse.ArgumentParser:
+ program_version_message = f"%(prog)s {__version__} ({__updated__})"
+ program_short_description = __doc__.split(".")[0]
+ program_license = f"""{program_short_description}
+
+ Created by Vincent Hatakeyama on {__date__}.
+ Copyright 2023 XCG Consulting. All rights reserved.
+
+ Licensed under the MIT License
+
+ Distributed on an "AS IS" basis without warranties
+ or conditions of any kind, either express or implied.
+
+ USAGE
+ """
+ parser = basic_parser(program_license, program_version_message)
+ project_path = os.path.realpath(".")
+ project_name = os.path.basename(project_path)
+ parser.add_argument(
+ "--name",
+ help="Container name to use [default: %(default)s]",
+ default=project_name,
+ )
+ return parser
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+ parser = __parser()
+ namespace = parser.parse_args(argv)
+ apply(namespace)
+ docker_redis = DockerRedis(DockerClient.client, namespace.name)
+ docker_redis.start()
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/odoo_scripts/docker_rednerd.py b/odoo_scripts/docker_rednerd.py
--- a/odoo_scripts/docker_rednerd.py
+++ b/odoo_scripts/docker_rednerd.py
@@ -4,7 +4,6 @@
Documentation: http://127.0.0.1:12345/docs if the server is started with port 12345
"""
import argparse
-import atexit
import logging
import os
import sys
@@ -20,7 +19,7 @@
from docker.types import Mount # type: ignore[import]
from .config import Config
-from .docker_client import DockerClient
+from .docker_client import DockerClient, DockerService
from .parsing import apply, basic_parser
_logger = logging.getLogger(__name__)
@@ -34,7 +33,7 @@
SERVE = "serve"
-class DockerRednerd:
+class DockerRednerd(DockerService):
"""
This class allow to start the rednerd server
"""
@@ -86,14 +85,15 @@
):
assert argument is not None, "Missing required argument %s" % i
# db_password can be none, in that case expect that any password would work
- super().__init__()
- self.name = f"rednerd_{project_name}"
- self.repository: str = repository
- self.version = version
- self.attach = attach
- self.base_image = f"{self.repository}:{self.version}"
- self.stop_at_exit = stop_at_exit
- self.docker_cli = docker_client
+ super().__init__(
+ docker_client,
+ f"rednerd_{project_name}",
+ repository,
+ version,
+ False,
+ attach,
+ stop_at_exit,
+ )
self.redner_user = redner_user
self.redner_password = redner_password
self.redner_admin_password = redner_admin_password
@@ -119,27 +119,12 @@
).name
"""Rednerd socket’s directory."""
self.socket_path = os.path.join(self.socket_dir, "rednerd.sock")
-
- # Retrieve the container if it already exists
- try:
- self.container = self.docker_cli.containers.get(self.name)
- except docker.errors.NotFound:
- self.container = None
self.force_pull = force_pull
self.api_key = None
"""Store the API key"""
- self._log_process: Optional[Process] = None
- def start(self) -> bool:
+ def _start(self) -> None:
"""Start rednerd container"""
- if self.force_pull or not self.docker_cli.images.list(self.base_image):
- _logger.info("Pulling %s", self.base_image)
- self.pull_image()
- if self.container is not None:
- # probably to make it work as a restart
- _logger.info("Container %s already exist", self.name)
- # stop and remove
- self.stop_and_remove()
# base parameters for container creation
ports: Dict[str, str] = {}
volumes: Dict[str, Dict[str, str]] = {}
@@ -158,7 +143,7 @@
_logger.info("migrate")
DockerClient.run(
repository=self.repository,
- tag=self.version,
+ tag=self.tag,
run_kwargs=dict(create_param, command=[MIGRATE]),
)
# setup admin password if provided
@@ -166,7 +151,7 @@
_logger.info("admin-password")
DockerClient.run(
repository=self.repository,
- tag=self.version,
+ tag=self.tag,
run_kwargs=dict(
create_param, command=[ADMIN_PASSWORD, self.redner_admin_password]
),
@@ -194,10 +179,10 @@
"bind": self.socket_dir,
"mode": "rw",
}
- self.container = self.docker_cli.containers.create(**create_param)
+ self._container = self.docker_cli.containers.create(**create_param)
_logger.info("Starting container %s", self.name)
_logger.debug(create_param)
- self.container.start()
+ self._container.start()
session: requests.sessions.Session
base_url: str
if hasattr(self, "socket_path"):
@@ -208,7 +193,7 @@
base_url = f"http://localhost:{self.redner_port}"
# if there is neither port or socket, no session nor base_url
if not self.attach:
- process = Process(target=DockerClient.print_log, args=(self.container,))
+ process = Process(target=DockerClient.print_log, args=(self._container,))
process.start()
self._log_process = process
# wait for redner to be started
@@ -233,7 +218,7 @@
if hasattr(self, "socket_path"):
_logger.info("Changing socket permission")
DockerClient.exec(
- self.container, ["chmod", "-R", "777", self.socket_path], "root"
+ self._container, ["chmod", "-R", "777", self.socket_path], "root"
)
if (
@@ -311,77 +296,6 @@
)
self.api_key = request.json()["value"]
- if self.stop_at_exit:
- atexit.register(self.stop_and_remove)
-
- if self.attach:
- DockerClient.print_log(self.container)
-
- return True
-
- def stop_and_remove(self):
- """Stop and remove the rednerd container"""
- try:
- self.stop() # call rednerd stop func
- self.remove() # call rednerd remove func
- except docker.errors.NotFound as exception:
- _logger.info("Stop and remove rednerd container, got: %s", exception)
- raise exception
-
- def stop(self):
- """
- Stop a rednerd container.
- """
- try:
- if not self.is_running:
- _logger.warning("Container %s is not running", self.name)
- _logger.info("Stopping %s container", self.name)
- self.container.stop()
- self.container.wait()
- except docker.errors.NotFound as exception:
- _logger.error("Stopping Redner container, got: %s", exception)
- raise
- if self._log_process and self._log_process.is_alive():
- _logger.info("Waiting for log process to finish")
- self._log_process.join()
-
- def remove(self):
- """Remove the rednerd container created"""
- try:
- _logger.info("Removing %s container", self.name)
- self.container.remove()
- except docker.errors.NotFound as exception:
- _logger.error("Removing rednerd container, got: %s", exception)
- raise exception
-
- def pull_image(self):
- """
- Pulls rednerd image
- """
- self.docker_cli.images.pull(repository=self.repository, tag=self.version)
-
- @property
- def is_running(self):
- """
- Verify the rednerd container status
- :return: Boolean if the status is running
- """
- if self.container is None:
- return False
- result = self.docker_cli.containers.list(filters={"id": self.container.id})
- return result and result[0].status == "running"
-
- @property
- def is_exited(self):
- """
- Verify the rednerd container status
- :return: Boolean if the status is exited
- """
- if self.container is None:
- return True
- result = self.docker_cli.containers.list(filters={"id": self.container.id})
- return result and result[0].status == "exited"
-
def __parser() -> argparse.ArgumentParser:
"""Return a parser for docker_rednerd"""
diff --git a/pyproject.toml b/pyproject.toml
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -67,6 +67,7 @@
odoo_conf_inject_env_var = "odoo_scripts.odoo_conf_inject_env_var:main"
list_modules = "odoo_scripts.list_modules:main"
update_duplicate_sources = "odoo_scripts.update_duplicate_sources:main [source_control]"
+docker_redis = "odoo_scripts.docker_redis:main [docker]"
[project.urls]
repository = "https://orus.io/xcg/odoo_scripts"