Skip to content
Snippets Groups Projects
Commit 60350f857f14 authored by Vincent Hatakeyama's avatar Vincent Hatakeyama
Browse files

:sparkles: update python docker requirements to use the API directly rather than using subprocesses

parent 9889f48ee32d
No related branches found
No related tags found
1 merge request!6Topic/default/docker api
Showing
with 708 additions and 528 deletions
...@@ -12,7 +12,7 @@ ...@@ -12,7 +12,7 @@
TAG_LATEST: branch/default TAG_LATEST: branch/default
DOCKER_IMAGE: xcgd/odoo_scripts DOCKER_IMAGE: xcgd/odoo_scripts
HTML_DOC_SOURCES: doc/_build/html HTML_DOC_SOURCES: doc/_build/html
SH_SCRIPTS: isort create_archive start SH_SCRIPTS: create_archive start
checkbashisms: checkbashisms:
allow_failure: true allow_failure: true
......
...@@ -23,6 +23,12 @@ ...@@ -23,6 +23,12 @@
Colored logs. Colored logs.
Docker calls are now done with the API rather than with subprocesses.
:file:`isort` shell script has been removed, it is now available as a python 3 script :ref:`docker_isort`.
Logging default is now the info level. Use `-q`/`--quiet` to decrease verbosity.
4.1 4.1
--- ---
......
...@@ -73,17 +73,6 @@ ...@@ -73,17 +73,6 @@
zsh script to create a tar.xz file containing all sources. zsh script to create a tar.xz file containing all sources.
isort
-----
Run dockerized isort on current directory. This uses a configuration file that is adapted to OCA guidelines for imports.
For help:
.. code-block:: SH
./isort --help
Scripts Scripts
======= =======
...@@ -94,6 +83,13 @@ ...@@ -94,6 +83,13 @@
The prerequisites for this module or one of its section can be installed by using pip3 or the package manager; the requirements are defined in ``setup.py``. The prerequisites for this module or one of its section can be installed by using pip3 or the package manager; the requirements are defined in ``setup.py``.
As there is no way of indicating sections (it is even handling them incorrectly), do not use ``python3 setup.py install``. As there is no way of indicating sections (it is even handling them incorrectly), do not use ``python3 setup.py install``.
docker_isort
------------
Run dockerized isort on current directory. This uses a configuration file that is adapted to OCA guidelines for imports.
This is part of the docker section.
conf2reST.py conf2reST.py
------------ ------------
...@@ -223,7 +219,7 @@ ...@@ -223,7 +219,7 @@
Generate completion file for python scripts (from the superproject for the scripts that need to be run from there, and with the required requirements too):: Generate completion file for python scripts (from the superproject for the scripts that need to be run from there, and with the required requirements too)::
for command_name in docker_dev_start docker_build docker_build_copy docker_build_clean do_tests conf2reST import_base_import import_jsonrpc import_sql ; do for command_name in docker_dev_start docker_build docker_build_copy docker_build_clean do_tests conf2reST import_base_import import_jsonrpc import_sql docker_flake8 docker_isort ; do
$command_name --help | ~/src/zsh-completion-generator/help2comp.py $command_name > ~/.local/share/zsh/completion/_$command_name $command_name --help | ~/src/zsh-completion-generator/help2comp.py $command_name > ~/.local/share/zsh/completion/_$command_name
done done
......
#!/bin/sh
/usr/bin/docker run --rm --volume "$PWD:/mnt" -ti -w /mnt xcgd/isort:odoo isort "$@"
#!/usr/bin/env python3 #!/usr/bin/env python3
"""Generate a reST file from configuration files. """Generate a reST file from configuration files.
""" """
import argparse
import configparser import configparser
import logging import logging
import os import os
...@@ -10,7 +9,7 @@ ...@@ -10,7 +9,7 @@
import yaml import yaml
from .parsing import add_verbosity_to_parser, logging_from_verbose from .parsing import apply, basic_parser
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
...@@ -50,8 +49,7 @@ ...@@ -50,8 +49,7 @@
str(__date__), str(__date__),
) )
# Argument parsing # Argument parsing
parser = argparse.ArgumentParser( parser = basic_parser(
description=program_license, description=program_license, version=program_version_message
formatter_class=argparse.RawDescriptionHelpFormatter,
) )
parser.add_argument( parser.add_argument(
...@@ -56,9 +54,5 @@ ...@@ -56,9 +54,5 @@
) )
parser.add_argument( parser.add_argument(
"-V", "--version", action="version", version=program_version_message
)
add_verbosity_to_parser(parser)
parser.add_argument(
"-d", "-d",
"--directory", "--directory",
help="Source directory [default: %(default)s]", help="Source directory [default: %(default)s]",
...@@ -67,7 +61,7 @@ ...@@ -67,7 +61,7 @@
parser.add_argument("-o", "--output", help="Output file name") parser.add_argument("-o", "--output", help="Output file name")
nmspc = parser.parse_args() nmspc = parser.parse_args()
logging_from_verbose(nmspc) apply(nmspc)
conf2rst(nmspc.directory, nmspc.output) conf2rst(nmspc.directory, nmspc.output)
......
#!/usr/bin/env python3 #!/usr/bin/env python3
# vim: set shiftwidth=4 softtabstop=4: # vim: set shiftwidth=4 softtabstop=4:
"""Script to run test with local modules """Script to run test with local modules
isort:skip_file
""" """
# Version 4.2.1 # Version 4.2.1
...@@ -6,6 +4,4 @@ ...@@ -6,6 +4,4 @@
""" """
# Version 4.2.1 # Version 4.2.1
import argparse
from configparser import ConfigParser
import logging import logging
import os import os
...@@ -10,3 +6,5 @@ ...@@ -10,3 +6,5 @@
import logging import logging
import os import os
import sys
from configparser import ConfigParser
from subprocess import call from subprocess import call
...@@ -12,3 +10,2 @@ ...@@ -12,3 +10,2 @@
from subprocess import call from subprocess import call
import sys
...@@ -14,4 +11,4 @@ ...@@ -14,4 +11,4 @@
import docker # apt python3-docker (1.9) or pip3 install docker from odoo_scripts.docker_postgresql import docker_run_postgresql
from psycopg2 import connect, OperationalError # apt python3-psycopg2 from psycopg2 import OperationalError, connect
...@@ -17,7 +14,2 @@ ...@@ -17,7 +14,2 @@
from .docker_dev_start import (
docker_run_postgresql,
flake8,
main as docker_dev_start_main,
)
from .config import Config from .config import Config
...@@ -23,9 +15,9 @@ ...@@ -23,9 +15,9 @@
from .config import Config from .config import Config
from .docker_dev_start import main as docker_dev_start_main
if docker.__version__ > "2.5.0": from .docker_flake8 import flake8
from docker import APIClient as docker_api from .docker_isort import isort
else: from .docker_postgresql import POSTGRES_PASSWORD
from docker import Client as docker_api from .parsing import apply, basic_parser
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
...@@ -29,5 +21,5 @@ ...@@ -29,5 +21,5 @@
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
__version__ = "1.0.2" __version__ = "2.0.0"
__date__ = "2018-04-13" __date__ = "2018-04-13"
...@@ -33,5 +25,5 @@ ...@@ -33,5 +25,5 @@
__date__ = "2018-04-13" __date__ = "2018-04-13"
__updated__ = "2020-09-02" __updated__ = "2020-09-15"
def main(argv=None): # IGNORE:C0111 def main(argv=None): # IGNORE:C0111
...@@ -60,6 +52,6 @@ ...@@ -60,6 +52,6 @@
str(__date__), str(__date__),
) )
# TODO the script assume it is launched from the parent super project # the script assume it is launched from the super project
project_path = os.path.realpath(".") project_path = os.path.realpath(".")
project_name = os.path.basename(project_path) project_name = os.path.basename(project_path)
...@@ -64,12 +56,5 @@ ...@@ -64,12 +56,5 @@
project_path = os.path.realpath(".") project_path = os.path.realpath(".")
project_name = os.path.basename(project_path) project_name = os.path.basename(project_path)
# XXX is this test still valid?
if project_name == "odoo_scripts":
logging.fatal(
"You must run this script from the super project"
" (docker_build.py)"
)
return 1
# TODO add a way to store configuration options in a project file # TODO add a way to store configuration options in a project file
# Argument parsing # Argument parsing
...@@ -73,19 +58,8 @@ ...@@ -73,19 +58,8 @@
# TODO add a way to store configuration options in a project file # TODO add a way to store configuration options in a project file
# Argument parsing # Argument parsing
parser = argparse.ArgumentParser( parser = basic_parser(
description=program_license, description=program_license, version=program_version_message
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument(
"-V", "--version", action="version", version=program_version_message
)
parser.add_argument(
"-v",
"--verbose",
dest="verbose",
action="count",
help="set verbosity level [default: %(default)s]",
) )
docker_group = parser.add_mutually_exclusive_group() docker_group = parser.add_mutually_exclusive_group()
docker_group.add_argument( docker_group.add_argument(
...@@ -173,13 +147,7 @@ ...@@ -173,13 +147,7 @@
# - db user for creation/remove # - db user for creation/remove
nmspc = parser.parse_args(argv) nmspc = parser.parse_args(argv)
verbose = nmspc.verbose apply(nmspc)
if not verbose:
logging.basicConfig(level=logging.WARN)
if verbose == 1:
logging.basicConfig(level=logging.INFO)
if verbose and verbose > 1:
logging.basicConfig(level=logging.DEBUG)
dbname = nmspc.dbname dbname = nmspc.dbname
odoo_db_user = nmspc.db_user odoo_db_user = nmspc.db_user
odoo_db_password = nmspc.db_password odoo_db_password = nmspc.db_password
...@@ -254,6 +222,5 @@ ...@@ -254,6 +222,5 @@
"-d", "-d",
dbname, dbname,
"--stop-after-init", "--stop-after-init",
"--max-cron-threads", "--max-cron-threads=0",
"0",
"--no-flake8", "--no-flake8",
...@@ -259,4 +226,5 @@ ...@@ -259,4 +226,5 @@
"--no-flake8", "--no-flake8",
"--no-isort",
"--no-dev", "--no-dev",
] ]
if odoo_db_user: if odoo_db_user:
...@@ -269,4 +237,5 @@ ...@@ -269,4 +237,5 @@
args.append("--dbport") args.append("--dbport")
args.append(dbport) args.append(dbport)
# only run flake8 and isort once rather than once per docker_dev_start call
flake8(odoo_type) flake8(odoo_type)
...@@ -272,4 +241,5 @@ ...@@ -272,4 +241,5 @@
flake8(odoo_type) flake8(odoo_type)
isort()
if nmspc.log_handler: if nmspc.log_handler:
for lh in nmspc.log_handler: for lh in nmspc.log_handler:
args.append("--log-handler") args.append("--log-handler")
...@@ -281,9 +251,8 @@ ...@@ -281,9 +251,8 @@
args.append(nmspc.LOAD_LANGUAGE) args.append(nmspc.LOAD_LANGUAGE)
if recreate_db: if recreate_db:
if start_postgresql: if start_postgresql:
docker_client = docker_api(base_url="unix://var/run/docker.sock") container, stop_method, socket_path = docker_run_postgresql(
name, stop_method, socket_path = docker_run_postgresql( project_name, postgresql_version
docker_client, project_name, postgresql_version, None
) )
# system user used in the pg docker image # system user used in the pg docker image
pg_user = "postgres" pg_user = "postgres"
...@@ -302,5 +271,6 @@ ...@@ -302,5 +271,6 @@
user=pg_user, user=pg_user,
database="postgres", database="postgres",
host=socket_path, host=socket_path,
post=5432, port=5432,
password=POSTGRES_PASSWORD,
) )
...@@ -306,4 +276,5 @@ ...@@ -306,4 +276,5 @@
) )
_logger.info("Creating role %s", odoo_db_user)
with connection.cursor() as cursor: with connection.cursor() as cursor:
# not injection safe but you are on your own machine # not injection safe but you are on your own machine
# with already full access to db # with already full access to db
...@@ -333,7 +304,9 @@ ...@@ -333,7 +304,9 @@
for extension in extensions: for extension in extensions:
_logger.info("Adding extension %s", extension) _logger.info("Adding extension %s", extension)
cursor.execute("CREATE EXTENSION %s" % extension) cursor.execute("CREATE EXTENSION %s" % extension)
stop_method() # TODO do we really need to stop the container each time?
# that seems faster
# stop_method()
else: else:
# TODO use psycopg2 (see above) # TODO use psycopg2 (see above)
# drop database # drop database
......
...@@ -11,8 +11,6 @@ ...@@ -11,8 +11,6 @@
import sys import sys
from subprocess import call, check_output from subprocess import call, check_output
import docker # apt python3-docker (1.9) or pip3 install docker
from .config import Config from .config import Config
from .docker_build_clean import clean from .docker_build_clean import clean
from .docker_build_copy import copy from .docker_build_copy import copy
...@@ -16,14 +14,8 @@ ...@@ -16,14 +14,8 @@
from .config import Config from .config import Config
from .docker_build_clean import clean from .docker_build_clean import clean
from .docker_build_copy import copy from .docker_build_copy import copy
from .parsing import basic_parser, logging_from_verbose from .docker_client import DockerClient
from .parsing import apply, basic_parser
# TODO change requirements to increase docker version to avoid that
if docker.__version__ > "2.5.0":
from docker import APIClient as docker_api
else:
from docker import Client as docker_api
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
...@@ -27,5 +19,5 @@ ...@@ -27,5 +19,5 @@
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
__version__ = "0.2.0" __version__ = "1.0.0"
__date__ = "2018-04-04" __date__ = "2018-04-04"
...@@ -31,5 +23,5 @@ ...@@ -31,5 +23,5 @@
__date__ = "2018-04-04" __date__ = "2018-04-04"
__updated__ = "2020-06-30" __updated__ = "2020-09-11"
def __parser(): def __parser():
...@@ -43,7 +35,7 @@ ...@@ -43,7 +35,7 @@
program_license = """%s program_license = """%s
Created by Vincent Hatakeyama on %s. Created by Vincent Hatakeyama on %s.
Copyright 2018, 2020 XCG Consulting. All rights reserved. Copyright 2018, 2019, 2020 XCG Consulting. All rights reserved.
Licensed under the MIT License Licensed under the MIT License
...@@ -66,7 +58,7 @@ ...@@ -66,7 +58,7 @@
project_path = os.path.realpath(".") project_path = os.path.realpath(".")
project_name = os.path.basename(project_path) project_name = os.path.basename(project_path)
if project_name == "odoo_scripts": if project_name == "odoo_scripts":
logging.fatal("You must run this script from the super project") _logger.fatal("You must run this script from the super project")
return 1 return 1
# TODO add a way to store configuration options in a project file # TODO add a way to store configuration options in a project file
...@@ -101,6 +93,4 @@ ...@@ -101,6 +93,4 @@
) )
# TODO (maybe) add argument for other build arg # TODO (maybe) add argument for other build arg
# TODO detect that user is member of docker group
nmspc = parser.parse_args(argv) nmspc = parser.parse_args(argv)
...@@ -106,5 +96,5 @@ ...@@ -106,5 +96,5 @@
nmspc = parser.parse_args(argv) nmspc = parser.parse_args(argv)
logging_from_verbose(nmspc) apply(nmspc)
ensureconf = nmspc.ensureconf ensureconf = nmspc.ensureconf
dev = nmspc.dev dev = nmspc.dev
push = nmspc.push push = nmspc.push
...@@ -114,7 +104,7 @@ ...@@ -114,7 +104,7 @@
project = c.image project = c.image
odoo_type = c.odoo_type odoo_type = c.odoo_type
image = "%s/%s:latest" % (registry, project) image = "%s/%s:latest" % (registry, project)
logging.debug("Docker image: %s", image) _logger.debug("Docker image: %s", image)
# TODO ensureconf # TODO ensureconf
if ensureconf: if ensureconf:
raise NotImplementedError raise NotImplementedError
...@@ -127,8 +117,6 @@ ...@@ -127,8 +117,6 @@
clean() clean()
# XXX needed? # XXX needed?
# sys.exit(0)
signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGTERM, signal_handler)
...@@ -132,6 +120,5 @@ ...@@ -132,6 +120,5 @@
signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGTERM, signal_handler)
# TODO docker build
buildargs = dict() buildargs = dict()
if os.path.exists(".hg"): if os.path.exists(".hg"):
...@@ -136,10 +123,16 @@ ...@@ -136,10 +123,16 @@
buildargs = dict() buildargs = dict()
if os.path.exists(".hg"): if os.path.exists(".hg"):
buildargs["REVISION"] = ( tags = check_output(["hg", "identify", "--tags"]).decode()
check_output("hg identify -i".split()).split()[0].decode("utf-8") if tags:
) buildargs["VERSION"] = tags
buildargs["CREATED"] = datetime.datetime.now().isoformat() buildargs["VCS_URL"] = check_output(
["hg", "paths", "default"]
).decode()
buildargs["VCS_REF"] = check_output(
["hg", "identify", "--id"]
).decode()
buildargs["BUILD_DATE"] = datetime.datetime.now().isoformat()
if nmspc.build_arg: if nmspc.build_arg:
for arg in nmspc.build_arg: for arg in nmspc.build_arg:
a = arg.split("=") a = arg.split("=")
buildargs[a[0]] = a[1] buildargs[a[0]] = a[1]
...@@ -142,8 +135,8 @@ ...@@ -142,8 +135,8 @@
if nmspc.build_arg: if nmspc.build_arg:
for arg in nmspc.build_arg: for arg in nmspc.build_arg:
a = arg.split("=") a = arg.split("=")
buildargs[a[0]] = a[1] buildargs[a[0]] = a[1]
logging.debug("Build args: %s", buildargs) _logger.debug("Build args: %s", buildargs)
dockerfile = "Dockerfile" dockerfile = "Dockerfile"
if dev: if dev:
debug_dockerfile = "Dockerfile.debug" debug_dockerfile = "Dockerfile.debug"
...@@ -172,5 +165,5 @@ ...@@ -172,5 +165,5 @@
dockerfile = debug_dockerfile dockerfile = debug_dockerfile
# TODO remove temp image # TODO remove temp image
docker_client = docker_api(base_url="unix://var/run/docker.sock") docker_client = DockerClient.client
pull = not nmspc.no_pull pull = not nmspc.no_pull
...@@ -176,6 +169,6 @@ ...@@ -176,6 +169,6 @@
pull = not nmspc.no_pull pull = not nmspc.no_pull
logging.debug("Docker Pull %s", pull) _logger.debug("Docker Pull %s", pull)
builder = docker_client.build( builder = docker_client.api.build(
path=".", path=".",
rm=True, rm=True,
pull=pull, pull=pull,
...@@ -183,8 +176,6 @@ ...@@ -183,8 +176,6 @@
tag=image, tag=image,
dockerfile=dockerfile, dockerfile=dockerfile,
) )
# this is for python docker 1.8-1.9
# TODO add compatibility with newer python docker
for line in builder: for line in builder:
d = json.loads(line.decode("utf-8")) d = json.loads(line.decode("utf-8"))
if "stream" in d: if "stream" in d:
...@@ -188,5 +179,6 @@ ...@@ -188,5 +179,6 @@
for line in builder: for line in builder:
d = json.loads(line.decode("utf-8")) d = json.loads(line.decode("utf-8"))
if "stream" in d: if "stream" in d:
logging.info(d["stream"]) for line in d["stream"].rstrip().split("\n"):
_logger.info(line)
if "errorDetail" in d: if "errorDetail" in d:
...@@ -192,5 +184,5 @@ ...@@ -192,5 +184,5 @@
if "errorDetail" in d: if "errorDetail" in d:
logging.fatal(d["errorDetail"]) _logger.fatal(d["errorDetail"])
return 1 return 1
if dev: if dev:
call(["rm", dockerfile]) call(["rm", dockerfile])
...@@ -194,7 +186,6 @@ ...@@ -194,7 +186,6 @@
return 1 return 1
if dev: if dev:
call(["rm", dockerfile]) call(["rm", dockerfile])
# TODO exit if build failed
# TODO docker tag with tags/bookmarks (unused so maybe no need) # TODO docker tag with tags/bookmarks (unused so maybe no need)
# TODO docker push (only when asked for) # TODO docker push (only when asked for)
......
...@@ -8,7 +8,7 @@ ...@@ -8,7 +8,7 @@
import sys import sys
from .list_modules import MODULES_LIST_FILE from .list_modules import MODULES_LIST_FILE
from .parsing import basic_parser, logging_from_verbose from .parsing import apply, basic_parser
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
...@@ -49,7 +49,7 @@ ...@@ -49,7 +49,7 @@
""" """
parser = __parser() parser = __parser()
nmspc = parser.parse_args(argv) nmspc = parser.parse_args(argv)
logging_from_verbose(nmspc) apply(nmspc)
# TODO check odoo_scripts before deleting anything # TODO check odoo_scripts before deleting anything
clean() clean()
return 0 return 0
......
...@@ -9,7 +9,7 @@ ...@@ -9,7 +9,7 @@
from .config import Config from .config import Config
from .list_modules import list_modules from .list_modules import list_modules
from .parsing import basic_parser, logging_from_verbose from .parsing import apply, basic_parser
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
...@@ -13,5 +13,5 @@ ...@@ -13,5 +13,5 @@
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
__version__ = "1.0.0" __version__ = "1.0.1"
__date__ = "2020-06-30" __date__ = "2020-06-30"
...@@ -17,5 +17,5 @@ ...@@ -17,5 +17,5 @@
__date__ = "2020-06-30" __date__ = "2020-06-30"
__updated__ = "2020-06-30" __updated__ = "2020-09-11"
def __parser(): def __parser():
...@@ -50,7 +50,7 @@ ...@@ -50,7 +50,7 @@
""" """
parser = __parser() parser = __parser()
nmspc = parser.parse_args(argv) nmspc = parser.parse_args(argv)
logging_from_verbose(nmspc) apply(nmspc)
copy() copy()
return 0 return 0
......
import logging
import os
import tarfile
from typing import Dict
import docker
_logger = logging.getLogger(__name__)
if docker.__version__ < "3.4.0":
_logger.warning(
"Unexepected python docker version: %s", docker.__version__
)
# TODO detect that user is member of docker group
class DockerClient:
client = docker.from_env()
@classmethod
def create_volume(
cls, volume_name: str, extra_labels: Dict[str, str] = None
) -> docker.models.volumes.Volume:
"""Return the volume passed in parameter, creating it if it does not
exists.
:param volume_name: name of the volume to create
:param extra_labels: extra labels to put on the volume (only on
creation)
"""
volumes = cls.client.volumes.list(filters={"name": volume_name})
if volumes:
_logger.debug("Volume %s already exists", volume_name)
return volumes[0]
else:
_logger.debug("Creating volume %s", volume_name)
labels = dict(odoo_scripts="")
if extra_labels:
labels.update(extra_labels)
return cls.client.volumes.create(name=volume_name, labels=labels)
@staticmethod
def put_file(src, container, dst):
# copied from https://stackoverflow.com/questions/46390309/how-to-copy-a-file-from-host-to-container-using-docker-py-docker-sdk # noqa: E501
os.chdir(os.path.dirname(src))
srcname = os.path.basename(src)
# TODO put that in a temporary directory
tar = tarfile.open(src + ".tar", mode="w")
try:
tar.add(srcname)
finally:
tar.close()
data = open(src + ".tar", "rb").read()
container.put_archive(os.path.dirname(dst), data)
This diff is collapsed.
#!/usr/bin/env python3
# vim: set shiftwidth=4 softtabstop=4:
"""Run flake8 in a docker
"""
import atexit
import logging
import os
import sys
from .config import Config
from .docker_client import DockerClient
from .parsing import apply, basic_parser
__version__ = "1.0.0"
__date__ = "2020-09-10"
__updated__ = "2020-09-10"
_logger = logging.getLogger(__name__)
def __parser():
program_version = __version__
program_build_date = str(__updated__)
program_version_message = "%%(prog)s %s (%s)" % (
program_version,
program_build_date,
)
program_shortdesc = __doc__.split(".")[0]
program_license = """%s
Created by Vincent Hatakeyama on %s.
Copyright 2020 XCG Consulting. All rights reserved.
Licensed under the MIT License
Distributed on an "AS IS" basis without warranties
or conditions of any kind, either express or implied.
USAGE
""" % (
program_shortdesc,
str(__date__),
)
parser = basic_parser(program_license, program_version_message)
return parser
def main(argv=None):
"""Copy modules for a build, callable version that parses arguments
"""
parser = __parser()
nmspc = parser.parse_args(argv)
apply(nmspc)
c = Config()
odoo_type = c.odoo_type
return flake8(odoo_type)
def flake8(odoo_type):
"""Run flake8
"""
client = DockerClient.client
repository = "xcgd/flake8"
if odoo_type in ("odoo7", "odoo8", "odoo9", "odoo10"):
tag = "2"
_logger.info("Running flake8 for Python 2")
else:
tag = "3"
_logger.info("Running flake8 for Python 3")
client.images.pull(repository, tag)
image = "{}:{}".format(repository, tag)
container = None
# XXX When pressing ctrl-c during the container creating, the container
# is not removed as there is no return of the value of the Container
# object. There is several ways to do that:
# - give the container a unique name we know, and use that to clean
# - maybe use some kind of time frame to identify the container
# - warn the user of a potential container leak by using signals
def stop_remove():
"""Stop then remove the container."""
if container:
if container.status == "running":
container.stop()
container.wait()
container.remove()
atexit.register(stop_remove)
# split create and start so in case of SIGTERM, container has fewer chances
# of being None
container = client.containers.create(
image,
command=["."],
volumes={os.environ["PWD"]: {"bind": "/mnt", "mode": "ro"}},
working_dir="/mnt",
detach=True,
)
container.start()
result = container.wait()
atexit.unregister(stop_remove)
atexit.register(container.remove)
print(container.logs().decode().rstrip())
if result.get("Error", None):
_logger.warning(result.decode())
container.remove(v=True)
atexit.unregister(container.remove)
return result["StatusCode"]
if __name__ == "__main__":
return_code = main(sys.argv[1:])
if return_code:
exit(return_code)
#!/usr/bin/env python3
# vim: set shiftwidth=4 softtabstop=4:
"""Run isort in a docker
"""
import atexit
import logging
import os
import sys
import docker
from .docker_client import DockerClient
from .parsing import apply, basic_parser
__version__ = "1.0.0"
__date__ = "2020-09-10"
__updated__ = "2020-09-10"
_logger = logging.getLogger(__name__)
def __parser():
program_version = __version__
program_build_date = str(__updated__)
program_version_message = "%%(prog)s %s (%s)" % (
program_version,
program_build_date,
)
program_shortdesc = __doc__.split(".")[0]
program_license = """%s
Created by Vincent Hatakeyama on %s.
Copyright 2020 XCG Consulting. All rights reserved.
Licensed under the MIT License
Distributed on an "AS IS" basis without warranties
or conditions of any kind, either express or implied.
USAGE
""" % (
program_shortdesc,
str(__date__),
)
parser = basic_parser(program_license, program_version_message)
return parser
def main(argv=None):
"""Copy modules for a build, callable version that parses arguments
"""
parser = __parser()
nmspc = parser.parse_args(argv)
apply(nmspc)
isort()
return 0
def isort(pull=True):
"""Run isort
"""
client = DockerClient.client
if pull:
_logger.info("Pulling isort")
# TODO find a better way to avoid error when pulling without a
# connection
try:
client.images.pull(repository="xcgd/isort", tag="odoo")
except docker.errors.APIError:
_logger.warning(
"No connection to server, using existing version, if any"
)
_logger.info("Running isort for Odoo")
container = None
# XXX When pressing ctrl-c during the container creating, the container
# is not removed as there is no return of the value of the Container
# object. There is several ways to do that:
# - give the container a unique name we know, and use that to clean
# - maybe use some kind of time frame to identify the container
# - warn the user of a potential container leak by using signals
def stop_remove():
"""Stop then remove the container."""
if container:
if container.status == "running":
container.stop()
container.wait()
container.remove()
atexit.register(stop_remove)
# split create and start so in case of SIGTERM, container has fewer chances
# of being None
container = client.containers.create(
"xcgd/isort:odoo",
command=["isort", "--check"],
volumes={os.environ["PWD"]: {"bind": "/mnt", "mode": "ro"}},
working_dir="/mnt",
detach=True,
)
container.start()
result = container.wait()
atexit.unregister(stop_remove)
atexit.register(container.remove)
print(container.logs().decode().rstrip())
if result.get("Error", None):
_logger.warning(result.decode())
container.remove(v=True)
atexit.unregister(container.remove)
return result["StatusCode"]
if __name__ == "__main__":
return_code = main(sys.argv[1:])
if return_code:
exit(return_code)
# vim: set shiftwidth=4 softtabstop=4:
import atexit
import logging
import tempfile
import time
from typing import Callable, Tuple
import docker
from .docker_client import DockerClient
_logger = logging.getLogger(__name__)
POSTGRES_PASSWORD = "this-could-be-anything"
def docker_run_postgresql(
project_name: str,
postgresql_version: str,
host_pg_port=None,
stop_at_exit: bool = True,
) -> Tuple[docker.models.containers.Container, Callable, str]:
"""
:param project_name:
:param postgresql_version:
:param host_pg_port: if None, put the socket in a temp directory,
otherwise publish the port
:param stop_at_exit: True to stop the container at exit
:return: container, method to call to stop it, socket directory
"""
docker_client = DockerClient.client
pg_repository = "postgres"
version = "{}-alpine".format(postgresql_version)
pg_image = "{}:{}".format(pg_repository, version)
name = "postgresql-{}-{}".format(postgresql_version, project_name)
# need to pull otherwise the containers.create might fail
docker_client.images.pull(repository=pg_repository, tag=version)
pg_data_volume_name = "postgresql_{}-{}".format(
postgresql_version, project_name
)
path: str = None
env = dict()
env["POSTGRES_PASSWORD"] = POSTGRES_PASSWORD
def stop_postgresql(pg_container: docker.models.containers.Container):
# TODO test if still exists
# maybe by looking at pg.status
# TODO factorized with other methods that start dockers
try:
_logger.info("Stopping postgresql")
# Need to stop too?
pg_container.stop()
pg_container.wait()
_logger.info("Removing container postgresql")
pg_container.remove()
except docker.errors.NotFound:
_logger.info("Postgresql already stopped")
# crash because docker or pg changes the owner of the directory
# temp_directory.cleanup()
for container in docker_client.containers.list():
if name == container.name:
_logger.info("Postgresql Container already running")
source = None
for mount_dict in container.attrs["Mounts"]:
if mount_dict["Destination"] == "/var/run/postgresql":
source = mount_dict["Source"]
return container, lambda: stop_postgresql(container), source
volumes = dict()
volumes[pg_data_volume_name] = {
"bind": "/var/lib/postgresql/data",
"mode": "rw",
}
port_bindings = dict()
if host_pg_port:
port_bindings[5432] = host_pg_port
else:
path = tempfile.TemporaryDirectory(
prefix="odoo_scripts_postgres_socket-"
).name
volumes[path] = {"bind": "/var/run/postgresql", "mode": "rw"}
_logger.debug("Creating postgresql container")
pg = docker_client.containers.create(
pg_image,
volumes=volumes,
ports=port_bindings,
name=name,
environment=env,
)
_logger.debug("Starting postgresql container")
pg.start()
def stop_pg():
return stop_postgresql(pg)
if stop_at_exit:
atexit.register(stop_pg)
_logger.debug("Waiting for postgres to start")
# give pg the time to start up
time.sleep(5)
return pg, stop_pg, path
# vim: set shiftwidth=4 softtabstop=4:
"""Py3o methods
"""
import atexit
import logging
from .docker_client import DockerClient
_logger = logging.getLogger(__name__)
# TODO move to docker_client directly
def find_container(name):
"""Return container object from its name
"""
for container in DockerClient.client.containers():
if name == container.names:
return container
def remove_and_stop(docker_client, name):
# TODO handle the case when the container has been created, not running
tostop = find_container(name)
docker_client.stop(tostop.get("Id"))
docker_client.remove_container(tostop.get("Id"))
def start_py3o(host_fusion_port=8765, stop_at_exit=True):
"""start a py3o container"""
docker_client = DockerClient.client
fusion_repository = "xcgd/py3o"
fusion_version = "1.0.0"
fusion_name = "py3o_fusion"
fusion_image = "{}:{}".format(fusion_repository, fusion_version)
try:
docker_client.pull(repository=fusion_repository, tag=fusion_version)
except Exception as e:
_logger.warning("Exception when trying to pull: %s", e)
if any(
"/{}".format(fusion_name) in container["Names"]
for container in docker_client.containers()
):
_logger.debug("%s Container already running", fusion_name)
remove_and_stop(docker_client, fusion_name)
# TODO handle --host-network option
port_bindings = {8765: host_fusion_port}
host_config = docker_client.create_host_config(
binds=[], port_bindings=port_bindings
)
_logger.debug("Starting %s container", fusion_name)
fusion = docker_client.create_container(
image=fusion_image, host_config=host_config, name=fusion_name
)
_logger.debug("Starting %s container", fusion_name)
docker_client.start(container=fusion.get("Id"))
def stop_py3o():
# TODO test if still exists
_logger.info("Stopping fusion")
docker_client.stop(fusion.get("Id"))
_logger.info("Removing containers")
docker_client.remove_container(fusion.get("Id"))
if stop_at_exit:
atexit.register(stop_py3o)
return fusion_name, stop_py3o
...@@ -16,7 +16,7 @@ ...@@ -16,7 +16,7 @@
add_importing_file_parsing, add_importing_file_parsing,
extract_model_lang_from_parsed, extract_model_lang_from_parsed,
) )
from .parsing import logging_from_verbose from .parsing import apply
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
...@@ -159,7 +159,7 @@ ...@@ -159,7 +159,7 @@
add_importing_file_parsing(parser) add_importing_file_parsing(parser)
nmspc = parser.parse_args(argv) nmspc = parser.parse_args(argv)
logging_from_verbose(nmspc) apply(nmspc)
return import_with_base_import( return import_with_base_import(
login=nmspc.login, login=nmspc.login,
......
...@@ -17,7 +17,7 @@ ...@@ -17,7 +17,7 @@
extract_model_lang_from_parsed, extract_model_lang_from_parsed,
) )
from .odoo import odoo_connect_parser, odoo_login from .odoo import odoo_connect_parser, odoo_login
from .parsing import logging_from_verbose from .parsing import apply
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
...@@ -21,5 +21,5 @@ ...@@ -21,5 +21,5 @@
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
__version__ = "1.0.1" __version__ = "1.0.2"
__date__ = "2020-02-17" __date__ = "2020-02-17"
...@@ -25,5 +25,5 @@ ...@@ -25,5 +25,5 @@
__date__ = "2020-02-17" __date__ = "2020-02-17"
__updated__ = "2020-02-26" __updated__ = "2020-09-11"
def import_with_jsonrpc( def import_with_jsonrpc(
...@@ -314,7 +314,7 @@ ...@@ -314,7 +314,7 @@
) )
nmspc = parser.parse_args(argv) nmspc = parser.parse_args(argv)
logging_from_verbose(nmspc) apply(nmspc)
extra_context = dict() extra_context = dict()
if nmspc.context: if nmspc.context:
......
...@@ -212,6 +212,7 @@ ...@@ -212,6 +212,7 @@
delimiter=nmspc.delimiter, delimiter=nmspc.delimiter,
) )
conn.close() conn.close()
return 0
if __name__ == "__main__": if __name__ == "__main__":
......
...@@ -9,9 +9,9 @@ ...@@ -9,9 +9,9 @@
import sys import sys
from .config import Config from .config import Config
from .parsing import basic_parser, logging_from_verbose from .parsing import apply, basic_parser
MODULES_LIST_FILE = "odoo_modules_list" MODULES_LIST_FILE = "odoo_modules_list"
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
...@@ -13,7 +13,7 @@ ...@@ -13,7 +13,7 @@
MODULES_LIST_FILE = "odoo_modules_list" MODULES_LIST_FILE = "odoo_modules_list"
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
__version__ = "1.0.2" __version__ = "1.0.3"
__date__ = "2020-02-26" __date__ = "2020-02-26"
...@@ -19,5 +19,5 @@ ...@@ -19,5 +19,5 @@
__date__ = "2020-02-26" __date__ = "2020-02-26"
__updated__ = "2020-06-18" __updated__ = "2020-09-11"
def main(argv=None): # IGNORE:C0111 def main(argv=None): # IGNORE:C0111
...@@ -53,7 +53,7 @@ ...@@ -53,7 +53,7 @@
# Argument parsing # Argument parsing
parser = basic_parser(program_license, program_version_message) parser = basic_parser(program_license, program_version_message)
nmspc = parser.parse_args() nmspc = parser.parse_args()
logging_from_verbose(nmspc) apply(nmspc)
list_modules() list_modules()
......
...@@ -8,8 +8,9 @@ ...@@ -8,8 +8,9 @@
def add_verbosity_to_parser(parser: argparse.ArgumentParser): def add_verbosity_to_parser(parser: argparse.ArgumentParser):
parser.add_argument( verbosity_group = parser.add_mutually_exclusive_group()
verbosity_group.add_argument(
"-v", "-v",
"--verbose", "--verbose",
dest="verbose", dest="verbose",
action="count", action="count",
...@@ -12,9 +13,18 @@ ...@@ -12,9 +13,18 @@
"-v", "-v",
"--verbose", "--verbose",
dest="verbose", dest="verbose",
action="count", action="count",
help="set verbosity level [default: %(default)s]", help="Increase verbosity level [default: %(default)s]",
default=0,
)
verbosity_group.add_argument(
"-q",
"--quiet",
dest="quiet",
action="count",
help="Decreate verbosity level [default: %(default)s]",
default=0,
) )
def logging_from_verbose(namespace: argparse.Namespace): def logging_from_verbose(namespace: argparse.Namespace):
...@@ -17,9 +27,7 @@ ...@@ -17,9 +27,7 @@
) )
def logging_from_verbose(namespace: argparse.Namespace): def logging_from_verbose(namespace: argparse.Namespace):
verbose = namespace.verbose verbosity = namespace.verbose - namespace.quiet
if not verbose: if verbosity == 0:
level = logging.WARN
elif verbose == 1:
level = logging.INFO level = logging.INFO
...@@ -25,5 +33,11 @@ ...@@ -25,5 +33,11 @@
level = logging.INFO level = logging.INFO
else: elif verbosity == -1:
level = logging.WARNING
elif verbosity == -2:
level = logging.ERROR
elif verbosity == -3:
level = logging.CRITICAL
elif verbosity > 0:
level = logging.DEBUG level = logging.DEBUG
coloredlogs.install( coloredlogs.install(
level, fmt="%(asctime)s %(levelname)8s %(name)s %(message)s" level, fmt="%(asctime)s %(levelname)8s %(name)s %(message)s"
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment