# HG changeset patch # User Vincent Hatakeyama <vincent.hatakeyama@xcg-consulting.fr> # Date 1593533682 -7200 # Tue Jun 30 18:14:42 2020 +0200 # Node ID 877a52709131e623a5e05b006dd247c05fb36057 # Parent 5d254b5a59f3a1240ac4e16375f08cb6880ac500 📠convert shell script to python and factorize some code, convert some scripts from zsh to sh 🚀 run shell linters diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,10 +3,20 @@ project: xcg/ci-templates - file: docker-build.gitlab-ci.yaml project: xcg/ci-templates +- file: deploy-doc.gitlab-ci.yaml + project: xcg/ci-templates +- file: shell-lint.gitlab-ci.yaml + project: xcg/ci-templates + ref: topic/default/checkbashisms variables: TAG_LATEST: branch/default DOCKER_IMAGE: xcgd/odoo_scripts + HTML_DOC_SOURCES: doc/_build/html + SH_SCRIPTS: isort create_archive start + +checkbashisms: + allow_failure: true import_jsonrpc_odoo11_test: stage: test @@ -35,24 +45,19 @@ build-documentation: stage: build - image: python:3 + image: python:3-alpine artifacts: paths: - doc/_build expire_in: 10m script: - - pip3 install -r requirements + - apk add make mercurial + # It might not be necessary to install this and the dependency (maybe just reuse the built image? but its done after) + - pip3 install . - cd doc - - pip3 install -r requirements + - pip3 install -r requirements hg-evolve - for language in en fr ; do LANGUAGE=$language BUILDDIRSUFFIX=/$(hg identify --debug --branch) make html ; done -push-documentation: - stage: deploy - image: - name: minio/mc:RELEASE.2020-01-13T22-49-03Z - entrypoint: ["/bin/busybox"] - script: - - mc config host add s3 $S3_ENDPOINT_URL $S3_KEY $S3_SECRET --api S3v2 - - mc cp --recursive doc/_build/html/ s3/xcg-io-doc/odoo_scripts +publish_documentation: only: - /^branch\/.*/ diff --git a/Dockerfile b/Dockerfile --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,21 @@ FROM python:3-alpine +ARG BUILD_DATE="" +ARG VCS_URL="" +ARG VCS_REF="" +ARG VERSION="" +LABEL org.opencontainers.image.revision=$VERSION +LABEL org.opencontainers.image.version=$VCS_REF +LABEL org.opencontainers.image.created=$BUILD_DATE +LABEL org.opencontainers.image.title="Odoo scripts" +LABEL org.opencontainers.image.source=$VCS_URL ADD . /usr/src/odoo_scripts RUN set -x ;\ apk add --no-cache --update zsh rsync postgresql-libs && \ + # mercurial && \ apk add --no-cache --virtual .build-deps gcc musl-dev postgresql-dev && \ - python3 -m pip install /usr/src/odoo_scripts --no-cache-dir && \ + python3 -m pip install /usr/src/odoo_scripts[import_sql,conf2reST,source_control] && \ + # hg-evolve --no-cache-dir && \ apk --purge del .build-deps && \ +# mkdir -p /etc/mercurial/hgrc.d && \ +# mv /usr/src/odoo_scripts/evolve.rc /etc/mercurial/hgrc.d/ && \ rm -rf /usr/src/odoo_scripts diff --git a/NEWS.rst b/NEWS.rst --- a/NEWS.rst +++ b/NEWS.rst @@ -2,6 +2,11 @@ History ======= +3.2 +--- + +Changed several shell scripts to python 3 scripts. + 3.1 --- diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -4,7 +4,7 @@ Most commands can be run directly or with the image build from ``Dockerfile`` (xcgd/odoo_scripts) by indicating the command to run. -Shell scripts +Shell Scripts ============= Those scripts are not installed when installing this python module. @@ -89,7 +89,9 @@ Those scripts are available when installing the package, and can also be run directly. -The recommend way to install this module is to run ``pip3 install -e .``. +The recommend way to install this module is to run ``pip3 install .``, eventually with the ``editable`` option. +Some scripts are in other sections, because they need some specialized library, in that case you need to indicate the name of section as in ``pip3 install ".[docker]"``. +The prerequisites for this module or one of its section can be installed by using pip3 or the package manager; the requirements are defined in ``setup.py``. conf2reST.py ------------ @@ -97,9 +99,10 @@ This script is used to produce a reST file from setup.cfg, the Dockerfile and the ``.hgconf``/``nest.yaml`` file. When the package is installed, the executable for this script is ``conf2reST``. +This is part of the conf2reST section. -update_duplicate_sources ------------------------- +update_duplicate_sources.py +--------------------------- This script will update a metaproject that only contain the sources of the one it is launched from. @@ -111,27 +114,25 @@ No change are made to the name of the directories/files included. +When the package is installed, the executable for this script is ``update_duplicate_sources``. +This is part of the source_control section. + docker_dev_start.py ------------------- -This script can be used to start an odoo from a docker but with the local addons modules mounted. +This script can be used to start an odoo from a docker but with the local addons modules mounted, and eventually a local copy of the odoo sources mounted too. Using it avoids having to create a virtual env for every project. -Before you use this script, you need to install these packages: - -.. code-block:: sh - - apt install python3-docker python3-psycopg2 python3-requests - Your user also needs to be in the docker group. -It expect the configuration file (`setup.cfg`_) to have the following keys: +It expects the configuration file (`setup.cfg`_) to have the following keys: - ``modules``: list of directories and files to include - … When the package is installed, the executable for this script is ``docker_dev_start``. +This is part of the docker section. docker_build.py --------------- @@ -141,6 +142,7 @@ It uses the super project’s `setup.cfg`_ and ``Dockerfile``. When the package is installed, the executable for this script is ``docker_build``. +This is part of the docker section. do_tests.py ----------- @@ -151,7 +153,7 @@ import_base_import and import_jsonrpc --------------------------------------- +------------------------------------- Import CSV files into an odoo. @@ -182,6 +184,7 @@ When inserting, the create_uid/write_uid are not set. create_date and write_date is updated as needed, and so is the date_update in ``ir.model.data``. +This is part of the import_sql section. setup.cfg ========= @@ -219,13 +222,8 @@ Generate completion file for python scripts (from the superproject for the scripts that need to be run from there, and with the required requirements too):: - docker_dev_start --help | ~/src/zsh-completion-generator/help2comp.py docker_dev_start > ~/.local/share/zsh/completion/_docker_dev_start - do_tests --help | ~/src/zsh-completion-generator/help2comp.py do_tests > ~/.local/share/zsh/completion/_do_tests - conf2reST --help | ~/src/zsh-completion-generator/help2comp.py conf2reST > ~/.local/share/zsh/completion/_conf2reST - import_base_import --help | ~/src/zsh-completion-generator/help2comp.py import_base_import > ~/.local/share/zsh/completion/_import_base_import - import_jsonrpc --help | ~/src/zsh-completion-generator/help2comp.py import_jsonrpc > ~/.local/share/zsh/completion/_import_jsonrpc - import_sql --help | ~/src/zsh-completion-generator/help2comp.py import_sql > ~/.local/share/zsh/completion/_import_sql - + for command_name in docker_dev_start docker_build docker_build_copy docker_build_clean do_tests conf2reST import_base_import import_jsonrpc import_sql ; do + $command_name --help | ~/src/zsh-completion-generator/help2comp.py $command_name > ~/.local/share/zsh/completion/_$command_name + done Alternatives: genzshcomp - diff --git a/create_archive b/create_archive --- a/create_archive +++ b/create_archive @@ -1,4 +1,4 @@ -#!/bin/zsh +#!/bin/sh # vim: set shiftwidth=4 softtabstop=4: # Create archive of project sources @@ -10,46 +10,44 @@ odoo_dir=$1 tar_file=$2 -here=$(dirname $0) -project_home=$(cd $here && cd .. && echo $PWD) -project_name=$(basename $project_home) +here=$(dirname "$0") +project_home=$(cd "$here" && cd .. && echo "$PWD") +project_name=$(basename "$project_home") python=python odoo_modules="$($python -B -c "import ConfigParser ; c = ConfigParser.ConfigParser() ; c.read('${project_home}/setup.cfg') ; print(' '.join(c.get('odoo_scripts', 'modules', '').split()))")" dependencies="$($python -B -c "import ConfigParser ; c = ConfigParser.ConfigParser() ; c.read('${project_home}/setup.cfg') ; print(c.has_section('odoo_scripts') and c.has_option('odoo_scripts', 'dependencies') and' '.join(c.get('odoo_scripts', 'dependencies', 'dependencies').split()) or 'dependencies')")" -if [[ -z $tar_file ]]; +if [ -z "$tar_file" ]; then tar_file=${project_name}.tar fi -pushd $project_home +cd "$project_home" || exit 1 # Create empty tar -tar cf ${project_name}.tar --files-from /dev/null +tar cf "${project_name}.tar" --files-from /dev/null # Add modules, in odoo_modules directory (whatever the original directory name) -for module in $(eval echo $odoo_modules) ; +for module in $(eval echo "$odoo_modules") ; do - tar uf $tar_file --transform="s|^$(dirname $module)|odoo_modules|" $module --show-transform --exclude-vcs --exclude-backups --exclude='*.pyc' --exclude='.drone.yml' + tar uf "$tar_file" --transform="s|^$(dirname "$module")|odoo_modules|" "$module" --show-transform --exclude-vcs --exclude-backups --exclude='*.pyc' --exclude='.drone.yml' --exclude='.gitlab-ci.yml' done # Add dependencies, in dependencies directory (whatever the original directory name) -for dep in $(eval echo $dependencies) ; +for dep in $(eval echo "$dependencies") ; do - tar uf $tar_file --transform="s|^$(dirname $dep)|dependencies|" $dep --exclude-vcs --exclude-backups --exclude='*.pyc' --exclude='.drone.yml' + tar uf "$tar_file" --transform="s|^$(dirname "$dep")|dependencies|" "$dep" --exclude-vcs --exclude-backups --exclude='*.pyc' --exclude='.drone.yml' --exclude='.gitlab-ci.yml' done # Add version number file if present -if [[ -f VERSION ]]; +if [ -f VERSION ]; then - tar uf $tar_file VERSION + tar uf "tar_file" VERSION fi -if [[ -d $odoo_dir ]]; +if [ -d "$odoo_dir" ]; then - tar uf $tar_file --transform="s|^$(echo $odoo_dir | sed -e "s|/\(.*\)|\1|")|odoo|" $odoo_dir --exclude-vcs --exclude-backups --exclude='*.pyc' + tar uf "$tar_file" --transform="s|^$(echo "$odoo_dir" | sed -e 's|/\(.*\)|\1|')|odoo|" "$odoo_dir" --exclude-vcs --exclude-backups --exclude='*.pyc' fi # Compress the file -xz $tar_file - -popd +xz "$tar_file" diff --git a/docker_build_copy b/docker_build_copy deleted file mode 100755 --- a/docker_build_copy +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/zsh -# vim: set shiftwidth=4 softtabstop=4: - -# Create docker script (copy) - -# template version 2.7 - -# TODO add a way to bypass the value, maybe with a key in the setup.cfg file -project_home=$PWD -static_dir=$project_home/static/ -odoo_modules="$(python3 -B -c "import configparser ; c = configparser.ConfigParser() ; c.read('${project_home}/setup.cfg') ; print (c.has_option('odoo_scripts', 'modules') and ' '.join(c.get('odoo_scripts', 'modules').split()) or '')")" - -# Copy modules when specified -if [[ -n "$odoo_modules" ]]; -then - if ! [ -x "$(command -v list_modules)" ]; - then - # TODO test that this works - $(dirname $0)/list_modules - else - list_modules - fi - mkdir -p $project_home/odoo_modules - set -e - rsync --delete -C --exclude='.hg*' --exclude='.git*' --links --exclude='*.pyc' -r --times $(eval echo $odoo_modules) $project_home/odoo_modules - # this only sync static from our modules - rsync --include='/*/static/***' --exclude='/*/*' -r --times --prune-empty-dirs $(eval echo $odoo_modules) $static_dir -fi diff --git a/isort b/isort --- a/isort +++ b/isort @@ -1,2 +1,2 @@ #!/bin/sh -/usr/bin/docker run --rm --volume $PWD:/mnt -ti -w /mnt xcgd/isort:odoo isort $* +/usr/bin/docker run --rm --volume "$PWD:/mnt" -ti -w /mnt xcgd/isort:odoo isort "$@" diff --git a/odoo_scripts/conf2reST.py b/odoo_scripts/conf2reST.py --- a/odoo_scripts/conf2reST.py +++ b/odoo_scripts/conf2reST.py @@ -10,7 +10,7 @@ import yaml -from .logging_utils import add_verbosity_to_parser, logging_from_verbose +from .parsing import add_verbosity_to_parser, logging_from_verbose _logger = logging.getLogger(__name__) diff --git a/odoo_scripts/config.py b/odoo_scripts/config.py new file mode 100644 --- /dev/null +++ b/odoo_scripts/config.py @@ -0,0 +1,78 @@ +"""Functions to read configuration. +""" +import configparser +import logging +import os +from glob import glob + +_logger = logging.getLogger(__name__) + + +class Config(object): + """Singleton for the configuration. + At the moment, only read from `setup.cfg`. + """ + + __instance = None + + def __new__(cls): + if Config.__instance is None: + Config.__instance = object.__new__(cls) + return Config.__instance + + def __init__(self): + setup_path = "setup.cfg" + section = "odoo_scripts" + config_parser = configparser.ConfigParser() + if not os.path.exists(setup_path): + _logger.warning("Missing %s", setup_path) + else: + config_parser.read(os.path.join(os.getcwd(), "setup.cfg")) + for key in ("modules", "dependencies", "other_sources"): + key_set = set() + for key_glob in config_parser.get( + section, key, fallback="" + ).split(): + for file in glob(key_glob): + if os.path.isdir(file): + key_set.add(file) + setattr(self, key, list(key_set)) + for key in ("module_list", "module_list_tests"): + setattr( + self, key, config_parser.get(section, key, fallback="").split() + ) + for key in ( + "db_user", + "db_password", + "load-language", + "pg.extensions", + ): + setattr( + self, + key.replace("-", "_").replace(".", "_"), + config_parser.get(section, key, fallback=None), + ) + self.registry = config_parser.get( + section, "registry", fallback="registry.xcg.io" + ) + project_path = os.path.realpath(".") + self.image = config_parser.get( + section, "image", fallback=os.path.basename(project_path) + ) + self.odoo_type = config_parser.get( + section, "odoo_type", fallback="odoo7" + ) + if self.odoo_type not in ( + "odoo7", + "odoo8", + "odoo10", + "odoo11", + "odoo13", + ): + _logger.warning("Unexpected odoo_type: %s", self.odoo_type) + self.postgresql_version = config_parser.get( + section, "postgresql_version", fallback="9.6" + ) + self.start_py3o = config_parser.get( + section, "start_py3o", fallback="no" + ) in ("yes", "true") diff --git a/odoo_scripts/do_tests.py b/odoo_scripts/do_tests.py --- a/odoo_scripts/do_tests.py +++ b/odoo_scripts/do_tests.py @@ -6,16 +6,17 @@ """ # Version 3.1 import argparse +from configparser import ConfigParser import logging import os from subprocess import call import sys import docker # apt python3-docker (1.9) or pip3 install docker -import configparser from psycopg2 import connect, OperationalError # apt python3-psycopg2 from . import docker_dev_start +from .config import Config if docker.__version__ > "2.5.0": from docker import APIClient as docker_api @@ -24,9 +25,9 @@ _logger = logging.getLogger(__name__) -__version__ = "1.0.0" +__version__ = "1.0.1" __date__ = "2018-04-13" -__updated__ = "2020-02-14" +__updated__ = "2020-06-30" def main(argv=None): # IGNORE:C0111 @@ -66,7 +67,6 @@ ) return 1 - setup_path = "setup.cfg" # TODO add a way to store configuration options in a project file # Argument parsing parser = argparse.ArgumentParser( @@ -192,52 +192,24 @@ # Get parameters from setup file - c = configparser.ConfigParser() - if not os.path.exists(setup_path): - _logger.fatal("Missing %s", setup_path) - return 12 - c.read(setup_path) + config = Config() + registry = config.registry + project = config.image + languages = config.load_language + postgresql_version = config.postgresql_version + odoo_type = config.odoo_type # TODO factorize with docker_dev_start - registry = ( - c.has_section("odoo_scripts") - and c.has_option("odoo_scripts", "registry") - and c.get("odoo_scripts", "registry") - ) or "registry.xcg.io" - project = ( - c.has_section("odoo_scripts") - and c.has_option("odoo_scripts", "image") - and c.get("odoo_scripts", "image") - ) or os.path.basename(project_path) - languages = ( - c.has_section("odoo_scripts") - and c.has_option("odoo_scripts", "load-language") - and c.get("odoo_scripts", "load-language") - ) or None - postgresql_version = ( - c.has_section("odoo_scripts") - and c.has_option("odoo_scripts", "postgresql_version") - and c.get("odoo_scripts", "postgresql_version") - ) or "9.6" - odoo_type = ( - c.has_section("odoo_scripts") - and c.has_option("odoo_scripts", "odoo_type") - and c.get("odoo_scripts", "odoo_type") - ) or "odoo7" image = "%s/%s:latest" % (registry, project) _logger.debug("Docker image: %s", image) - extensions = ( - c.has_section("odoo_scripts") - and c.has_option("odoo_scripts", "pg.extensions") - and c.get("odoo_scripts", "pg.extensions").split() - or [] - ) + extensions = config.pg_extensions # read from odoo.conf if it exists sample_conf = "conf/dev/odoo.conf" if os.path.exists(sample_conf): _logger.info("Reading from sample configuration %s", sample_conf) + c = ConfigParser() c.read(sample_conf) if not odoo_db_user: if c.has_option("options", "db_user"): @@ -257,18 +229,16 @@ if start_postgresql: odoo_db_user = "odoo" else: - if not odoo_db_user and c.has_option("odoo_scripts", "db_user"): - odoo_db_user = c.get("odoo_scripts", "db_user") + if not odoo_db_user and config.db_user is not None: + odoo_db_user = config.db_user if not odoo_db_user: _logger.warning("No database user found or given") if not odoo_db_password: if start_postgresql: odoo_db_password = "odoo" else: - if not odoo_db_password and c.has_option( - "odoo_scripts", "db_password" - ): - odoo_db_password = c.get("odoo_scripts", "db_password") + if not odoo_db_password and config.db_password: + odoo_db_password = config.db_password if not odoo_db_password: _logger.warning("No database password found or given") diff --git a/odoo_scripts/docker_build.py b/odoo_scripts/docker_build.py --- a/odoo_scripts/docker_build.py +++ b/odoo_scripts/docker_build.py @@ -1,40 +1,38 @@ #!/usr/bin/env python3 # vim: set shiftwidth=4 softtabstop=4: """Script to locally build a docker image - """ # Version 3.1 -import argparse -import configparser import datetime import json import logging import os -import shutil import signal import sys from subprocess import call, check_output import docker # apt python3-docker (1.9) or pip3 install docker -from .list_modules import MODULES_LIST_FILE -from .logging_utils import add_verbosity_to_parser, logging_from_verbose +from .config import Config +from .docker_build_clean import clean +from .docker_build_copy import copy +from .parsing import basic_parser, logging_from_verbose +# TODO change requirements to increase docker version to avoid that if docker.__version__ > "2.5.0": from docker import APIClient as docker_api else: from docker import Client as docker_api + _logger = logging.getLogger(__name__) -__version__ = "0.1.3" +__version__ = "0.2.0" __date__ = "2018-04-04" -__updated__ = "2020-03-09" +__updated__ = "2020-06-30" -def main(argv=None): # IGNORE:C0111 - """Parse arguments and docker build - """ +def __parser(): program_version = __version__ program_build_date = str(__updated__) program_version_message = "%%(prog)s %s (%s)" % ( @@ -44,41 +42,36 @@ program_shortdesc = __doc__.split(".")[0] program_license = """%s - Created by Vincent Hatakeyama on %s. - Copyright 2018, 2020 XCG Consulting. All rights reserved. + Created by Vincent Hatakeyama on %s. + Copyright 2018, 2020 XCG Consulting. All rights reserved. - Licensed under the MIT License + Licensed under the MIT License - Distributed on an "AS IS" basis without warranties - or conditions of any kind, either express or implied. + Distributed on an "AS IS" basis without warranties + or conditions of any kind, either express or implied. -USAGE -""" % ( + USAGE + """ % ( program_shortdesc, str(__date__), ) + parser = basic_parser(program_license, program_version_message) + return parser + +def main(argv=None): # IGNORE:C0111 + """Parse arguments and docker build + """ # TODO the script assume it is launched from the parent super project project_path = os.path.realpath(".") project_name = os.path.basename(project_path) if project_name == "odoo_scripts": - logging.fatal( - "You must run this script from the super project" - " (./odoo_scripts/docker_build.py)" - ) - return + logging.fatal("You must run this script from the super project") + return 1 - setup_path = "setup.cfg" # TODO add a way to store configuration options in a project file # Argument parsing - parser = argparse.ArgumentParser( - description=program_license, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument( - "-V", "--version", action="version", version=program_version_message - ) - add_verbosity_to_parser(parser) + parser = __parser() # TODO add tag option, and maybe force tag option parser.add_argument( "--ensureconf", @@ -116,42 +109,22 @@ dev = nmspc.dev push = nmspc.push - c = configparser.ConfigParser() - if not os.path.exists(setup_path): - logging.fatal("Missing %s", setup_path) - return 12 - c.read(setup_path) - - registry = ( - c.has_section("odoo_scripts") - and c.has_option("odoo_scripts", "registry") - and c.get("odoo_scripts", "registry") - ) or "registry.xcg.io" - project = ( - c.has_section("odoo_scripts") - and c.has_option("odoo_scripts", "image") - and c.get("odoo_scripts", "image") - ) or os.path.basename(project_path) - odoo_type = ( - c.has_section("odoo_scripts") - and c.has_option("odoo_scripts", "odoo_type") - and c.get("odoo_scripts", "odoo_type") - ) or "odoo7" + c = Config() + registry = c.registry + project = c.image + odoo_type = c.odoo_type image = "%s/%s:latest" % (registry, project) logging.debug("Docker image: %s", image) # TODO ensureconf if ensureconf: raise NotImplementedError # call build copy - # TODO handle the case when odoo_scripts is not installed - cmd = ["docker_build_copy"] - logging.debug(" ".join(cmd)) - call(cmd) + copy() # clean on exit def signal_handler(code, frame): # TODO handle the case when odoo_scripts is not installed - __clean() + clean() # XXX needed? # sys.exit(0) @@ -230,27 +203,6 @@ return 0 -def clean(argv=None): - """Clean up after a build - """ - parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter - ) - add_verbosity_to_parser(parser) - nmspc = parser.parse_args(argv) - logging_from_verbose(nmspc) - # TODO check odoo_scripts before deleting anything - __clean() - - -def __clean(): - if os.path.exists(MODULES_LIST_FILE): - os.remove(MODULES_LIST_FILE) - for dir in ("odoo_modules", "static"): - if os.path.exists(dir): - shutil.rmtree(dir) - - if __name__ == "__main__": return_code = main(sys.argv[1:]) if return_code: diff --git a/odoo_scripts/docker_build.py b/odoo_scripts/docker_build_clean.py copy from odoo_scripts/docker_build.py copy to odoo_scripts/docker_build_clean.py --- a/odoo_scripts/docker_build.py +++ b/odoo_scripts/docker_build_clean.py @@ -1,40 +1,23 @@ #!/usr/bin/env python3 # vim: set shiftwidth=4 softtabstop=4: -"""Script to locally build a docker image - +"""Tools to help build an image """ -# Version 3.1 -import argparse -import configparser -import datetime -import json import logging import os import shutil -import signal import sys -from subprocess import call, check_output - -import docker # apt python3-docker (1.9) or pip3 install docker from .list_modules import MODULES_LIST_FILE -from .logging_utils import add_verbosity_to_parser, logging_from_verbose - -if docker.__version__ > "2.5.0": - from docker import APIClient as docker_api -else: - from docker import Client as docker_api +from .parsing import basic_parser, logging_from_verbose _logger = logging.getLogger(__name__) -__version__ = "0.1.3" -__date__ = "2018-04-04" -__updated__ = "2020-03-09" +__version__ = "1.0.0" +__date__ = "2020-06-30" +__updated__ = "2020-06-30" -def main(argv=None): # IGNORE:C0111 - """Parse arguments and docker build - """ +def __parser(): program_version = __version__ program_build_date = str(__updated__) program_version_message = "%%(prog)s %s (%s)" % ( @@ -44,211 +27,42 @@ program_shortdesc = __doc__.split(".")[0] program_license = """%s - Created by Vincent Hatakeyama on %s. - Copyright 2018, 2020 XCG Consulting. All rights reserved. + Created by Vincent Hatakeyama on %s. + Copyright 2020 XCG Consulting. All rights reserved. - Licensed under the MIT License + Licensed under the MIT License - Distributed on an "AS IS" basis without warranties - or conditions of any kind, either express or implied. + Distributed on an "AS IS" basis without warranties + or conditions of any kind, either express or implied. -USAGE -""" % ( + USAGE + """ % ( program_shortdesc, str(__date__), ) + parser = basic_parser(program_license, program_version_message) + return parser - # TODO the script assume it is launched from the parent super project - project_path = os.path.realpath(".") - project_name = os.path.basename(project_path) - if project_name == "odoo_scripts": - logging.fatal( - "You must run this script from the super project" - " (./odoo_scripts/docker_build.py)" - ) - return - setup_path = "setup.cfg" - # TODO add a way to store configuration options in a project file - # Argument parsing - parser = argparse.ArgumentParser( - description=program_license, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument( - "-V", "--version", action="version", version=program_version_message - ) - add_verbosity_to_parser(parser) - # TODO add tag option, and maybe force tag option - parser.add_argument( - "--ensureconf", - help="ensureconf [default: %(default)s]", - action="store_true", - ) - parser.add_argument( - "--push", help="Push image [default: %(default)s]", action="store_true" - ) - parser.add_argument( - "--dev", - help="add dev feature to generated image [default: %(default)s]", - action="store_true", - ) - parser.add_argument( - "--build-arg", - help="build arg for the image, formated like FOO=BAR " - "[default: %(default)s]", - default=None, - nargs="*", - ) - parser.add_argument( - "--no-pull", - help="indicate to docker to not pull the base image " - "[default: %(default)s]", - action="store_true", - ) - # TODO (maybe) add argument for other build arg - - # TODO detect that user is member of docker group - +def main(argv=None): + """Clean up after a build, callable version that parses arguments + """ + parser = __parser() nmspc = parser.parse_args(argv) logging_from_verbose(nmspc) - ensureconf = nmspc.ensureconf - dev = nmspc.dev - push = nmspc.push - - c = configparser.ConfigParser() - if not os.path.exists(setup_path): - logging.fatal("Missing %s", setup_path) - return 12 - c.read(setup_path) - - registry = ( - c.has_section("odoo_scripts") - and c.has_option("odoo_scripts", "registry") - and c.get("odoo_scripts", "registry") - ) or "registry.xcg.io" - project = ( - c.has_section("odoo_scripts") - and c.has_option("odoo_scripts", "image") - and c.get("odoo_scripts", "image") - ) or os.path.basename(project_path) - odoo_type = ( - c.has_section("odoo_scripts") - and c.has_option("odoo_scripts", "odoo_type") - and c.get("odoo_scripts", "odoo_type") - ) or "odoo7" - image = "%s/%s:latest" % (registry, project) - logging.debug("Docker image: %s", image) - # TODO ensureconf - if ensureconf: - raise NotImplementedError - # call build copy - # TODO handle the case when odoo_scripts is not installed - cmd = ["docker_build_copy"] - logging.debug(" ".join(cmd)) - call(cmd) - - # clean on exit - def signal_handler(code, frame): - # TODO handle the case when odoo_scripts is not installed - __clean() - # XXX needed? - - # sys.exit(0) - - signal.signal(signal.SIGINT, signal_handler) - signal.signal(signal.SIGTERM, signal_handler) - - # TODO docker build - buildargs = dict() - if os.path.exists(".hg"): - buildargs["REVISION"] = ( - check_output("hg identify -i".split()).split()[0].decode("utf-8") - ) - buildargs["CREATED"] = datetime.datetime.now().isoformat() - if nmspc.build_arg: - for arg in nmspc.build_arg: - a = arg.split("=") - buildargs[a[0]] = a[1] - logging.debug("Build args: %s", buildargs) - dockerfile = "Dockerfile" - if dev: - debug_dockerfile = "Dockerfile.debug" - call(["cp", dockerfile, debug_dockerfile]) - - with open(debug_dockerfile, "a") as myfile: - myfile.write("\n# Developer helpers\n" "RUN apt-get update -qq\n") - myfile.write( - "RUN DEBIAN_FRONTEND=noninteractive apt-get install -y -qq " - ) - if odoo_type in ("odoo11", "odoo13"): - myfile.write( - "python3-watchdog python3-ipdb python3-pyinotify\n" - ) - elif odoo_type in ("odoo10",): - myfile.write("python-ipdb\n") - myfile.write( - "RUN pip install watchdog --disable-pip-version-check " - "--system --no-cache-dir --only-binary wheel" - ) - elif odoo_type in ("odoo7", "odoo8"): - myfile.write("python-ipdb\n") - myfile.write("RUN pip install pyinotify") - dockerfile = debug_dockerfile - # TODO remove temp image - - docker_client = docker_api(base_url="unix://var/run/docker.sock") - pull = not nmspc.no_pull - logging.debug("Docker Pull %s", pull) - builder = docker_client.build( - path=".", - rm=True, - pull=pull, - buildargs=buildargs, - tag=image, - dockerfile=dockerfile, - ) - # this is for python docker 1.8-1.9 - # TODO add compatibility with newer python docker - for line in builder: - d = json.loads(line.decode("utf-8")) - if "stream" in d: - logging.info(d["stream"]) - if "errorDetail" in d: - logging.fatal(d["errorDetail"]) - return 1 - if dev: - call(["rm", dockerfile]) - # TODO exit if build failed - - # TODO docker tag with tags/bookmarks (unused so maybe no need) - # TODO docker push (only when asked for) - if push: - raise NotImplementedError - # XXX call cleanup more intelligently - signal_handler(0, None) + # TODO check odoo_scripts before deleting anything + clean() return 0 -def clean(argv=None): +def clean(): """Clean up after a build """ - parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter - ) - add_verbosity_to_parser(parser) - nmspc = parser.parse_args(argv) - logging_from_verbose(nmspc) - # TODO check odoo_scripts before deleting anything - __clean() - - -def __clean(): if os.path.exists(MODULES_LIST_FILE): os.remove(MODULES_LIST_FILE) - for dir in ("odoo_modules", "static"): - if os.path.exists(dir): - shutil.rmtree(dir) + for directory in ("odoo_modules", "static"): + if os.path.exists(directory): + shutil.rmtree(directory) if __name__ == "__main__": diff --git a/odoo_scripts/docker_build.py b/odoo_scripts/docker_build_copy.py copy from odoo_scripts/docker_build.py copy to odoo_scripts/docker_build_copy.py --- a/odoo_scripts/docker_build.py +++ b/odoo_scripts/docker_build_copy.py @@ -1,40 +1,24 @@ #!/usr/bin/env python3 # vim: set shiftwidth=4 softtabstop=4: -"""Script to locally build a docker image - +"""Tools to help build an image """ -# Version 3.1 -import argparse -import configparser -import datetime -import json import logging import os -import shutil -import signal import sys -from subprocess import call, check_output - -import docker # apt python3-docker (1.9) or pip3 install docker +from subprocess import call -from .list_modules import MODULES_LIST_FILE -from .logging_utils import add_verbosity_to_parser, logging_from_verbose - -if docker.__version__ > "2.5.0": - from docker import APIClient as docker_api -else: - from docker import Client as docker_api +from .config import Config +from .list_modules import list_modules +from .parsing import basic_parser, logging_from_verbose _logger = logging.getLogger(__name__) -__version__ = "0.1.3" -__date__ = "2018-04-04" -__updated__ = "2020-03-09" +__version__ = "1.0.0" +__date__ = "2020-06-30" +__updated__ = "2020-06-30" -def main(argv=None): # IGNORE:C0111 - """Parse arguments and docker build - """ +def __parser(): program_version = __version__ program_build_date = str(__updated__) program_version_message = "%%(prog)s %s (%s)" % ( @@ -44,211 +28,80 @@ program_shortdesc = __doc__.split(".")[0] program_license = """%s - Created by Vincent Hatakeyama on %s. - Copyright 2018, 2020 XCG Consulting. All rights reserved. + Created by Vincent Hatakeyama on %s. + Copyright 2020 XCG Consulting. All rights reserved. - Licensed under the MIT License + Licensed under the MIT License - Distributed on an "AS IS" basis without warranties - or conditions of any kind, either express or implied. + Distributed on an "AS IS" basis without warranties + or conditions of any kind, either express or implied. -USAGE -""" % ( + USAGE + """ % ( program_shortdesc, str(__date__), ) + parser = basic_parser(program_license, program_version_message) + return parser - # TODO the script assume it is launched from the parent super project - project_path = os.path.realpath(".") - project_name = os.path.basename(project_path) - if project_name == "odoo_scripts": - logging.fatal( - "You must run this script from the super project" - " (./odoo_scripts/docker_build.py)" - ) - return - setup_path = "setup.cfg" - # TODO add a way to store configuration options in a project file - # Argument parsing - parser = argparse.ArgumentParser( - description=program_license, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument( - "-V", "--version", action="version", version=program_version_message - ) - add_verbosity_to_parser(parser) - # TODO add tag option, and maybe force tag option - parser.add_argument( - "--ensureconf", - help="ensureconf [default: %(default)s]", - action="store_true", - ) - parser.add_argument( - "--push", help="Push image [default: %(default)s]", action="store_true" - ) - parser.add_argument( - "--dev", - help="add dev feature to generated image [default: %(default)s]", - action="store_true", - ) - parser.add_argument( - "--build-arg", - help="build arg for the image, formated like FOO=BAR " - "[default: %(default)s]", - default=None, - nargs="*", - ) - parser.add_argument( - "--no-pull", - help="indicate to docker to not pull the base image " - "[default: %(default)s]", - action="store_true", - ) - # TODO (maybe) add argument for other build arg - - # TODO detect that user is member of docker group - +def main(argv=None): + """Copy modules for a build, callable version that parses arguments + """ + parser = __parser() nmspc = parser.parse_args(argv) logging_from_verbose(nmspc) - ensureconf = nmspc.ensureconf - dev = nmspc.dev - push = nmspc.push - - c = configparser.ConfigParser() - if not os.path.exists(setup_path): - logging.fatal("Missing %s", setup_path) - return 12 - c.read(setup_path) - - registry = ( - c.has_section("odoo_scripts") - and c.has_option("odoo_scripts", "registry") - and c.get("odoo_scripts", "registry") - ) or "registry.xcg.io" - project = ( - c.has_section("odoo_scripts") - and c.has_option("odoo_scripts", "image") - and c.get("odoo_scripts", "image") - ) or os.path.basename(project_path) - odoo_type = ( - c.has_section("odoo_scripts") - and c.has_option("odoo_scripts", "odoo_type") - and c.get("odoo_scripts", "odoo_type") - ) or "odoo7" - image = "%s/%s:latest" % (registry, project) - logging.debug("Docker image: %s", image) - # TODO ensureconf - if ensureconf: - raise NotImplementedError - # call build copy - # TODO handle the case when odoo_scripts is not installed - cmd = ["docker_build_copy"] - logging.debug(" ".join(cmd)) - call(cmd) - - # clean on exit - def signal_handler(code, frame): - # TODO handle the case when odoo_scripts is not installed - __clean() - # XXX needed? - - # sys.exit(0) - - signal.signal(signal.SIGINT, signal_handler) - signal.signal(signal.SIGTERM, signal_handler) - - # TODO docker build - buildargs = dict() - if os.path.exists(".hg"): - buildargs["REVISION"] = ( - check_output("hg identify -i".split()).split()[0].decode("utf-8") - ) - buildargs["CREATED"] = datetime.datetime.now().isoformat() - if nmspc.build_arg: - for arg in nmspc.build_arg: - a = arg.split("=") - buildargs[a[0]] = a[1] - logging.debug("Build args: %s", buildargs) - dockerfile = "Dockerfile" - if dev: - debug_dockerfile = "Dockerfile.debug" - call(["cp", dockerfile, debug_dockerfile]) - - with open(debug_dockerfile, "a") as myfile: - myfile.write("\n# Developer helpers\n" "RUN apt-get update -qq\n") - myfile.write( - "RUN DEBIAN_FRONTEND=noninteractive apt-get install -y -qq " - ) - if odoo_type in ("odoo11", "odoo13"): - myfile.write( - "python3-watchdog python3-ipdb python3-pyinotify\n" - ) - elif odoo_type in ("odoo10",): - myfile.write("python-ipdb\n") - myfile.write( - "RUN pip install watchdog --disable-pip-version-check " - "--system --no-cache-dir --only-binary wheel" - ) - elif odoo_type in ("odoo7", "odoo8"): - myfile.write("python-ipdb\n") - myfile.write("RUN pip install pyinotify") - dockerfile = debug_dockerfile - # TODO remove temp image - - docker_client = docker_api(base_url="unix://var/run/docker.sock") - pull = not nmspc.no_pull - logging.debug("Docker Pull %s", pull) - builder = docker_client.build( - path=".", - rm=True, - pull=pull, - buildargs=buildargs, - tag=image, - dockerfile=dockerfile, - ) - # this is for python docker 1.8-1.9 - # TODO add compatibility with newer python docker - for line in builder: - d = json.loads(line.decode("utf-8")) - if "stream" in d: - logging.info(d["stream"]) - if "errorDetail" in d: - logging.fatal(d["errorDetail"]) - return 1 - if dev: - call(["rm", dockerfile]) - # TODO exit if build failed - - # TODO docker tag with tags/bookmarks (unused so maybe no need) - # TODO docker push (only when asked for) - if push: - raise NotImplementedError - # XXX call cleanup more intelligently - signal_handler(0, None) + copy() return 0 -def clean(argv=None): - """Clean up after a build +def copy(): + """Copy modules for a build """ - parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter + modules = set() + c = Config() + modules = [os.path.realpath(module) for module in c.modules] + # copy files + _logger.info("Copying Odoo modules files to odoo_modules") + cmd = ( + [ + "rsync", + "--delete", + "--cvs-exclude", + "--exclude=.hg*", + "--exclude=.git*", + "--links", + "--exclude='*.pyc'", + "-r", + "--times", + ] + + modules + + ["odoo_modules"] ) - add_verbosity_to_parser(parser) - nmspc = parser.parse_args(argv) - logging_from_verbose(nmspc) - # TODO check odoo_scripts before deleting anything - __clean() - - -def __clean(): - if os.path.exists(MODULES_LIST_FILE): - os.remove(MODULES_LIST_FILE) - for dir in ("odoo_modules", "static"): - if os.path.exists(dir): - shutil.rmtree(dir) + _logger.debug(" ".join(cmd)) + call(cmd) + # this only sync static from our modules + _logger.info("Copying static files to static") + cmd = ( + [ + "rsync", + "--delete", + "--cvs-exclude", + "--exclude=.hg*", + "--exclude=.git*", + "--links", + "--include='/*/static/***'", + "--exclude='/*/*'", + "-r", + "--times", + "--prune-empty-dirs", + ] + + modules + + ["static"] + ) + _logger.debug(" ".join(cmd)) + call(cmd) + list_modules() if __name__ == "__main__": diff --git a/odoo_scripts/docker_dev_start.py b/odoo_scripts/docker_dev_start.py --- a/odoo_scripts/docker_dev_start.py +++ b/odoo_scripts/docker_dev_start.py @@ -9,7 +9,6 @@ import argparse import atexit from configparser import ConfigParser -import glob import logging import os import pwd @@ -19,6 +18,8 @@ import docker # apt python3-docker (1.9) or pip install docker from psycopg2 import connect, OperationalError # apt python3-psycopg2 + +from .config import Config from . import docker_build if docker.__version__ > "2.5.0": @@ -135,7 +136,6 @@ ) return 1 - setup_path = "setup.cfg" # TODO add a way to store configuration options in a project file # Argument parsing parser = argparse.ArgumentParser( @@ -381,64 +381,29 @@ _logger.fatal("No database name given for restore") return 13 - c = ConfigParser() - if not os.path.exists(setup_path): - _logger.fatal("Missing %s", setup_path) - return 12 - c.read(setup_path) + c = Config() - modules = [] - if c.has_option("odoo_scripts", "modules"): - directories_only = list() - for entry in c.get("odoo_scripts", "modules").split(): - for file in glob.glob(entry): - if os.path.isdir(file): - directories_only.append(file) - modules.extend(directories_only) + modules = c.modules _logger.debug("addon modules: %s", ",".join(modules)) - if not db_user and c.has_option("odoo_scripts", "db_user"): - db_user = c.get("odoo_scripts", "db_user") - if not db_password and c.has_option("odoo_scripts", "db_password"): - db_password = c.get("odoo_scripts", "db_password") - if c.has_option("odoo_scripts", "load-language"): - load_language = c.get("odoo_scripts", "load-language") + if not db_user and c.db_user is not None: + db_user = c.db_user + if not db_password and c.db_password is not None: + db_password = c.db_password + if c.load_language is not None: + load_language = c.load_language - registry = ( - c.has_section("odoo_scripts") - and c.has_option("odoo_scripts", "registry") - and c.get("odoo_scripts", "registry") - ) or "registry.xcg.io" - project = ( - c.has_section("odoo_scripts") - and c.has_option("odoo_scripts", "image") - and c.get("odoo_scripts", "image") - ) or os.path.basename(project_path) - odoo_type = ( - c.has_section("odoo_scripts") - and c.has_option("odoo_scripts", "odoo_type") - and c.get("odoo_scripts", "odoo_type") - ) or "odoo7" + registry = c.registry + project = c.image + odoo_type = c.odoo_type image = "%s/%s:latest" % (registry, project) - postgresql_version = ( - c.has_section("odoo_scripts") - and c.has_option("odoo_scripts", "postgresql_version") - and c.get("odoo_scripts", "postgresql_version") - ) or "9.6" - module_list = ( - c.has_section("odoo_scripts") - and c.has_option("odoo_scripts", "module_list") - and c.get("odoo_scripts", "module_list").split() - ) or [] + postgresql_version = c.postgresql_version + module_list = c.module_list if nmspc.start_py3o: _logger.debug("Command line start py3o %d", nmspc.start_py3o) start_py3o_stack = nmspc.start_py3o == 1 else: - start_py3o_stack = ( - c.has_section("odoo_scripts") - and c.has_option("odoo_scripts", "start_py3o") - and c.get("odoo_scripts", "start_py3o") in ("yes", "true") - ) + start_py3o_stack = c.start_py3o _logger.debug("No command line start py3o %d", start_py3o_stack) _logger.debug("Docker image: %s", image) @@ -487,7 +452,7 @@ if nmspc.test or nmspc.stop_after_init: arg.append("--stop-after-init") if nmspc.test_default: - test_modules = c.get("odoo_scripts", "module_list_tests").split() or [] + test_modules = c.module_list_tests str_modules = ",".join(test_modules) arg.append("-u %s" % str_modules) arg.append("--test-enable") diff --git a/odoo_scripts/list_modules.py b/odoo_scripts/list_modules.py --- a/odoo_scripts/list_modules.py +++ b/odoo_scripts/list_modules.py @@ -5,21 +5,19 @@ # # template version 2.7 -import argparse -import configparser import logging -import os import sys -from .logging_utils import add_verbosity_to_parser, logging_from_verbose +from .config import Config +from .parsing import basic_parser, logging_from_verbose MODULES_LIST_FILE = "odoo_modules_list" _logger = logging.getLogger(__name__) -__version__ = "1.0.1" +__version__ = "1.0.2" __date__ = "2020-02-26" -__updated__ = "2020-02-26" +__updated__ = "2020-06-18" def main(argv=None): # IGNORE:C0111 @@ -53,25 +51,15 @@ str(__date__), ) # Argument parsing - parser = argparse.ArgumentParser( - description=program_license, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument( - "-V", "--version", action="version", version=program_version_message - ) - add_verbosity_to_parser(parser) - + parser = basic_parser(program_license, program_version_message) nmspc = parser.parse_args() logging_from_verbose(nmspc) list_modules() def list_modules(filename: str = MODULES_LIST_FILE): - c = configparser.ConfigParser() - c.read(os.path.join(os.getcwd(), "setup.cfg")) with open(filename, "w") as f: - f.write(",".join(c.get("odoo_scripts", "module_list").split())) + f.write(",".join(Config().module_list)) if __name__ == "__main__": diff --git a/odoo_scripts/odoo.py b/odoo_scripts/odoo.py --- a/odoo_scripts/odoo.py +++ b/odoo_scripts/odoo.py @@ -1,4 +1,4 @@ -"""Function to ease connection to an odoo server +"""Function to ease connection to an Odoo server """ import argparse import logging diff --git a/odoo_scripts/update_duplicate_sources.py b/odoo_scripts/update_duplicate_sources.py new file mode 100644 --- /dev/null +++ b/odoo_scripts/update_duplicate_sources.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +# vim: set shiftwidth=4 softtabstop=4: +"""This script update another repository that is a copy of the sources +""" +import logging +import os +import shutil +import sys +from subprocess import call + +import hglib + +from .config import Config +from .parsing import basic_parser, logging_from_verbose + +_logger = logging.getLogger(__name__) + +__version__ = "1.0.0" +__date__ = "2020-06-19" +__updated__ = "2020-06-30" + + +def __parser(): + program_version = __version__ + program_build_date = str(__updated__) + program_version_message = "%%(prog)s %s (%s)" % ( + program_version, + program_build_date, + ) + program_shortdesc = __doc__.split(".")[0] + program_license = """%s + + Created by Vincent Hatakeyama on %s. + Copyright 2018, 2020 XCG Consulting. All rights reserved. + + Licensed under the MIT License + + Distributed on an "AS IS" basis without warranties + or conditions of any kind, either express or implied. + + USAGE + """ % ( + program_shortdesc, + str(__date__), + ) + parser = basic_parser(program_license, program_version_message) + return parser + + +def main(argv=None) -> int: + """Copy modules for a build, callable version that parses arguments + """ + parser = __parser() + nmspc = parser.parse_args(argv) + logging_from_verbose(nmspc) + # XXX probably needs an option to indicate path to use, defaulting to + # default + _update_duplicate_sources() + return 0 + + +def _update_duplicate_sources(**kwargs): + # detect mercurial/git. if git, fail. + if os.path.exists(".git"): + raise Exception("Git source duplication not handled for now") + _update_duplicate_sources_mercurial(**kwargs) + + +def _update_duplicate_sources_mercurial( + directory: str = ".", path: str = "default" +): + """ + :arg directory: path to the base directory + :arg path: path to use (from `.hg/hgrc`) + """ + hg = hglib.open(directory) + b = str.encode + + def u(data: bytes) -> str: + return data.decode() + + # get branch and tags + branch = hg.identify(branch=True).strip() + _logger.info("Current branch %s", u(branch)) + tags = hg.identify(tags=True).split() + if b"tip" in tags: + tags.remove(b"tip") + _logger.info("Current tags %s", ", ".join(u(tag) for tag in tags)) + # find the duplicate repository URL + remote_path = hg.paths(b(path)) + duplicate_path = b("{}_sources".format(u(remote_path))) + _logger.info("Clone path %s", u(duplicate_path)) + # clone the duplicate repository + duplicate_destination = b"sources" + if os.path.exists(duplicate_destination): + raise Exception( + "Destination directory '{}' exists!".format( + u(duplicate_destination) + ) + ) + _logger.info("Cloning in %s", u(duplicate_destination)) + hg.clone(source=duplicate_path, dest=duplicate_destination) + hg_duplicate = hglib.open(duplicate_destination) + # update clone to same branch if it exists. + branch_exists_in_duplicate = False + for branch_name, num, id_ in hg_duplicate.branches(): + if branch_name == branch: + hg_duplicate.update(rev=num) + branch_exists_in_duplicate = True + _logger.info( + "Found branch %s in duplicate (%d, %s)", + u(branch_name), + num, + id_, + ) + break + # TODO otherwise put ourselves after parent branch (that is not done in + # shell script) + if not branch_exists_in_duplicate: + hg_duplicate.update(b"tip") + _logger.info( + "Branch %s not found in duplicate, updating to tip", u(branch) + ) + # rsync content inside the other directory + c = Config() + cmd = ( + [ + "rsync", + "--delete", + "--cvs-exclude", + "--exclude=.hg*", + "--exclude=.git*", + "--exclude='*.pyc'", + "--copy-unsafe-links", + "--recursive", + "--relative", + "--times", + ] + + c.modules + + c.dependencies + + c.other_sources + + [u(duplicate_destination)] + ) + _logger.debug(" ".join(cmd)) + call(cmd) + # create the branch if it does not exists + if not branch_exists_in_duplicate: + hg_duplicate.branch(name=branch) + # commit + _logger.info("Copying files") + hg_duplicate.commit(message=b"Update sources", addremove=True) + # retag + if tags: + _logger.info("Applying tags") + hg_duplicate.tag(tags) + else: + _logger.info("No tag to apply") + # push + _logger.info("Pushing duplicate") + hg_duplicate.push(newbranch=not branch_exists_in_duplicate) + # clean up cloned repository + _logger.info("Cleaning up duplicate sources") + shutil.rmtree(duplicate_destination) + + +if __name__ == "__main__": + return_code = main(sys.argv[1:]) + if return_code: + exit(return_code) diff --git a/requirements b/requirements --- a/requirements +++ b/requirements @@ -1,5 +1,9 @@ -# used in conf2reST.py -PyYAML +# This includes the requirements for all extras odoorpc==0.7.0 requests_toolbelt==0.8.0 python-dateutil>=2.7.0 +PyYAML +docker +psycopg2 +mercurial >=5.2 +python-hglib diff --git a/setup.cfg b/setup.cfg --- a/setup.cfg +++ b/setup.cfg @@ -14,12 +14,12 @@ [bumpversion:file:start] +[bumpversion:file:odoo_scripts/do_tests.py] + [bumpversion:file:odoo_scripts/docker_build.py] [bumpversion:file:odoo_scripts/docker_dev_start.py] -[bumpversion:file:odoo_scripts/do_tests.py] - [bumpversion:file:odoo_scripts/import_base_import.py] [bumpversion:file:odoo_scripts/import_jsonrpc.py] @@ -27,4 +27,3 @@ [bumpversion:file:odoo_scripts/import_sql.py] [bumpversion:file:setup.py] - diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -15,23 +15,33 @@ "odoorpc==0.7.0", "requests_toolbelt==0.8.0", "python-dateutil>=2.7.0", - # How to indicate to install python3-docker? - "psycopg2", # used by import_sql ], + extras_require={ + "conf2reST": ["PyYAML"], + "docker": ["docker"], + "import_sql": ["psycopg2"], + "source_control": [ + # Only mercurial 5.2 support Python 3.5+ + "mercurial >=5.2", + "python-hglib", + ], + }, entry_points={ "console_scripts": [ "import_base_import=odoo_scripts.import_base_import:main", "import_jsonrpc=odoo_scripts.import_jsonrpc:main", - "import_sql=odoo_scripts.import_sql:main", - "docker_dev_start=odoo_scripts.docker_dev_start:main", + "import_sql=odoo_scripts.import_sql:main [import_sql]", + "docker_dev_start=odoo_scripts.docker_dev_start:main [docker]", "do_tests=odoo_scripts.do_tests:main", - "docker_build=odoo_scripts.docker_build:main", - "docker_build_clean=odoo_scripts.docker_build:clean", - "conf2reST=odoo_scripts.conf2reST:main", + "docker_build=odoo_scripts.docker_build:main [docker]", + "docker_build_clean=odoo_scripts.docker_build_clean:main", + "docker_build_copy=odoo_scripts.docker_build_copy:main", + "conf2reST=odoo_scripts.conf2reST:main [conf2reST]", "list_modules=odoo_scripts.list_modules:main", - # TODO add all the other python scripts + "update_duplicate_sources=" + "odoo_scripts.update_duplicate_sources:main [source_control]", ] }, - scripts=["docker_build_copy", "update_duplicate_sources"], + scripts=[], python_requires=">=3.6", ) diff --git a/start b/start --- a/start +++ b/start @@ -7,8 +7,8 @@ # ODOO_TYPE can be set instead of putting it in setup.cfg # # Version 3.1 -here=$(dirname $0) -project_home=$(cd $here && cd .. && echo $PWD) +here=$(dirname "$0") +project_home=$(cd "$here" && cd .. && echo "$PWD") if [ -x "$(command -v python3)" ]; then python="python3" @@ -54,7 +54,7 @@ if test -z "$module_path"; then echo "DEBUG - No odoo found by using module path" - virtualenv_name=$(basename $(readlink -f $(dirname start))) + virtualenv_name=$(basename "$(readlink -f "$(dirname start)")") echo "DEBUG - Trying $virtualenv_name virtualenvwrapper" # assume you use virtualenvwrapper, and try the same env name as the project venv_python=$WORKON_HOME/$virtualenv_name/bin/python @@ -83,9 +83,9 @@ echo "DEBUG - Odoo found by using module path ($module_path)" fi - odoo=$(dirname $(dirname $module_path)) + odoo=$(dirname "$(dirname $module_path)") - if expr match $odoo ^.*\.egg$ || [ "$ODOO_TYPE" = "odoo10" ] ; + if expr "$odoo" : ^.*\.egg$ || [ "$ODOO_TYPE" = "odoo10" ] ; then odoo_addons_path=${ODOO_ADDONS_PATH:-/opt/odoo/sources/odoo} else @@ -98,13 +98,13 @@ # args=$* -if echo $args | grep -q -e "\( \|^\)--db_host"; +if echo "$args" | grep -q -e '\( \|^\)--db_host'; then db_host="" else db_host="--db_host=localhost" fi -if echo $args | grep -q -e "\( \|^\)-c" -e "\( \|^\)--config"; +if echo "$args" | grep -q -e '\( \|^\)-c' -e '\( \|^\)--config'; then config="" addons_path="" @@ -116,7 +116,7 @@ then config="-c ${project_home}/conf/dev/odoo.conf" fi - if echo $args | grep -q -e "\( \|^\)--addons-path"; + if echo "$args" | grep -q -e '\( \|^\)--addons-path'; then addons_path="" else @@ -141,7 +141,7 @@ fi fi # use load-language from setup.cfg if none already provided -if echo $args | grep -q -e "\( \|^\)-c" -e "\( \|^\)--load-language"; +if echo "$args" | grep -q -e '\( \|^\)-c' -e '\( \|^\)--load-language'; then echo "INFO - load-language flag detected, no reading from setup.cfg" else @@ -157,5 +157,6 @@ # echo "INFO - Odoo Version: $($odoo_bin --version)" -echo "DEBUG - command line is: ${PRE_ODOO_BIN}$(which $odoo_bin) $config $db_host $addons_path $args $load_language" -${PRE_ODOO_BIN}$(which $odoo_bin) $config $db_host $addons_path $args $load_language +echo "DEBUG - command line is: ${PRE_ODOO_BIN}$(command -v $odoo_bin) $config $db_host $addons_path $args $load_language" +# shellcheck disable=SC2046,SC2086 +${PRE_ODOO_BIN}$(command -v $odoo_bin) $config $db_host $addons_path $args $load_language diff --git a/update_duplicate_sources b/update_duplicate_sources deleted file mode 100755 --- a/update_duplicate_sources +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/zsh -# vim: set shiftwidth=4 softtabstop=4 tabstop=4 expandtab: -# -# This script updates another repository that is a copy of the sources. -# Made to be shared with external people. -# The resulting repository won't have the full history log but only tags pushed when running this -# script. Also all sources will be merged within one repository (as in no confman / .hgconf). -# -# Assertions: -# - this is run from the metaproject via "odoo_scripts/update_duplicate_sources" -# - the metaproject is on a tag (run "hg id" to make sure) -# - the source project already exists on orus.io and its path is the same as the metaproject with -# the "_sources" suffix -# - the metaproject is the same type as the sources one - -here=$(dirname $0) -project_home=$(cd $here && cd .. && echo $PWD) - -$here/list_modules - -odoo_modules=($(python3 -B -c "import configparser ; c = configparser.ConfigParser() ; c.read('${project_home}/setup.cfg') ; print(c.has_section('odoo_scripts') and c.has_option('odoo_scripts', 'modules') and ' '.join(c.get('odoo_scripts', 'modules').split()) or '' )")) -dependencies=($(python3 -B -c "import configparser ; c = configparser.ConfigParser() ; c.read('${project_home}/setup.cfg') ; print(c.has_section('odoo_scripts') and c.has_option('odoo_scripts', 'dependencies') and ' '.join(c.get('odoo_scripts', 'dependencies').split()) or '' )")) -other_sources=($(python3 -B -c "import configparser ; c = configparser.ConfigParser() ; c.read('${project_home}/setup.cfg') ; print(c.has_section('odoo_scripts') and c.has_option('odoo_scripts', 'other_sources') and ' '.join(c.get('odoo_scripts', 'other_sources').split()) or '' )")) - -# color stuff -autoload colors && colors -for COLOR in RED GREEN YELLOW BLUE MAGENTA CYAN BLACK WHITE; do - eval $COLOR='$fg_no_bold[${(L)COLOR}]' - eval BOLD_$COLOR='$fg_bold[${(L)COLOR}]' -done -eval RESET='$reset_color' - -pushd $project_home - -if [ -e ".hg" ]; then - branch="$(hg id -b)" - tags=($(hg id -t)) - project_url=$(hg paths default) - source_url=${project_url}_sources - source_name=$(basename $source_url) - hg clone $source_url || exit 1 - pushd $source_name - hg update $branch - branch_exist=$? - popd - echo "${YELLOW}INFO ${RESET} - branch_exist=$branch_exist" - # Remove any content inside it - rm -rf ${source_name}/* - rsync -rRLt --exclude='.hg*' --exclude='.git' $(eval echo $odoo_modules $dependencies $other_sources) $source_name - pushd $source_name - hg addremove - if [[ "x$branch_exist" != "x0" ]]; - then - hg branch $branch - fi - hg commit -m"Update sources" - # Tagging process only works if done locally - if [[ "$tags" != "tip" ]]; then - id_to_tag=$(hg id -i) - for tag in $tags; do - echo "${YELLOW}INFO ${RESET} - Add tag $tag" - hg tag $tag -r $id_to_tag - done - fi - if [[ "x$branch_exist" != "x0" ]]; - then - echo "${YELLOW}INFO ${RESET} - Pushing new branch $branch" - hg push --new-branch --publish - else - echo "${YELLOW}INFO ${RESET} - Pushing branch $branch" - hg push --publish - fi - popd - rm -rf ${source_name} -else - echo "${RED}ERROR${RESET} - Repository VCS unknown/not handled" - popd - exit 1 -fi - -popd