docker_dev_start.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Launch a docker image but bind the local directory inside the container
and define the module path automatically.

isort:skip_file
"""
# Version 2.19
import argparse
import atexit
import glob
import logging
import os
import pwd
from subprocess import call
import sys

try:
    from configparser import ConfigParser
except ImportError:
    from ConfigParser import SafeConfigParser as ConfigParser

import docker  # apt python-docker (1.9) or pip install docker
from psycopg2 import connect, OperationalError  # apt python-psycopg2

if docker.__version__ > '2.5.0':
    from docker import APIClient as docker_api
else:
    from docker import Client as docker_api

# TODO auto create list of module

_logger = logging.getLogger(__name__)

__version__ = '1.1.10'
__date__ = '2017-08-11'
__updated__ = '2019-01-17'


def which(program):
    """Return path of program if it exists
from:
https://stackoverflow.com/questions/377017/test-if-executable-exists-in-python/377028#377028
    """
    def is_exe(fpath):
        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)

    fpath, fname = os.path.split(program)
    if fpath:
        if is_exe(program):
            return program
    else:
        for path in os.environ["PATH"].split(os.pathsep):
            exe_file = os.path.join(path, program)
            if is_exe(exe_file):
                return exe_file

    return None


def flake8(odoo_type):
    """Run flake8
    """
    _logger.info('Running flake8')
    if odoo_type in ('odoo7', 'odoo8', 'odoo9', 'odoo10'):
        image = 'xcgd/flake8:2'
    else:
        image = 'xcgd/flake8:3'
    call([
        'docker', 'run', '--rm', '--volume',
        '{}:/mnt'.format(os.environ['PWD']), image,
        '/mnt'])


def isort():
    """Run isort
    """
    _logger.info('Running isort')
    call([
        'docker', 'run', '--rm', '--volume',
        '{}:/mnt'.format(os.environ['PWD']), 'xcgd/isort',
        'sh', '-c', 'cd /mnt && isort -c'])


def main(argv=None):  # IGNORE:C0111
    """Parse arguments and launch conversion
    """
    program_version = __version__
    program_build_date = str(__updated__)
    program_version_message = '%%(prog)s %s (%s)' % (
        program_version, program_build_date)
    program_shortdesc = __doc__.split(".")[0]
    program_license = '''%s

  Created by Vincent Hatakeyama on %s.
  Copyright 2017, 2018 XCG Consulting. All rights reserved.

  Licensed under the MIT License

  Distributed on an "AS IS" basis without warranties
  or conditions of any kind, either express or implied.

USAGE
''' % (program_shortdesc, str(__date__))

    project_path = os.path.realpath('.')
    project_name = os.path.basename(project_path)
    if project_name == 'odoo_scripts':
        logging.fatal(
            "You must run this script from the super project"
            " (./odoo_scripts/docker_dev_start.py)")
        return 1

    setup_path = 'setup.cfg'
    # TODO add a way to store configuration options in a project file
    # Argument parsing
    parser = argparse.ArgumentParser(
        description=program_license,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument(
        '-V', '--version', action='version',
        version=program_version_message)
    parser.add_argument(
        '-v', '--verbose', dest='verbose', action='count',
        help="set verbosity level [default: %(default)s]")
    parser.add_argument(
        '--db_user',
        help="Database user [default: %(default)s]",
        default=None,
    )
    parser.add_argument(
        '--db_password',
        help="Database user password [default: %(default)s]",
        default=None,
    )
    parser.add_argument(
        '--max-cron-threads',
        help="max cron threads [default: %(default)s]",
        default=None,
    )
    parser.add_argument(
        '-d',
        '--database',
        help="Database [default: %(default)s]",
        default=None,
    )
    parser.add_argument(
        '-p',
        '--dbport',
        help="Change database port",
        default=None,
    )
    group = parser.add_mutually_exclusive_group()
    group.add_argument(
        '-u',
        '--update',
        help="Module to update (will also set --i18n-overwrite)"
             " [default: %(default)s]\n"
             "Options --update, --test and --test-default"
             " cannot be used at the same time",
        default=None,
    )
    group.add_argument(
        '-t',
        '--test',
        help="Modules to test (will also set --log-level=test)"
             " [default: %(default)s]\n"
             "Options --update, --test and --test-default"
             " cannot be used at the same time",
        default=None,
    )
    group.add_argument(
        '--test-default',
        help="Test all modules in module_list_test"
             " [default: %(default)s]\n"
             "Options --update, --test and --test-default"
             " cannot be used at the same time",
        action='store_true',
    )
    parser.add_argument(
        '-i',
        '--install',
        help="Module to install [default: %(default)s]",
        default=None,
    )
    parser.add_argument(
        '--without-demo',
        help="Module to avoid including demo data [default: %(default)s]",
        default=None,
    )
    parser.add_argument(
        '--create-user',
        help="Create user in database (using $user) [default: %(default)s]",
        action='store_true',
    )
    parser.add_argument(
        '--install-default',
        help=("Append the module_list from setup.cfg to the modules to install"
              " [default: %(default)s]"),
        action='store_true',
    )
    parser.add_argument(
        '--start-postgresql',
        help="Start a Postgresql docker",
        action='store_true',
    )
    parser.add_argument(
        '--no-flake8',
        help="Do not run flake8",
        action='store_false',
        dest='flake8',
    )
    parser.add_argument(
        '--no-isort',
        help="Do not run isort",
        action='store_false',
        dest='isort',
    )
    parser.add_argument(
        '--odoo-help',
        help="Pass --help to odoo binary [default: %(default)s]",
        action='store_true',
    )
    parser.add_argument(
        '--stop-after-init',
        help="Pass --stop-after-init to odoo binary [default: %(default)s]",
        action='store_true',
    )
    build_group = parser.add_mutually_exclusive_group()
    build_group.add_argument(
        '--no-build',
        help="Do not build locally [default: %(default)s]",
        action='store_true',
    )
    build_group.add_argument(
        '--force-build',
        help="Force new local build [default: %(default)s]",
        action='store_true',
    )
    network_group = parser.add_mutually_exclusive_group()
    network_group.add_argument(
        '--host-network',
        help="Use host network [default: %(default)s]",
        action='store_false',
    )
    network_group.add_argument(
        '--docker-network',
        help=("Use docker network (works better with"
              " python-netifaces installed) [default: %(default)s]"),
        action='store_true',
    )
    parser.add_argument(
        '--no-dev',
        help="No dev options",
        action='store_false',
        dest='dev',
    )
    parser.add_argument(
        '--load-language',
        help="specifies the languages for the translations you want to be"
             " loaded",
        default=None,
        dest='LOAD_LANGUAGE',
    )
    # TODO there are more
    odoo_log_levels = ['info', 'warn', 'debug', 'debug_sql']
    parser.add_argument(
        '--log-level',
        help="Override odoo log level (for tests and install)",
        default=None,
        choices=odoo_log_levels,
    )
    parser.add_argument(
        '--restore',
        help="Database to restore [default: %(default)s]",
        default=None,
        dest='DUMP',
    )
    parser.add_argument(
        '--config',
        help="Configuration file to use, if any [default: %(default)s]",
        default=os.path.join('conf', 'dev', 'odoo.conf'),
    )

    # TODO detect that user is member of docker group

    # TODO add a way to add options to docker
    # TODO add a way to add arg to odoo

    nmspc = parser.parse_args(argv)
    verbose = nmspc.verbose
    if not verbose:
        logging.basicConfig(level=logging.WARN)
    elif verbose == 1:
        logging.basicConfig(level=logging.INFO)
    else:
        logging.basicConfig(level=logging.DEBUG)
    db_user = nmspc.db_user
    db_password = nmspc.db_password
    use_host_network = nmspc.host_network
    start_postgresql = nmspc.start_postgresql
    run_flake8 = nmspc.flake8
    run_isort = nmspc.isort
    odoo_help = nmspc.odoo_help
    dev = nmspc.dev
    restore_filename = nmspc.DUMP
    database = nmspc.database

    if restore_filename:
        if not database:
            _logger.fatal('No database name given for restore')
            return 13

    c = ConfigParser()
    if not os.path.exists(setup_path):
        _logger.fatal('Missing %s', setup_path)
        return 12
    c.read(setup_path)

    modules = []
    if c.has_option('odoo_scripts', 'modules'):
        for entry in c.get('odoo_scripts', 'modules').split():
            modules.extend(glob.glob(entry))
    _logger.debug("addon modules: %s", modules)

    registry = (
        c.has_section('odoo_scripts') and
        c.has_option('odoo_scripts', 'registry') and
        c.get('odoo_scripts', 'registry')) or 'dockerhub.xcg.io'
    project = (
        c.has_section('odoo_scripts') and
        c.has_option('odoo_scripts', 'image') and
        c.get('odoo_scripts', 'image')) or os.path.basename(project_path)
    odoo_type = (
        c.has_section('odoo_scripts') and
        c.has_option('odoo_scripts', 'odoo_type') and
        c.get('odoo_scripts', 'odoo_type')) or 'odoo7'
    image = "%s/%s:latest" % (registry, project)
    postgresql_version = (
        c.has_section('odoo_scripts') and
        c.has_option('odoo_scripts', 'postgresql_version') and
        c.get('odoo_scripts', 'postgresql_version')) or '9.6'
    _logger.debug("Docker image: %s", image)
    # detect if docker image already exists
    docker_client = docker_api(base_url='unix://var/run/docker.sock')
    image_list = docker_client.images(name=image, quiet=True)
    if not image_list:
        _logger.info("Image %s does not exist", image)
    else:
        _logger.info("Image %s exists", image)
    if (not image_list and not nmspc.no_build) or nmspc.force_build:
        _logger.info("Building image %s", image)
        import docker_build
        arguments = [
            '--dev'
        ]
        if verbose == 1:
            arguments.append('-v')
        elif verbose and verbose > 1:
            arguments.append('-vv')
        failed = docker_build.main(arguments)
        if failed:
            return failed

    # options is only used with subprocess call
    options = [
        '--name',
        project_name,
        '--rm',
        '--tty',
        '--interactive',
    ]
    if not use_host_network:
        options.append('--publish')
        options.append('8069:8069')
    binds = []
    arg = [
        'start',
    ]
    if db_user:
        arg.append('--db_user %s' % db_user)
    if db_password:
        arg.append('--db_password %s' % db_password)
    if nmspc.update:
        arg.append('-u %s' % nmspc.update)
        arg.append('--i18n-overwrite')
    if nmspc.test:
        arg.append('-u %s' % nmspc.test)
        arg.append('--test-enable')
        if odoo_type == 'odoo7':
            arg.append('--log-level=test')
    if nmspc.test or nmspc.stop_after_init:
        arg.append('--stop-after-init')
    if nmspc.test_default:
        test_modules = c.get('odoo_scripts', 'module_list_tests').split() or []
        str_modules = ','.join(test_modules)
        arg.append('-u %s' % str_modules)
        arg.append('--test-enable')
        if odoo_type == 'odoo7':
            arg.append('--log-level=test')
        arg.append('--stop-after-init')
    if database:
        arg.append('-d %s' % database)
    if nmspc.install or nmspc.install_default:
        modules_to_install = []
        if nmspc.install:
            modules_to_install.extend(nmspc.install.split(','))
        if nmspc.install_default:
            modules_to_install.extend((
                c.has_section('odoo_scripts') and
                c.has_option('odoo_scripts', 'module_list') and
                c.get('odoo_scripts', 'module_list').split()) or [])
        if modules_to_install:
            arg.append('-i')
            arg.append(','.join(modules_to_install))
    if nmspc.without_demo:
        arg.append('--without-demo')
        arg.append(nmspc.without_demo)
    if nmspc.max_cron_threads:
        arg.append('--max-cron-threads=%s' % nmspc.max_cron_threads)
    if nmspc.log_level:
        arg.append('--log-level')
        arg.append(nmspc.log_level)
    if nmspc.LOAD_LANGUAGE:
        arg.append('--load-language')
        arg.append(nmspc.LOAD_LANGUAGE)

    # auto detect local ip
    if use_host_network:
        local_ip = '127.0.0.1'
        options.append('--network')
        options.append('host')
    else:
        local_ip = None
        try:
            from netifaces import ifaddresses, AF_INET  # apt python-netifaces
            iface_name = 'docker0'
            addresses = [
                i['addr']
                for i in ifaddresses(
                    iface_name
                ).setdefault(AF_INET, [{'addr': 'No IP addr'}])
            ]
            if addresses:
                local_ip = addresses[0]
        except ImportError:
            _logger.warn("Consider installing python netifaces"
                         " to ease local IP detection")
        if not local_ip:
            import socket
            _logger.info('Contacting Google Public DNS to find our IP')
            local_ip = [
                (
                    s.connect(('8.8.8.8', 53)),
                    s.getsockname()[0],
                    s.close()
                ) for s in [
                    socket.socket(socket.AF_INET, socket.SOCK_DGRAM)]
            ][0][1]

        _logger.debug('IP found %s', local_ip)

    arg.append('--db_host')
    arg.append(local_ip)

    if nmspc.dbport:
        arg.append("--db_port")
        arg.append('%s' % nmspc.dbport)

    # auto detect local conf
    local_conf_path = nmspc.config
    local_conf_dir = os.path.dirname(os.path.realpath(local_conf_path))
    user = db_user
    password = db_password
    if os.path.isfile(local_conf_path):
        _logger.info('Local configuration file found: %s', local_conf_path)
        odoo_conf_path = (
            '/etc/odoo' if odoo_type in ('odoo11', 'odoo12', )
            else '/opt/odoo/etc'
        )
        binds.append('{}:{}'.format(os.path.join(
            project_path, local_conf_dir), odoo_conf_path))
        cp_local = ConfigParser()
        cp_local.read(local_conf_path)
        if not user:
            user = cp_local.get('options', 'db_user')
        if not password:
            password = cp_local.get(
                'options',
                'db_password',
            )
        # doing this avoid having to change the value of redis_host
        # used by smile_redis_session_store
        redis_host = (
             cp_local.has_section('options') and
             cp_local.has_option('options', 'redis_host') and
             cp_local.get('options', 'redis_host')) or None
        if redis_host:
            print('add redis host')
            options.append('--add-host')
            options.append('{}:{}'.format(redis_host, local_ip))
    else:
        _logger.info('No configuration file at: %s', local_conf_path)
    # default values if nothing else
    if not user:
        user = 'odoo'
    if not password:
        password = 'odoo'

    # data volume handling
    if odoo_type != 'odoo7':
        data_volume_name = '{}_data'.format(project_name)
        _logger.debug('Using data volume %s', data_volume_name)
        createVolume(docker_client, data_volume_name)
        # make sure the permission in the volume are correct
        mount_opts = '{}:/mnt/data'.format(data_volume_name)
        # TODO replace by something cleaner if possible
        call([
            'docker', 'run', '--rm', '-v',
            mount_opts, 'busybox', 'chmod', '-R', '777', '/mnt/data',
        ])

        binds.append('{}:/mnt/data'.format(data_volume_name))
        arg.append('--data-dir /mnt/data')
    else:
        _logger.debug('No data volume for this odoo version')

    # avoid the duplication of unbind volumes with all addons
    # additionnal_addons only in odoo < 10
    for extra_volume in ['additional_addons', 'var', 'data']:
        volume_name = '{}_opt_odoo_{}'.format(project_name, extra_volume)
        # volume = createVolume(docker_client, volume_name)
        binds.append('{}:/opt/odoo/{}'.format(volume_name, extra_volume))

    if start_postgresql and not odoo_help:
        host_pg_port = nmspc.dbport if nmspc.dbport else 5433
        name, stop_postgresql = docker_run_postgresql(
            docker_client, project_name, postgresql_version, host_pg_port)

        arg.append('--db_port')
        arg.append(str(host_pg_port))

        # Check that connection can be done, try to create user if asked to
        # TODO: handle the case where the database is still starting up
        # TODO: (and remove the sleep)
        try:
            if local_ip or start_postgresql:
                port = host_pg_port if start_postgresql else None
                connect(
                    user=user,
                    password=password,
                    database='postgres',
                    host=local_ip,
                    port=port,
                )
            else:
                connect(
                    user=user,
                    password=password,
                    database='postgres',
                )

        except OperationalError as exception:
            if nmspc.create_user:
                _logger.debug('Cannot connect to database with user %s', user)
                _logger.debug(exception)
                _logger.info('Creating user %s', user)
                if start_postgresql:
                    connection = connect(
                        user='postgres',
                        database='postgres',
                        host=local_ip,
                        port=host_pg_port,
                    )
                else:
                    loginname = pwd.getpwuid(os.getuid())[0]
                    connection = connect(user=loginname, database='postgres')
                with connection.cursor() as cursor:
                    # not injection safe but you are on your own machine
                    # with already full access to db
                    cursor.execute(
                        'CREATE ROLE %s LOGIN CREATEDB PASSWORD %%s' % user,
                        (password,))
                connection.commit()
                connection.close()
            else:
                _logger.fatal(
                    'Cannot connect to database with user %s', user)
                _logger.fatal(exception)
                _logger.info(
                    "You can add the --create-user argument to create it")
                return 16
        # restore
        if restore_filename:
            restore_basename = os.path.basename(restore_filename)
            _logger.info("Copying dump file in docker")
            call([
                'docker', 'cp', restore_filename,
                '{}:/tmp/{}'.format(name, restore_basename)])
            _logger.info("Creating database")
            call([
                'docker', 'exec', name, 'createdb', '-U', user, database])
            _logger.info("Restoring database")
            restore = call([
                'docker', 'exec', name, 'pg_restore', '-U', user, '-O',
                '-d', database, '/tmp/{}'.format(restore_basename)])
            if not restore:
                return 15
            _logger.info("Removing dump file in docker")
            call(['docker', 'exec', name, '/tmp/{}'.format(restore_basename)])

    if not start_postgresql and not odoo_help and restore_filename:
        _logger.info("Creating database %s", database)
        createdb = call([
            'createdb', '-U', user, database])
        if not createdb:
            return 17
        _logger.info("Restoring database %s", database)
        restore = call([
            'pg_restore', '-U', user, '-O',
            '-d', database, restore_filename])
        if not restore:
            return 15

    # volume magic
    for module in modules:
        binds.append(
            '%s:/mnt/addons/%s' % (
                os.path.realpath(os.path.join(
                    project_path, module)),
                os.path.basename(module))
        )
    all_addons_dir = ['/mnt/addons']
    if odoo_type in ('odoo7', 'odoo8'):
        all_addons_dir.append('/opt/odoo/sources/odoo/addons')
    if all_addons_dir:
        arg.append('--addons-path')
        arg.append(','.join(all_addons_dir))

    if run_flake8:
        flake8(odoo_type)

    if run_isort:
        isort()

    if dev:
        if odoo_type in ('odoo11', 'odoo10'):
            # ipdb should not be run if not interactive
            arg.append('--dev=xml,reload,ipdb')
        if odoo_type in ('odoo8',):
            arg.append('--auto-reload')
    if odoo_help:
        arg.append('--help')

    # use call to allow usage of pdb
    for bind in binds:
        options.append('--volume')
        options.append(bind)

    cmd = ['docker', 'run']
    cmd.extend(options)
    cmd.append(image)
    cmd.extend(arg)
    _logger.debug(' '.join(cmd))
    result = call(cmd)
    return result


def createVolume(docker_client, data_volume_name):
    """Return the volume passed in parameter, creating it if it does not exists
    """
    volumes = docker_client.volumes(filters={'name': data_volume_name})
    if volumes['Volumes']:
        _logger.debug('Volume %s already exist', data_volume_name)
        return volumes['Volumes'][0]
    else:
        _logger.debug('Creating volume %s', data_volume_name)
        return docker_client.create_volume(name=data_volume_name)


def isRunning(docker_client, container_id):
    """Return true if the container is still running
    """
    return len(docker_client.containers(
        filters={'id': container_id}, quiet=True))


def docker_run_postgresql(
    docker_client, project_name, postgresql_version, host_pg_port=None,
    stop_at_exit=True,
):
    """

    :param docker_client:
    :param project_name:
    :param postgresql_version:
    :param host_pg_port: if None, put the socket in /tmp, else publish the port
    :param stop_at_exit:
    :return:
    """
    pg_repository = 'postgres'
    version = '{}-alpine'.format(postgresql_version)
    pg_image = '{}:{}'.format(pg_repository, version)
    name = 'postgresql-{}-{}'.format(postgresql_version, project_name)
    docker_client.pull(repository=pg_repository, tag=version)
    pg_data_volume_name = 'postgresql_{}-{}'.format(
        postgresql_version, project_name)

    # volume = createVolume(docker_client, pg_data_volume_name)
    binds = [
        '{}:/var/lib/postgresql'.format(pg_data_volume_name),
    ]
    port_bindings = {}
    if host_pg_port:
        port_bindings[5432] = host_pg_port
    else:
        binds.append('/tmp:/var/run/postgresql')
    host_config = docker_client.create_host_config(
        binds=binds,
        port_bindings=port_bindings,
    )
    if any(
        '/{}'.format(name) in container['Names']
        for container in docker_client.containers()
    ):
        _logger.debug('Postgresql Container already running')
        return name, False
    _logger.debug('Creating postgresql container')
    pg = docker_client.create_container(
        image=pg_image,
        host_config=host_config,
        name=name,
    )

    _logger.debug('Starting postgresql container')
    docker_client.start(pg.get('Id'))

    def stop_postgresql():
        # TODO test if still exists
        try:
            _logger.info('Stopping postgresql')
            docker_client.stop(pg.get('Id'))
            _logger.info('Removing container postgresql')
            docker_client.remove_container(pg.get('Id'))
        except docker.errors.NotFound:
            _logger.info('Postgresql already stopped')

    if stop_at_exit:
        atexit.register(stop_postgresql)
    _logger.debug("Waiting for postgres to start")
    # give pg the time to start up
    import time
    time.sleep(5)
    return name, stop_postgresql


if __name__ == "__main__":
    return_code = main(sys.argv[1:])
    if return_code:
        exit(return_code)