do_tests.py

#!/usr/bin/env python
# vim: set shiftwidth=4 softtabstop=4:
"""Script to run test with local modules

isort:skip_file
"""
# Version 2.22
import argparse
import logging
import os
from subprocess import call
import sys

import docker  # apt python-docker (1.9) or pip install docker
from six.moves import configparser as configparser
from psycopg2 import connect, OperationalError  # apt python-psycopg2

import docker_dev_start

if docker.__version__ > '2.5.0':
    from docker import APIClient as docker_api
else:
    from docker import Client as docker_api

_logger = logging.getLogger(__name__)

__version__ = '0.1.1'
__date__ = '2018-04-13'
__updated__ = '2018-10-24'


def main(argv=None):  # IGNORE:C0111
    """Parse arguments and docker build
    """
    program_version = __version__
    program_build_date = str(__updated__)
    program_version_message = '%%(prog)s %s (%s)' % (
        program_version, program_build_date)
    program_shortdesc = __doc__.split(".")[0]
    program_license = '''%s

  Created by Vincent Hatakeyama on %s.
  Copyright 2018 XCG Consulting. All rights reserved.

  Licensed under the MIT License

  Distributed on an "AS IS" basis without warranties
  or conditions of any kind, either express or implied.

USAGE
''' % (program_shortdesc, str(__date__))

    # TODO the script assume it is launched from the parent super project
    project_path = os.path.realpath('.')
    project_name = os.path.basename(project_path)
    if project_name == 'odoo_scripts':
        logging.fatal(
            "You must run this script from the super project"
            " (./odoo_scripts/docker_build.py)")
        return 1

    setup_path = 'setup.cfg'
    # TODO add a way to store configuration options in a project file
    # Argument parsing
    parser = argparse.ArgumentParser(
        description=program_license,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument(
        '-V', '--version', action='version',
        version=program_version_message)
    parser.add_argument(
        '-v', '--verbose', dest='verbose', action='count',
        help="set verbosity level [default: %(default)s]")
    docker_group = parser.add_mutually_exclusive_group()
    docker_group.add_argument(
        '--docker',
        help="Use docker to launch tests",
        action='store_true',
        default=True,
        dest='docker',
    )
    docker_group.add_argument(
        '--no-docker',
        help="Do not use docker to launch tests",
        action='store_false',
        dest='docker',
    )
    parser.add_argument(
        '-d',
        '--dbname',
        help="Database name [default: %(default)s]",
        default='%s_test' % project_name,
    )
    parser.add_argument(
        '--db_user',
        help="Odoo database user [default: %(default)s]",
        default=None,
    )
    parser.add_argument(
        '--db_password',
        help="Odoo database password [default: %(default)s]",
        default=None,
    )
    odoo_log_levels = ['info', 'warn', 'debug']  # TODO there are more
    parser.add_argument(
        '--log-level',
        help="Override odoo log level (for tests and install)",
        default=None,
        choices=odoo_log_levels,
    )
    parser.add_argument(
        '--install-log-level',
        help="Override odoo log level (for install)",
        default='warn',
        choices=odoo_log_levels,
    )
    parser.add_argument(
        '--test-log-level',
        help="Override odoo log level (for test)",
        default=None,
        choices=odoo_log_levels,
    )
    parser.add_argument(
        '--no-create-database',
        dest='createdb',
        action='store_false',
    )
    parser.add_argument(
        '--test',
        help="Modules to test (override defaults from setup.cfg)",
        default=None,
    )
    parser.add_argument(
        '-p',
        '--dbport',
        help="Database port [default: %(default)s]",
        default=None,
    )
    parser.add_argument(
        '--start-postgresql',
        help="Start a Postgresql docker [default: %(default)s]",
        action='store_true',
    )
    # TODO options
    # - db host/uri (include socket)
    # - db user for creation/remove

    nmspc = parser.parse_args(argv)
    verbose = nmspc.verbose
    if not verbose:
        logging.basicConfig(level=logging.WARN)
    if verbose == 1:
        logging.basicConfig(level=logging.INFO)
    if verbose and verbose > 1:
        logging.basicConfig(level=logging.DEBUG)
    dbname = nmspc.dbname
    odoo_db_user = nmspc.db_user
    odoo_db_password = nmspc.db_password
    recreate_db = nmspc.createdb
    install_log_level = (
        nmspc.log_level
        if nmspc.log_level else nmspc.install_log_level)
    test_log_level = (
        nmspc.log_level
        if nmspc.log_level else nmspc.test_log_level)
    override_tested_module = nmspc.test
    dbport = nmspc.dbport
    start_postgresql = nmspc.start_postgresql

    # Get parameters from setup file

    c = configparser.ConfigParser()
    if not os.path.exists(setup_path):
        _logger.fatal('Missing %s', setup_path)
        return 12
    c.read(setup_path)

    # TODO factorize with docker_dev_start
    registry = (
        c.has_section('odoo_scripts') and
        c.has_option('odoo_scripts', 'registry') and
        c.get('odoo_scripts', 'registry')) or 'registry.xcg.io'
    project = (
        c.has_section('odoo_scripts') and
        c.has_option('odoo_scripts', 'image') and
        c.get('odoo_scripts', 'image')) or os.path.basename(project_path)
    languages = (
        c.has_section('odoo_scripts') and
        c.has_option('odoo_scripts', 'load-language') and
        c.get('odoo_scripts', 'load-language')) or None
    postgresql_version = (
        c.has_section('odoo_scripts') and
        c.has_option('odoo_scripts', 'postgresql_version') and
        c.get('odoo_scripts', 'postgresql_version')) or '9.6'
    odoo_type = (
        c.has_section('odoo_scripts') and
        c.has_option('odoo_scripts', 'odoo_type') and
        c.get('odoo_scripts', 'odoo_type')) or 'odoo7'
    image = "%s/%s:latest" % (registry, project)
    _logger.debug("Docker image: %s", image)
    extensions = (
         c.has_section('odoo_scripts') and
         c.has_option('odoo_scripts', 'pg.extensions') and
         c.get('odoo_scripts', 'pg.extensions').split() or []
    )

    # read from odoo.conf if it exists

    sample_conf = 'conf/dev/odoo.conf'
    if os.path.exists(sample_conf):
        _logger.info('Reading from sample configuration %s', sample_conf)
        c.read(sample_conf)
        if not odoo_db_user:
            if c.has_option('options', 'db_user'):
                odoo_db_user = c.get('options', 'db_user')
        if not odoo_db_password:
            if c.has_option('options', 'db_password'):
                odoo_db_password = c.get('options', 'db_password')
        if (
            c.has_option('options', 'unaccent') and
            c.getboolean('options', 'unaccent')
        ):
            extensions.append('unaccent')
    else:
        _logger.debug('No sample configuration %s', sample_conf)
        extensions.append('unaccent')

    if not odoo_db_user:
        if start_postgresql:
            odoo_db_user = 'odoo'
        else:
            if not odoo_db_user and c.has_option('odoo_scripts', 'db_user'):
                odoo_db_user = c.get('odoo_scripts', 'db_user')
            if not odoo_db_user:
                _logger.warning("No database user found or given")
    if not odoo_db_password:
        if start_postgresql:
            odoo_db_password = 'odoo'
        else:
            if not odoo_db_password and c.has_option('odoo_scripts', 'db_password'):
                odoo_db_password = c.get('odoo_scripts', 'db_password')
            if not odoo_db_password:
                _logger.warning("No database password found or given")

    # Do stuff

    if nmspc.docker:
        # TODO detect that user is member of docker group
        args = [
            '-d',
            dbname,
            '--stop-after-init',
            '--max-cron-threads',
            '0',
            '--no-flake8',
            '--no-dev',
        ]
        if odoo_db_user:
            args.append('--db_user')
            args.append(odoo_db_user)
        if odoo_db_password:
            args.append('--db_password')
            args.append(odoo_db_password)
        if dbport:
            args.append('--dbport')
            args.append(dbport)

    docker_dev_start.flake8(odoo_type)
    if start_postgresql:
        args.append('--start-postgresql')
    if recreate_db:
        if start_postgresql:
            docker_client = docker_api(base_url='unix://var/run/docker.sock')
            name, stop_method = docker_dev_start.docker_run_postgresql(
                docker_client, project_name, postgresql_version, None,
            )
            try:
                if start_postgresql:
                    connect(
                        user=odoo_db_user,
                        password=odoo_db_password,
                        database='postgres',
                        host='/tmp',
                    )
            except OperationalError:
                connection = connect(
                    user='pg',
                    database='postgres',
                    host='/tmp',
                )
                with connection.cursor() as cursor:
                    # not injection safe but you are on your own machine
                    # with already full access to db
                    cursor.execute(
                        'CREATE ROLE %s LOGIN CREATEDB PASSWORD %%s' %
                        odoo_db_user,
                        (odoo_db_password,))
                connection.commit()
                connection.close()
            odoo_connection = connect(
                user='pg',
                database='postgres',
                host='/tmp',
            )
            odoo_connection.autocommit = True
            with odoo_connection.cursor() as cursor:
                _logger.debug("Drop database %s", dbname)
                cursor.execute('DROP DATABASE IF EXISTS %s' % dbname)
                _logger.debug("Create database %s", dbname)
                cursor.execute('CREATE DATABASE %s OWNER %s' % (
                    dbname, odoo_db_user))
            odoo_connection = connect(
                user='pg',
                database=dbname,
                host='/tmp',
            )
            odoo_connection.autocommit = True
            with odoo_connection.cursor() as cursor:
                for extension in extensions:
                    _logger.info("Adding extension %s", extension)
                    cursor.execute('CREATE EXTENSION %s' % extension)
            stop_method()
        else:
            # TODO use psycopg2 (see above)
            # drop database
            _logger.info("Drop any existing database %s", dbname)
            dropdb = ['dropdb', dbname, '--if-exists', '--no-password']
            if dbport:
                dropdb.append('-p')
                dropdb.append(dbport)
            result = call(dropdb)
            if result:
                return result
            # create database
            _logger.info("Create database %s", dbname)
            if not odoo_db_user:
                _logger.fatal(
                    'Owner of database is mandatory when creating database')
                return 14
            create_db = [
                'createdb', dbname, '-O', odoo_db_user, '--no-password']
            if dbport:
                create_db.append('-p')
                create_db.append(dbport)
            result = call(create_db)
            if result:
                return result
            # add unaccent if needed
            for extension in extensions:
                _logger.info("Adding extension %s", extension)
                create_extension = [
                    'psql', dbname, '-c', 'CREATE EXTENSION %s;' % extension]
                if dbport:
                    create_extension.append('-p')
                    create_extension.append(dbport)
                result = call(create_extension)
                if result:
                    return result

        # TODO start odoo and detect if install fails
        if nmspc.docker:
            install_args = list(args)
            install_args.append('--install-default')
            if languages:
                install_args.append('--load-language')
                install_args.append(languages)
            if install_log_level:
                install_args.append('--log-level')
                install_args.append(install_log_level)
            result = docker_dev_start.main(install_args)

            if result:
                return result
        else:
            raise NotImplementedError
    # start odoo and detect if test fails
    if nmspc.docker:
        test_args = list(args)
        if override_tested_module:
            test_args.append('--test')
            test_args.append(override_tested_module)
        else:
            test_args.append('--test-default')
        if test_log_level:
            test_args.append('--log-level')
            test_args.append(test_log_level)
        return docker_dev_start.main(test_args)
    else:
        raise NotImplementedError
    return 255


if __name__ == "__main__":
    exit(main(sys.argv[1:]))