add possibility to run tests with a postgresql container

do_tests.py

@@ -9,5 +9,7 @@
 from subprocess import call
 import sys
 
-import ConfigParser
+import docker  # apt python-docker (1.9) or pip install docker
+from six.moves import configparser as configparser
+from psycopg2 import connect, OperationalError  # apt python-psycopg2
 
@@ -13,6 +15,10 @@
 
-# local file
-import docker_dev_start
+from . import docker_dev_start
+
+if docker.__version__ > '2.5.0':
+    from docker import APIClient as docker_api
+else:
+    from docker import Client as docker_api
 
 _logger = logging.getLogger(__name__)
 
@@ -16,5 +22,5 @@
 
 _logger = logging.getLogger(__name__)
 
-__version__ = '0.0.2'
+__version__ = '0.1.0'
 __date__ = '2018-04-13'
@@ -20,5 +26,5 @@
 __date__ = '2018-04-13'
-__updated__ = '2018-06-25'
+__updated__ = '2018-07-31'
 
 
 def main(argv=None):  # IGNORE:C0111
@@ -128,6 +134,11 @@
         help="Database port [default: %(default)s]",
         default=None,
     )
+    parser.add_argument(
+        '--start-postgresql',
+        help="Start a Postgresql docker [default: %(default)s]",
+        action='store_true',
+    )
     # TODO options
     # - db host/uri (include socket)
     # - db user for creation/remove
@@ -153,6 +164,7 @@
         if nmspc.log_level else nmspc.test_log_level)
     override_tested_module = nmspc.test
     dbport = nmspc.dbport
+    start_postgresql = nmspc.start_postgresql
 
     # Get parameters from setup file
 
@@ -156,9 +168,9 @@
 
     # Get parameters from setup file
 
-    c = ConfigParser.ConfigParser()
+    c = configparser.ConfigParser()
     if not os.path.exists(setup_path):
         logging.fatal('Missing %s', setup_path)
         return 12
     c.read(setup_path)
 
@@ -160,8 +172,9 @@
     if not os.path.exists(setup_path):
         logging.fatal('Missing %s', setup_path)
         return 12
     c.read(setup_path)
 
+    # TODO factorize with docker_dev_start
     registry = (
         c.has_section('odoo_scripts') and
         c.has_option('odoo_scripts', 'registry') and
@@ -174,6 +187,10 @@
         c.has_section('odoo_scripts') and
         c.has_option('odoo_scripts', 'load-language') and
         c.get('odoo_scripts', 'load-language')) or None
+    postgresql_version = (
+         c.has_section('odoo_scripts') and
+         c.has_option('odoo_scripts', 'postgresql_version') and
+         c.get('odoo_scripts', 'postgresql_version')) or '9.6'
     image = "%s/%s:latest" % (registry, project)
     logging.debug("Docker image: %s", image)
 
@@ -184,7 +201,19 @@
         logging.info('Reading from sample configuration %s', sample_conf)
         c.read(sample_conf)
         if not odoo_db_user:
-            odoo_db_user = c.get('options', 'db_user')
+            if c.has_option('options', 'db_user'):
+                odoo_db_user = c.get('options', 'db_user')
+            elif start_postgresql:
+                odoo_db_user = 'odoo'
+            else:
+                _logger.warning("No database user found or given")
+        if not odoo_db_password:
+            if c.has_option('options', 'db_password'):
+                odoo_db_password = c.get('options', 'db_password')
+            elif start_postgresql:
+                odoo_db_password = 'odoo'
+            else:
+                _logger.warning("No database password found or given")
         if (
             c.has_option('options', 'unaccent') and
             c.getboolean('options', 'unaccent')
@@ -210,9 +239,6 @@
         if odoo_db_user:
             args.append('--db_user')
             args.append(odoo_db_user)
-        else:
-            # TODO find from odoo version and conf file
-            pass
         if odoo_db_password:
             args.append('--db_password')
             args.append(odoo_db_password)
@@ -221,4 +247,6 @@
             args.append(dbport)
 
     docker_dev_start.flake8()
+    if start_postgresql:
+        args.append('--start-postgresql')
     if recreate_db:
@@ -224,30 +252,58 @@
     if recreate_db:
-        # XXX use psycopg2
-        # drop database
-        logging.info("Drop any existing database %s", dbname)
-        dropdb = ['dropdb', dbname, '--if-exists', '--no-password']
-        if dbport:
-            dropdb.append('-p')
-            dropdb.append(dbport)
-        result = call(dropdb)
-        if result:
-            return result
-        # create database
-        logging.info("Create database %s", dbname)
-        if not odoo_db_user:
-            logging.fatal(
-                'Owner of database is mandatory when creating database')
-            return 14
-        create_db = ['createdb', dbname, '-O', odoo_db_user, '--no-password']
-        if dbport:
-            create_db.append('-p')
-            create_db.append(dbport)
-        result = call(create_db)
-        if result:
-            return result
-        # add unaccent if needed
-        for extension in extensions:
-            logging.info("Adding extension %s", extension)
-            create_extension = [
-                'psql', dbname, '-c', 'CREATE EXTENSION %s;' % extension]
+        if start_postgresql:
+            docker_client = docker_api(base_url='unix://var/run/docker.sock')
+            name, stop_method = docker_dev_start.docker_run_postgresql(
+                docker_client, project_name, postgresql_version, None,
+            )
+            try:
+                if start_postgresql:
+                    connect(
+                        user=odoo_db_user,
+                        password=odoo_db_password,
+                        database='postgres',
+                        host='/tmp',
+                    )
+            except OperationalError as exception:
+                connection = connect(
+                    user='pg',
+                    database='postgres',
+                    host='/tmp',
+                )
+                with connection.cursor() as cursor:
+                    # not injection safe but you are on your own machine
+                    # with already full access to db
+                    cursor.execute(
+                        'CREATE ROLE %s LOGIN CREATEDB PASSWORD %%s' %
+                        odoo_db_user,
+                        (odoo_db_password,))
+                connection.commit()
+                connection.close()
+            odoo_connection = connect(
+                user='pg',
+                database='postgres',
+                host='/tmp',
+            )
+            odoo_connection.autocommit = True
+            with odoo_connection.cursor() as cursor:
+                _logger.debug("Drop database %s", dbname)
+                cursor.execute('DROP DATABASE IF EXISTS %s' % dbname)
+                _logger.debug("Create database %s", dbname)
+                cursor.execute('CREATE DATABASE %s OWNER %s' % (
+                    dbname, odoo_db_user))
+            odoo_connection = connect(
+                user='pg',
+                database=dbname,
+                host='/tmp',
+            )
+            odoo_connection.autocommit = True
+            with odoo_connection.cursor() as cursor:
+                for extension in extensions:
+                    _logger.info("Adding extension %s", extension)
+                    cursor.execute('CREATE EXTENSION %s' % extension)
+            stop_method()
+        else:
+            # TODO use psycopg2 (see above)
+            # drop database
+            _logger.info("Drop any existing database %s", dbname)
+            dropdb = ['dropdb', dbname, '--if-exists', '--no-password']
             if dbport:
@@ -253,6 +309,6 @@
             if dbport:
-                create_extension.append('-p')
-                create_extension.append(dbport)
-            result = call(create_extension)
+                dropdb.append('-p')
+                dropdb.append(dbport)
+            result = call(dropdb)
             if result:
                 return result
@@ -257,5 +313,29 @@
             if result:
                 return result
+            # create database
+            _logger.info("Create database %s", dbname)
+            if not odoo_db_user:
+                _logger.fatal(
+                    'Owner of database is mandatory when creating database')
+                return 14
+            create_db = ['createdb', dbname, '-O', odoo_db_user, '--no-password']
+            if dbport:
+                create_db.append('-p')
+                create_db.append(dbport)
+            result = call(create_db)
+            if result:
+                return result
+            # add unaccent if needed
+            for extension in extensions:
+                _logger.info("Adding extension %s", extension)
+                create_extension = [
+                    'psql', dbname, '-c', 'CREATE EXTENSION %s;' % extension]
+                if dbport:
+                    create_extension.append('-p')
+                    create_extension.append(dbport)
+                result = call(create_extension)
+                if result:
+                    return result
 
         # TODO start odoo and detect if install fails
         if nmspc.docker:
@@ -267,9 +347,9 @@
             if install_log_level:
                 install_args.append('--log-level')
                 install_args.append(install_log_level)
-            docker_dev_start.main(install_args)
+            result = docker_dev_start.main(install_args)
 
             if result:
                 return result
         else:
             raise NotImplementedError
@@ -271,9 +351,9 @@
 
             if result:
                 return result
         else:
             raise NotImplementedError
-    # TODO start odoo and detect if test fails
+    # start odoo and detect if test fails
     if nmspc.docker:
         test_args = list(args)
         if override_tested_module:
@@ -291,6 +371,4 @@
 
 
 if __name__ == "__main__":
-    return_code = main(sys.argv[1:])
-    if return_code:
-        exit(return_code)
+    exit(main(sys.argv[1:]))

docker_dev_start.py

@@ -30,5 +30,5 @@
 
 _logger = logging.getLogger(__name__)
 
-__version__ = '1.1.3'
+__version__ = '1.1.4'
 __date__ = '2017-08-11'
@@ -34,5 +34,5 @@
 __date__ = '2017-08-11'
-__updated__ = '2018-07-12'
+__updated__ = '2018-07-31'
 
 
 def which(program):
@@ -131,8 +131,8 @@
     parser.add_argument(
         '-p',
         '--dbport',
-        help="Database port [default: %(default)s]",
-        default='5432',
+        help="Change database port",
+        default=None,
     )
     group = parser.add_mutually_exclusive_group()
     group.add_argument(
@@ -489,10 +489,7 @@
         binds.append('{}:/opt/odoo/{}'.format(volume_name, extra_volume))
 
     if start_postgresql and not odoo_help:
-        pg_repository = 'xcgd/postgresql'
-        pg_image = '{}:{}'.format(pg_repository, postgresql_version)
-        name = 'postgresql-{}-{}'.format(postgresql_version, project_name)
-        docker_client.pull(repository=pg_repository, tag=postgresql_version)
-        pg_data_volume_name = 'postgresql_{}-{}'.format(
-            postgresql_version, project_name)
+        host_pg_port = nmspc.dbport if nmspc.dbport else 5433
+        name, stop_postgresql = docker_run_postgresql(
+            docker_client, project_name, postgresql_version, host_pg_port)
 
@@ -498,30 +495,4 @@
 
-        host_pg_port = 5433
-        # volume = createVolume(docker_client, pg_data_volume_name)
-        host_config = docker_client.create_host_config(
-            binds=['{}:/var/lib/postgresql'.format(pg_data_volume_name)],
-            port_bindings={5432: host_pg_port},
-        )
-        logging.debug('Creating postgresql container')
-        pg = docker_client.create_container(
-            image=pg_image,
-            host_config=host_config,
-            name=name,
-        )
-
-        logging.debug('Starting postgresql container')
-        docker_client.start(pg.get('Id'))
-
-        def stop_postgresql():
-            logging.info('Stopping postgresql')
-            docker_client.stop(pg.get('Id'))
-            logging.info('Removing container postgresql')
-            docker_client.remove_container(pg.get('Id'))
-        atexit.register(stop_postgresql)
-
-        # give pg the time to start up
-        import time
-        time.sleep(5)
         arg.append('--db_port')
         arg.append(str(host_pg_port))
 
@@ -667,6 +638,74 @@
         filters={'id': container_id}, quiet=True))
 
 
+def docker_run_postgresql(
+    docker_client, project_name, postgresql_version, host_pg_port=None,
+    stop_at_exit=True,
+):
+    """
+
+    :param docker_client:
+    :param project_name:
+    :param postgresql_version:
+    :param host_pg_port: if None, put the socket in /tmp, else publish the port
+    :param stop_at_exit:
+    :return:
+    """
+    pg_repository = 'xcgd/postgresql'
+    pg_image = '{}:{}'.format(pg_repository, postgresql_version)
+    name = 'postgresql-{}-{}'.format(postgresql_version, project_name)
+    docker_client.pull(repository=pg_repository, tag=postgresql_version)
+    pg_data_volume_name = 'postgresql_{}-{}'.format(
+        postgresql_version, project_name)
+
+    # volume = createVolume(docker_client, pg_data_volume_name)
+    binds = [
+        '{}:/var/lib/postgresql'.format(pg_data_volume_name),
+    ]
+    port_bindings = {}
+    if host_pg_port:
+        port_bindings[5432] = host_pg_port
+    else:
+        binds.append('/tmp:/var/run/postgresql')
+    host_config = docker_client.create_host_config(
+        binds=binds,
+        port_bindings=port_bindings,
+    )
+    if any(
+        '/{}'.format(name) in container['Names']
+        for container in docker_client.containers()
+    ):
+        _logger.debug('Postgresql Container already running')
+        return name, False
+    _logger.debug('Creating postgresql container')
+    pg = docker_client.create_container(
+        image=pg_image,
+        host_config=host_config,
+        name=name,
+    )
+
+    _logger.debug('Starting postgresql container')
+    docker_client.start(pg.get('Id'))
+
+    def stop_postgresql():
+        # TODO test if still exists
+        try:
+            _logger.info('Stopping postgresql')
+            docker_client.stop(pg.get('Id'))
+            _logger.info('Removing container postgresql')
+            docker_client.remove_container(pg.get('Id'))
+        except docker.errors.NotFound:
+            _logger.info('Postgresql already stopped')
+
+    if stop_at_exit:
+        atexit.register(stop_postgresql)
+    _logger.debug("Waiting for postgres to start")
+    # give pg the time to start up
+    import time
+    time.sleep(5)
+    return name, stop_postgresql
+
+
 if __name__ == "__main__":
     return_code = main(sys.argv[1:])
     if return_code: