#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Launch a docker image but bind the local directory inside the container
and define the module path automatically.
"""
import argparse
import logging
import pwd
import signal
from subprocess import call
import sys
import os
import ConfigParser
import docker # apt python-docker (1.9)
from psycopg2 import connect, OperationalError # apt python-psycopg2
from requests.exceptions import ConnectionError
# TODO auto create list of module
_logger = logging.getLogger(__name__)
__version__ = '0.1.0'
__date__ = '2017-08-11'
__updated__ = '2017-09-07'
# Check the version of the "docker" package
if docker.__version__.split('.') < [2]:
print("the version of the docker package is {} instead of 1.9"
.format(docker.__version__))
sys.exit(1)
def main(argv=None): # IGNORE:C0111
"""Parse arguments and launch conversion
"""
if argv is not None:
sys.argv.extend(argv)
program_version = __version__
program_build_date = str(__updated__)
program_version_message = '%%(prog)s %s (%s)' % (
program_version, program_build_date)
program_shortdesc = __doc__.split(".")[0]
program_license = '''%s
Created by Vincent Hatakeyama on %s.
Copyright 2017 XCG Consulting. All rights reserved.
Licensed under the MIT License
Distributed on an "AS IS" basis without warranties
or conditions of any kind, either express or implied.
USAGE
''' % (program_shortdesc, str(__date__))
# TODO the script assume it is launched from the parent super project
project_path = os.path.realpath('.')
project_name = os.path.basename(project_path)
setup_path = 'setup.cfg'
# TODO add a way to store configuration options in a project file
# Argument parsing
parser = argparse.ArgumentParser(
description=program_license,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument(
'-V', '--version', action='version',
version=program_version_message)
parser.add_argument(
'-v', '--verbose', dest='verbose', action='count',
help="set verbosity level [default: %(default)s]")
parser.add_argument(
'--db_user',
help="Database user [default: %(default)s]",
default=None,
)
parser.add_argument(
'--db_password',
help="Database user password [default: %(default)s]",
default=None,
)
parser.add_argument(
'-d',
'--database',
help="Database [default: %(default)s]",
default=None,
)
parser.add_argument(
'-p',
'--dbport',
help="Database port [default: %(default)s]",
default=5432,
)
group = parser.add_mutually_exclusive_group()
group.add_argument(
'-u',
'--update',
help="Module to update (will also set --i18n-overwrite)"
" [default: %(default)s]\n"
"Options --update, --test and --test-default"
" cannot be used at the same time",
default=None,
)
group.add_argument(
'-t',
'--test',
help="Modules to test (will also set --log-level=test)"
" [default: %(default)s]\n"
"Options --update, --test and --test-default"
" cannot be used at the same time",
default=None,
)
group.add_argument(
'--test-default',
help="Test all modules in module_list_test"
" [default: %(default)s]\n"
"Options --update, --test and --test-default"
" cannot be used at the same time",
action='store_true',
)
parser.add_argument(
'-i',
'--install',
help="Module to install [default: %(default)s]",
default=None,
)
parser.add_argument(
'--without-demo',
help="Module to avoid including demo data [default: %(default)s]",
default=None,
)
parser.add_argument(
'--create-user',
help="Create user in database (using $user) [default: %(default)s]",
action='store_true',
)
parser.add_argument(
'--install-default',
help=("Append the module_list from setup.cfg to the modules to install"
" [default: %(default)s]"),
action='store_true',
)
parser.add_argument(
'--start-postgresql',
help="Start a Postgresql docker",
action='store_true',
)
network_group = parser.add_mutually_exclusive_group()
network_group.add_argument(
'--host-network',
help="Use host network [default: %(default)s]",
action='store_false',
)
network_group.add_argument(
'--docker-network',
help=("Use docker network (works better with"
" python-netifaces installed) [default: %(default)s]"),
action='store_true',
)
# TODO detect that user is member of docker group
# TODO add a way to add options to docker
# TODO add a way to add arg to odoo
nmspc = parser.parse_args()
verbose = nmspc.verbose
if not verbose:
logging.basicConfig(level=logging.WARN)
if verbose == 1:
logging.basicConfig(level=logging.INFO)
if verbose and verbose > 1:
logging.basicConfig(level=logging.DEBUG)
db_user = nmspc.db_user
db_password = nmspc.db_password
use_host_network = nmspc.host_network
start_postgresql = nmspc.start_postgresql
if project_name == 'odoo_scripts':
logging.fatal(
"You must run this script from the super project"
" (./odoo_scripts/docker_dev_start.py)")
return
c = ConfigParser.ConfigParser()
logging.debug('setup file path %s', setup_path)
# TODO test that setup file exists
c.read(setup_path)
addon_dirs = c.get('odoo_scripts', 'addon_dirs', '').split()
logging.debug("addon directories: %s", addon_dirs)
registry = (
c.has_section('odoo_scripts') and
c.has_option('odoo_scripts', 'registry') and
c.get('odoo_scripts', 'registry')) or 'dockerhub.xcg.io'
project = (
c.has_section('odoo_scripts') and
c.has_option('odoo_scripts', 'image') and
c.get('odoo_scripts', 'image')) or os.path.basename(project_path)
odoo_type = (
c.has_section('odoo_scripts') and
c.has_option('odoo_scripts', 'odoo_type') and
c.get('odoo_scripts', 'odoo_type')) or 'odoo7'
image = "%s/%s:latest" % (registry, project)
postgresql_version = (
c.has_section('odoo_scripts') and
c.has_option('odoo_scripts', 'postgresql_version') and
c.get('odoo_scripts', 'postgresql_version')) or '9.6'
logging.debug("Docker image: %s", image)
# detect if docker image already exists
docker_client = docker.Client(base_url='unix://var/run/docker.sock')
image_list = docker_client.images(name=image, quiet=True)
if not image_list:
logging.warn("Image %s does not exist", image)
else:
logging.info("Test image found")
# options is only used with subprocess call
options = [
'--name',
project_name,
'--rm',
'--publish',
'8069:8069',
'--tty',
'--interactive',
]
binds = []
arg = [
'start',
]
if db_user:
arg.append('--db_user %s' % db_user)
if db_password:
arg.append('--db_password %s' % db_user)
if nmspc.update:
arg.append('-u %s' % nmspc.update)
arg.append('--i18n-overwrite')
if nmspc.test:
arg.append('-u %s' % nmspc.test)
arg.append('--test-enable')
if odoo_type == 'odoo7':
arg.append('--log-level=test')
arg.append('--stop-after-init')
if nmspc.test_default:
test_modules = c.get('odoo_scripts', 'module_list_tests').split() or []
str_modules = ','.join(test_modules)
arg.append('-u %s' % str_modules)
arg.append('--test-enable')
if odoo_type == 'odoo7':
arg.append('--log-level=test')
arg.append('--stop-after-init')
if nmspc.database:
arg.append('-d %s' % nmspc.database)
if nmspc.install or nmspc.install_default:
modules_to_install = []
if nmspc.install:
modules_to_install.extend(nmspc.install.split(','))
if nmspc.install_default:
modules_to_install.extend((
c.has_section('odoo_scripts') and
c.has_option('odoo_scripts', 'module_list') and
c.get('odoo_scripts', 'module_list').split()) or [])
if modules_to_install:
arg.append('-i')
arg.append(','.join(modules_to_install))
if nmspc.without_demo:
arg.append('--without-demo %s' % nmspc.database)
# auto detect local ip
if use_host_network:
local_ip = '127.0.0.1'
options.append('--network')
options.append('host')
else:
local_ip = None
try:
from netifaces import ifaddresses, AF_INET # apt python-netifaces
iface_name = 'docker0'
addresses = [
i['addr']
for i in ifaddresses(
iface_name
).setdefault(AF_INET, [{'addr': 'No IP addr'}])
]
if addresses:
local_ip = addresses[0]
except ImportError:
logging.warn("Consider installing python netifaces"
" to ease local IP detection")
if not local_ip:
import socket
logging.info('Contacting Google Public DNS to find our IP')
local_ip = [
(
s.connect(('8.8.8.8', 53)),
s.getsockname()[0],
s.close()
) for s in [
socket.socket(socket.AF_INET, socket.SOCK_DGRAM)]
][0][1]
logging.debug('IP found %s', local_ip)
arg.append('--db_host')
arg.append(local_ip)
if nmspc.dbport:
arg.append("--db_port")
arg.append(nmspc.dbport)
# auto detect local conf
local_conf_dir = os.path.join('conf', 'dev')
local_conf_path = os.path.join(local_conf_dir, 'odoo.conf')
user = db_user
password = db_password
if os.path.isfile(local_conf_path):
logging.info('Local configuration file found: %s' % local_conf_path)
binds.append('%s:/opt/odoo/etc' % os.path.join(
project_path, local_conf_dir))
cp_local = ConfigParser.ConfigParser()
cp_local.read(local_conf_path)
if not user:
user = cp_local.get('options', 'db_user', 'pg')
if not password:
password = cp_local.get(
'options',
'db_password',
'THIS-IS-NOT-USED-DONOT-CHANGE',
)
# data volume handling
if odoo_type != 'odoo7':
data_volume_name = '{}_data'.format(project_name)
logging.debug('Using data volume %s', data_volume_name)
volume = getVolume(docker_client, data_volume_name)
# make sure the permission in the volume are correct
mount_opts = '{}:/mnt/data'.format(data_volume_name)
# TODO replace by something cleaner if possible
call([
'docker', 'run', '--rm', '-v',
mount_opts, 'busybox', 'chmod', '777', '/mnt/data',
])
binds.append('{}:/mnt/data'.format(data_volume_name))
arg.append('--data-dir /mnt/data')
else:
logging.debug('No data volume for this odoo version')
# avoid the duplication of unbind volumes with all addons
for extra_volume in ['additional_addons', 'var', 'data']:
volume_name = '{}_opt_odoo_{}'.format(project_name, extra_volume)
# volume = getVolume(docker_client, volume_name)
binds.append('{}:/opt/odoo/{}'.format(volume_name, extra_volume))
if start_postgresql:
pg_repository = 'xcgd/postgresql'
pg_image = '{}:{}'.format(pg_repository, postgresql_version)
name = 'pg{}'.format(postgresql_version)
docker_client.pull(repository=pg_repository, tag=postgresql_version)
pg_data_volume_name = 'postgresql_{}-{}'.format(
postgresql_version, project_name)
host_pg_port = 5433
# volume = getVolume(docker_client, pg_data_volume_name)
host_config = docker_client.create_host_config(
binds=['{}:/var/lib/postgresql'.format(pg_data_volume_name)],
port_bindings={5432: host_pg_port},
)
logging.debug('Creating postgresql container')
pg = docker_client.create_container(
image=pg_image,
host_config=host_config,
name=name,
)
logging.debug('Starting postgresql container')
docker_client.start(pg.get('Id'))
# give pg the time to start up
import time
time.sleep(5)
arg.append('--db_port')
arg.append(str(host_pg_port))
# Check that connection can be done, try to create user if asked to
# TODO: handle the case where the database is still starting up
# TODO: (and remove the sleep)
try:
if local_ip or start_postgresql:
port = host_pg_port if start_postgresql else None
_ = connect(
user=user,
password=password,
database='postgres',
host=local_ip,
port=port,
)
else:
_ = connect(
user=user,
password=password,
database='postgres',
)
except OperationalError as exception:
if nmspc.create_user:
logging.info('Cannot connect to database with user %s', user)
logging.info(exception)
logging.info('Creating user %s', user)
if start_postgresql:
connection = connect(
user='pg',
database='postgres',
host=local_ip,
port=host_pg_port,
)
else:
loginname = pwd.getpwuid(os.getuid())[0]
connection = connect(user=loginname, database='postgres')
with connection.cursor() as cursor:
# not injection safe but you are on your own machine
# with already full access to db
cursor.execute(
'CREATE ROLE %s LOGIN CREATEDB PASSWORD %%s' % user,
(password,))
connection.commit()
connection.close()
else:
logging.warn('Cannot connect to database with user %s', user)
logging.warn(exception)
# volume magic
for addons_dir in addon_dirs:
binds.append(
'%s:/mnt/addons/%s' % (
os.path.realpath(os.path.join(project_path, addons_dir)),
addons_dir))
# also check directory for symbolic links
for subdir, dirs, files in os.walk(
os.path.realpath(os.path.join(project_path, addons_dir))
):
for directory in dirs:
if os.path.islink(os.path.join(subdir, directory)):
binds.append(
'%s:/mnt/addons/%s' % (
os.path.realpath(os.path.join(
project_path, addons_dir, directory)),
os.path.join(addons_dir, directory))
)
all_addons_dir = [
'/opt/odoo/sources/odoo/addons',
]
all_addons_dir.extend(
'/mnt/addons/%s' % addons_dir for addons_dir in addon_dirs)
arg.append('--addons-path')
arg.append(','.join(all_addons_dir))
if start_postgresql:
# use docker-py to be able to stop both the database and odoo
# add volumes
odoo_host_config = docker_client.create_host_config(
binds=binds,
port_bindings={8069: 8069},
network_mode='host' if use_host_network else 'bridge',
)
logging.debug('Creating odoo container')
odoo = docker_client.create_container(
name=project_name, host_config=odoo_host_config,
image=image, command=arg, tty=True)
logging.debug('Starting odoo container')
docker_client.start(odoo.get('Id'))
def signal_handler(code, frame):
if code == signal.SIGINT:
logging.debug('You pressed Ctrl+C!')
if isRunning(docker_client, odoo.get('Id')):
logging.info('Stopping odoo')
docker_client.stop(odoo.get('Id'))
logging.info('Removing container odoo')
docker_client.remove_container(odoo.get('Id'))
if start_postgresql:
logging.info('Stopping postgresql')
docker_client.stop(pg.get('Id'))
logging.info('Removing container postgresql')
docker_client.remove_container(pg.get('Id'))
sys.exit(0)
# TODO add a kill of pg when crashing
signal.signal(signal.SIGINT, signal_handler)
logging.info('Press Ctrl+C to quit')
# print docker logs of odoo
stream = docker_client.logs(odoo.get('Id'), stream=True, follow=True)
while isRunning(docker_client, odoo.get('Id')):
try:
for log in stream:
sys.stdout.write(log)
except ConnectionError:
# If there is no log for some time requests throw some errors
# we ignore them
pass
# Clean up, just in case
signal_handler(signal.SIGINT, None)
# TODO add handling of signal to restart odoo
else:
# use call to allow usage of pdb
for bind in binds:
options.append('--volume')
options.append(bind)
cmd = ['docker', 'run']
cmd.extend(options)
cmd.append(image)
cmd.extend(arg)
logging.debug(cmd)
call(cmd)
def getVolume(docker_client, data_volume_name):
"""Return the volume passed in parameter, creating it if it does not exists
"""
volumes = docker_client.volumes(filters={'name': data_volume_name})
if volumes['Volumes']:
logging.debug('Volume %s already exist', data_volume_name)
return volumes['Volumes'][0]
else:
logging.debug('Creating volume %s', data_volume_name)
return docker_client.create_volume(name=data_volume_name)
def isRunning(docker_client, container_id):
"""Return true if the container is still running
"""
return len(docker_client.containers(filters={'id': container_id}, quiet=True))
if __name__ == "__main__":
return_code = main()
if return_code:
exit(return_code)