odoo_scripts_common

#!/bin/zsh
# vim: set shiftwidth=2 softtabstop=2:
#
# Common functions and env var.
#
# This file is meant to be sourced.

# assume we are called from the project home
project_home=$PWD
project_name=$(basename "$project_home")

if ! type python3 > /dev/null; then
    python="python2"
else
    python="python3"
fi

# handy colorized variables
esc=$(printf '\033')
RESET="${esc}[0m"
DEBUG="[${esc}[36mDEBUG${RESET}]"
INFO="[${esc}[34mINFO ${RESET}]"
WARN="[${esc}[33mWARN ${RESET}]"
FATAL="[${esc}[41m${esc}[37mFATAL${RESET}]"
GREEN="${esc}[32m"
OK="[${GREEN} OK${RESET}  ]"
RED="${esc}[31m"
KO="[${esc}[41m${esc}[37m KO  ${RESET}]"
WHITE_BOLD="${esc}[1;37m"

function read_configuration_key () {
  # Read a configuration key from any ini file
  # first argument is file path
  # second argument is section
  # third argument is the key to use
  # fourth argument is default value, if any
  # fifth argument is separator, default to a space
  file=$1
  section=$2
  configuration_key=$3
  default_value=$4
  separator=${5- }
  echo $($python -B -c "from six.moves import configparser ; c = configparser.ConfigParser() ; c.read('${file}') ; print('$separator'.join(c.get('$section', '$configuration_key').split()) if c.has_option('$section', '$configuration_key') else '$default_value')")
}

function read_odoo_scripts_configuration_key () {
  # Return configuration value from odoo_scripts configuration file
  # first argument is the key to use
  # second argument is default value, if any
  configuration_key=${1}
  default_value=$2
  read_configuration_key "${project_home}/setup.cfg" odoo_scripts $configuration_key $default_value
}

function read_odoo_scripts_expandable_configuration_key () {
  # return configuration value by appending any expanded values from defined super projects. glob key as appropriate, for module for example.
  # first argument is the key to use
  # second optional argument is the separator to use (default to comma)
  configuration_key=${1}
  separator=${2-,}
  # read expanded configurations, only done with projects in python 3
  if type python3 > /dev/null; then
    configuration_value=$(python3 -B -c "from odoo_scripts.config import Configuration ; c = Configuration('$project_home') ; print('$separator'.join(c.$configuration_key))")
  else
    # read only local values
    configuration_value=$($python -B -c "from six.moves import configparser ; import os ; c = configparser.ConfigParser() ; c.read(os.path.join('$project_home', 'setup.cfg')) ; print('$separator'.join(c.get('odoo_scripts', '$configuration_key').split() if c.has_option('odoo_scripts', '$configuration_key') else []))")
    if [[ $configuration_key == modules ]]; then
      configuration_value=$($python -B -c "import os;from glob import glob;print('$separator'.join([file for path in '${configuration_value}'.split('$separator') for file in glob(path) if os.path.isdir(file)]))")
    fi
  fi
  echo $configuration_value
}

# function to be used in pipes
function colorize () {
  cat | sed -E \
  -e "s/(.*) (TEST) (.*) (ERROR)(.*)/${esc}[2m\1${esc}[22m ${esc}[34m${esc}[7m\2${esc}[27m \3${esc}[0m ${esc}[41m${esc}[97m${esc}[1m\4${esc}[49m${esc}[31m\5${esc}[0m/" \
  -e "s/^([[:digit:]]{4,}-[[:digit:]]{2}-[[:digit:]]{2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2},[[:digit:]]{3}) ([[:digit:]]+) (TEST) ([^ ]+) ([^:]+:) (OK)?(.*)/\1 ${esc}[2m\2${esc}[22m ${esc}[36m${esc}[7m\3${esc}[0m ${esc}[2m\4${esc}[22m ${esc}[38;5;39m\5 ${esc}[36m${esc}[7m\6${esc}[27m${esc}[39m${esc}[36m\7$RESET/" \
  -e "s/^([[:digit:]]{4,}-[[:digit:]]{2}-[[:digit:]]{2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2},[[:digit:]]{3}) ([[:digit:]]+) (INFO) ([^ ]+) (odoo\.addons\.[^:]*\.tests\.[^:]*:) (Starting .*)/\1 ${esc}[2m\2${esc}[22m ${esc}[36m${esc}[7m\3${esc}[0m ${esc}[2m\4${esc}[22m ${esc}[38;5;39m\5 ${esc}[36m\6$RESET/" \
  -e "s/^([[:digit:]]{4,}-[[:digit:]]{2}-[[:digit:]]{2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2},[[:digit:]]{3}) ([[:digit:]]+) (INFO) ([^ ]+) (odoo\.(tests\.(runner|result|stats)|modules\.module):) (.*)/\1 ${esc}[2m\2${esc}[22m ${esc}[36m${esc}[7m\3${esc}[0m ${esc}[2m\4${esc}[22m ${esc}[38;5;39m\5 ${esc}[36m\8$RESET/" \
  -e "s/^([[:digit:]]{4,}-[[:digit:]]{2}-[[:digit:]]{2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2},[[:digit:]]{3}) ([[:digit:]]+) (INFO) ([^ ]+) (o(doo|penerp)\.(modules\.loading|tools\.translate)|openerp\.addons\.base\.ir\.ir_translation|odoo\.addons\.base\.models\.ir_module): (.*)/\1 ${esc}[2m\2${esc}[22m ${esc}[32m${esc}[7m\3${esc}[0m ${esc}[2m\4${esc}[22m ${esc}[38;5;22m\5: ${esc}[38;5;64m\8$RESET/" \
  -e "s/^([[:digit:]]{4,}-[[:digit:]]{2}-[[:digit:]]{2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2},[[:digit:]]{3}) ([[:digit:]]+) (INFO) ([^ ]+) ([^:]*:) (=+)/\1 ${esc}[2m\2${esc}[22m ${esc}[32m${esc}[7m\3${esc}[0m ${esc}[2m\4${esc}[22m ${esc}[38;5;28m\5 ${esc}[31m\6$RESET/" \
  -e "s/^([[:digit:]]{4,}-[[:digit:]]{2}-[[:digit:]]{2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2},[[:digit:]]{3}) ([[:digit:]]+) (INFO) ([^ ]+) ([^:]*:)( ?[^=].*)/\1 ${esc}[2m\2${esc}[22m ${esc}[32m${esc}[7m\3${esc}[0m ${esc}[2m\4${esc}[22m ${esc}[38;5;28m\5${esc}[32m\6$RESET/" \
  -e "s/^([[:digit:]]{4,}-[[:digit:]]{2}-[[:digit:]]{2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2},[[:digit:]]{3}) ([[:digit:]]+) (DEBUG) ([^ ]+) ([^:]*:) (.*)/\1 ${esc}[2m\2${esc}[22m ${esc}[34m${esc}[7m\3${esc}[0m ${esc}[2m\4${esc}[22m ${esc}[38;5;24m\5 ${esc}[34m\6$RESET/" \
  -e "s/^([[:digit:]]{4,}-[[:digit:]]{2}-[[:digit:]]{2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2},[[:digit:]]{3}) ([[:digit:]]+) (WARNING) ([^ ]+) ([^:]*:) (.*)/\1 ${esc}[2m\2${esc}[22m ${esc}[33m${esc}[7m\3${esc}[0m ${esc}[2m\4${esc}[22m ${esc}[38;5;94m\5 ${esc}[33m\6$RESET/" \
  -e "s/^([[:digit:]]{4,}-[[:digit:]]{2}-[[:digit:]]{2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2},[[:digit:]]{3}) ([[:digit:]]+) (ERROR) ([^ ]+) (odoo\.(tests\.(runner|result|stats)|modules\.module):) (FAIL)?(.*)/\1 ${esc}[2m\2${esc}[22m ${esc}[35m${esc}[7m\3${esc}[0m ${esc}[2m\4${esc}[22m ${esc}[38;5;88m\5 ${esc}[35m${esc}[7m\8${esc}[27m\9$RESET/" \
  -e "s/^([[:digit:]]{4,}-[[:digit:]]{2}-[[:digit:]]{2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2},[[:digit:]]{3}) ([[:digit:]]+) (ERROR) ([^ ]+) ([^:]*:) (FAIL)?(.*)/\1 ${esc}[2m\2${esc}[22m ${esc}[31m${esc}[7m\3${esc}[0m ${esc}[2m\4${esc}[22m ${esc}[38;5;88m\5 ${esc}[31m${esc}[7m\6${esc}[27m\7$RESET/" \
  -e "s/^([[:digit:]]{4,}-[[:digit:]]{2}-[[:digit:]]{2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2},[[:digit:]]{3}) ([[:digit:]]+) (CRITICAL) ([^ ]+) ([^:]*:) (.*)/\1 ${esc}[2m\2${esc}[22m ${esc}[1;31m${esc}[7m\3${esc}[0m ${esc}[2m\4${esc}[22m ${esc}[31m\5 ${esc}[1;31m\6$RESET/" \
  -e "s/ FAILED/ ${esc}[41m${esc}[97m${esc}[1mFAILED${esc}[0m/" \
}

# The analyze method does several things:
# - print a resume of failure, errors and passes
# - create a metrics file with the number of tests (when the CI environment variable equals true)
# - return an error code based on the parsing of the log file. This is meant to be used on Odoo that did not exit with
#   a non-zero value when tests fail.
# The method takes the logfile as its only argument.
function analyze () {
  logfile=$1
  # Odoo >= 14 detection
  # Odoo 17 uses odoo.tests.result rather than odoo.tests.runner
  if [ $(grep -P 'odoo\.tests\.(runner|result): ([[:digit:]]+) failed, [[:digit:]]+ error\(s\) of [[:digit:]]+ tests when loading database' $logfile --count) -gt 0 ];
  then
    # Valid with odoo 14, 15, 16, 17
    # Add junit tests results to qunit tests results
    tests_ran=$(grep -P 'odoo\.tests\.(runner|result): ([[:digit:]]+) failed, [[:digit:]]+ error\(s\) of \K[[:digit:]]+' $logfile -o)
    quilt_tests_ran=$(grep -P ': ([[:digit:]]+) / [[:digit:]]+ tests failed.' $logfile --only-matching | sed -n "s,: [^ ]* / \([^ ]\),\1,p" | perl -ne '$sum += $_ } { print $sum')
    tests_failures=$(grep -P 'odoo\.tests\.(runner|result): \K([[:digit:]]+)' $logfile --only-matching)
    quilt_tests_failures=$(grep -P ': ([[:digit:]]+) / [[:digit:]]+ tests failed.' $logfile --only-matching | sed -n "s,: \([^ ]*\) /.*,\1,p" | perl -ne '$sum += $_ } { print $sum')
    tests_errors=$(grep -P 'odoo\.tests\.(runner|result): ([[:digit:]]+) failed, \K[[:digit:]]+' $logfile -o)
    import_errors_number=0
  else
    # Valid with odoo 7, 11, 13
    tests_ran=$(grep -P 'o(penerp|doo).(modules.module|[^ ]+): Ran \K([[:digit:]]+)' $logfile -o | perl -ne '$sum += $_ } { print $sum')
    if [ -z "$tests_ran" ];
    then
      tests_ran=0
    fi
    tests_failures=$(grep -P 'o(penerp|doo).(modules.module|[^ ]+): Module [a-zA-Z0-9._-]+: \K([[:digit:]]+)' $logfile -o | perl -ne '$sum += $_ } { print $sum')
    # that might be doable in the perl above
    if [ -z "$tests_failures" ];
    then
      tests_failures=0
    fi
    tests_errors=$(grep -P 'o(penerp|doo).(modules.module|[^ ]+): Module [a-z0-9_]+: [[:digit:]]+ failures, \K([[:digit:]]+)' $logfile -o | perl -ne '$sum += $_ } { print $sum')
    if [ -z "$tests_errors" ];
    then
      tests_errors=0
    fi
    import_errors=$(grep -P 'o(penerp|doo).(modules.module|[^ ]+): Can not `import [a-z0-9_]+`.' $logfile)
    import_errors_number=$(grep -P 'o(penerp|doo).(modules.module|[^ ]+): Can not `import [a-z0-9_]+`.' $logfile -c)
    if [[ "$import_errors_number" -ne 0 ]];
    then
      test_errors=$((test_errors+import_errors_number))
    fi
  fi
  # create an OpenMetrics metrics file
  if [ "$CI" = "true" ];
  then
    echo -e "odoo_scripts_tests_ran $tests_ran\nodoo_scripts_tests_failures $tests_failures\nodoo_scripts_tests_error $tests_error\n# EOF\n" > metrics
  fi
  if [ "$ODOO_TYPE" = 'odoo7' ] || [ "$ODOO_TYPE" = 'odoo8' ] || [ "$ODOO_TYPE" = 'odoo9' ] || [ "$ODOO_TYPE" = 'odoo10' ] || [ "$ODOO_TYPE" = 'odoo11' ];
  then
    ok=$(grep ' OK' $logfile -c)
  else
    ok=0
  fi
  # odoo 13 uses FAIL as the error message, in the form: FAIL: TestReorderingRule.test_reordering_rule
  failed=$(grep ' FAIL\(ED\|:\)\| CRITICAL\| ERROR [^ ]* openerp.modules.module: Can not .import openerp\.addons\.\| TEST .* ERROR:\|Failed to initialize database\| ERROR \([[:alnum:]]\|_\)* .\+\.test_.\+ ERROR: \|invalid module names, ignored:' $logfile -c)
  if [[ -n "$import_errors_number" ]];
  then
    failed=$((failed+$import_errors_number))
  fi
  warnings=$(grep ' WARNING ' $logfile -c)
  # XXX Not sure this is needed anymore with the tests_failures/tests_errors detection
  if [[ $failed -eq 0 ]];
  then
    # simple way to detect yaml failure
    failed=$(grep 'o(penerp|doo).modules.loading: At least one test failed when loading the modules.' $logfile -c)
  fi
  echo ''
  echo "${esc}[9m-----$RESET ${esc}[7m${esc}[1mTest results$RESET ${esc}[9m-----$RESET"
  printf "$tests_failures failed, $tests_errors error(s) of $tests_ran tests"
  if [ "$ODOO_TYPE" = 'odoo7' ] || [ "$ODOO_TYPE" = 'odoo8' ] || [ "$ODOO_TYPE" = 'odoo9' ] || [ "$ODOO_TYPE" = 'odoo10' ];
  then
    echo " ($ok modules passing)"
  else
    if [ "$ODOO_TYPE" = 'odoo11' ];
    then
      echo " ($ok test files passing)"
    else
      echo ''
    fi
  fi
  echo ''

  if [[ $failed -gt 0 ]];
  then
    echo -e "${RED}Failures${RESET}:"

    # Highlight the failure matches in red (31).
    GREP_COLORS='mt=0;31' grep "CRITICAL\|FAIL\(ED\|:\)" $logfile
    GREP_COLORS='mt=0;31' grep --color=always "ERROR [^ ]* openerp\.tools\.yaml_import:" $logfile
    GREP_COLORS='mt=0;31' grep --color=always "ERROR [^ ]* openerp.modules.module: Can not \`import openerp\.addons\..*\`" $logfile
    # this one is for odoo 7
    GREP_COLORS='mt=0;31' grep --color=always "TEST .* ERROR:" $logfile
    # this one is for odoo 13 for tests that produces errors (failures are matched with FAIL: )
    GREP_COLORS='mt=0;31' grep --color=always "ERROR [^ ]* .\+\.test_.\+ ERROR:" $logfile
    GREP_COLORS='mt=0;31' grep --color=always "invalid module names, ignored" $logfile
    if [[ -n "$import_errors" ]];
    then
      echo "$import_errors"
    fi
  fi

  echo ''
  if [[ $tests_failures -gt 0 ]] || [[ $tests_errors -gt 0 ]];
  then
    echo "$FATAL Tests failed (found failure and errors in Ran N failed/failures, M errors)"
    return 42
  fi
  if [[ $failed -gt 0 ]];
  then
    echo "$FATAL Tests failed (found failure with grep)"
    return 20
  fi
  if [[ "${FAIL_ON_WARNING:-False}" != "False" ]];
  then
    if [[ $warnings -gt 0 ]];
    then
      echo "$FATAL Tests failed ($warnings WARNING found)"
      return 2
    fi
  fi
  return 0
}

# Make ODOO_TYPE available to files that sources this file.
ODOO_TYPE=${ODOO_TYPE:-$(read_odoo_scripts_configuration_key odoo_type odoo8)}