improve failure and error detections, mainly for odoo 13

NEWS.rst

@@ -9,6 +9,8 @@
 
 Expose ``logging_from_verbose``.
 
+Change failure and error detection in ``run_tests``.
+
 3.2
 ---
 

do_tests

@@ -102,6 +102,7 @@
     echo "[${YELLOW}INFO${RESET}] Create database '$dbname' with owner $dbowner"
     createdb $PG_HOST $PG_USER -O $dbowner $dbname --no-password || { echo ${RED}FATAL${RESET} - Cannot create test database ; exit 1; }
 fi
+# TODO should also add all extensions from setup.cfg listed in pg.extensions
 if [[ "$unaccent" == "True" ]];
 then
     echo "[${YELLOW}INFO${RESET}] Add unaccent to '$dbname'"

run_tests

@@ -48,10 +48,27 @@
 # Catch both stdout & stderr here as Odoo 10 uses stderr, unlike previous Odoo versions.
 $here/start $update -d $dbname $db_user_param  --test-enable $loglevel --max-cron=0 --stop-after-init $* 2>&1 | tee $logfile | sed -e 's/\(.*\) \(INFO\) \(.*\)/\o033[2m\1\o033[22m \o033[32m\o033[7m\2\o033[27m \3\o033[39m/' -e 's/\(.*\) \(DEBUG\) \(.*\)/\o033[2m\1\o033[22m \o033[33m\o033[7m\2\o033[27m \2\o033[39m/' -e 's/\(.*\) \(WARNING\) \(.*\)/\o033[2m\1\o033[22m \o033[33m\o033[7m\2\o033[27m \3\o033[39m/' -e 's/\(.*\) \(ERROR\) \(.*\)/\o033[2m\1\o033[22m \o033[31m\o033[7m\2\o033[27m \3\o033[0m/' -e 's/ FAILED/ \o033[41m\o033[97m\o033[1mFAILED\o033[0m/' -e 's/ FAIL\(.*\)/ \o033[41m\o033[97m\o033[1mFAIL\o033[49m\o033[31m\1\o033[0m/' -e 's/\(.*\) \(CRITICAL\) \(.*\)/\o033[2m\1\o033[22m \o033[33m\o033[7m\2\o033[27m \3\o033[39m/' -e 's/\(.*\) \(TEST\) \(.*\) \(ERROR\)\(.*\)/\o033[2m\1\o033[22m \o033[34m\o033[7m\2\o033[27m \3\o033[0m \o033[41m\o033[97m\o033[1m\4\o033[49m\o033[31m\5\o033[0m/' -e 's/\(.*\) \(TEST\) \(.*\) \(OK\)/\o033[2m\1\o033[22m \o033[34m\o033[7m\2\o033[27m \3\o033[0m \o033[32m\o033[7m\4\o033[27m\o033[39m/' -e 's/\(.*\) \(TEST\) \(.*\)/\o033[2m\1\o033[22m \o033[34m\o033[7m\2\o033[27m \3 \o033[0m/'
 start_status=$pipestatus[1]
-ok=$(grep ' OK' $logfile -c)
-# TODO detect yaml tests failures
-failed=$(grep ' FAILED\| CRITICAL\| ERROR [^ ]* openerp.modules.module: Can not .import openerp\.addons\.\| TEST .* ERROR:\|Failed to initialize database' $logfile -c)
-warnings=$(grep ' FAILED' $logfile -c)
+# Valid with odoo 7, 11, 13 and probably the rest too
+tests_ran=$(grep -P 'o(penerp|doo).modules.module: Ran \K([[:digit:]]+)' $logfile -o | perl -ne '$sum += $_ } { print $sum')
+tests_failures=$(grep -P 'o(penerp|doo).modules.module: Module [a-z0-9_]+: \K([[:digit:]]+)' $logfile -o | perl -ne '$sum += $_ } { print $sum')
+# that might be doable in the perl above
+if [ -z "$tests_failures" ];
+then
+    tests_failures=0
+fi
+tests_errors=$(grep -P 'o(penerp|doo).modules.module: Module [a-z0-9_]+: [[:digit:]]+ failures, \K([[:digit:]]+)' $logfile -o | perl -ne '$sum += $_ } { print $sum')
+if [ -z "$tests_errors" ];
+then
+    tests_errors=0
+fi
+if [ $ODOO_TYPE != 'odoo13' ];
+then
+    ok=$(grep ' OK' $logfile -c)
+fi
+# odoo 13 uses FAIL as the error message, in the form: FAIL: TestReorderingRule.test_reordering_rule
+failed=$(grep ' FAIL\(ED\|:\)\| CRITICAL\| ERROR [^ ]* openerp.modules.module: Can not .import openerp\.addons\.\| TEST .* ERROR:\|Failed to initialize database\| ERROR \([[:alnum:]]\|_\)* .\+\.test_.\+ ERROR: ' $logfile -c)
+warnings=$(grep ' WARNING ' $logfile -c)
+# XXX Not sure this is needed anymore with the tests_failures/tests_errors detection
 if [[ $failed -eq 0 ]];
 then
     # simple way to detect yaml failure
@@ -59,7 +76,11 @@
 fi
 echo ''
 echo '\e[2m*****\e[22m \e[1mTest results\e[21m \e[2m*****\e[22m'
-echo "$ok modules passing, $failed modules failing"
+echo "Ran $tests_ran tests, $tests_failures failures, $tests_errors errors"
+if [ $ODOO_TYPE != 'odoo13' ];
+then
+    echo "$ok modules passing"
+fi
 echo ''
 
 if [[ $ok -gt 0 ]];
@@ -72,6 +93,6 @@
 
 if [[ $failed -gt 0 ]];
 then
-    echo -e '\033[31mFAILED\033[0m:'
+    echo -e '\033[31mFailures\033[0m:'
 
     # Highlight the failure matches in red (31).
@@ -76,5 +97,5 @@
 
     # Highlight the failure matches in red (31).
-    GREP_COLOR='0;31' grep --color=always "CRITICAL\|FAILED\|FAIL:" $logfile
+    GREP_COLOR='0;31' grep --color=always "CRITICAL\|FAIL\(ED\|:\)" $logfile
     GREP_COLOR='0;31' grep --color=always "ERROR [^ ]* openerp\.tools\.yaml_import:" $logfile
     GREP_COLOR='0;31' grep --color=always "ERROR [^ ]* openerp.modules.module: Can not \`import openerp\.addons\..*\`" $logfile
@@ -79,3 +100,4 @@
     GREP_COLOR='0;31' grep --color=always "ERROR [^ ]* openerp\.tools\.yaml_import:" $logfile
     GREP_COLOR='0;31' grep --color=always "ERROR [^ ]* openerp.modules.module: Can not \`import openerp\.addons\..*\`" $logfile
+    # this one is for odoo 7
     GREP_COLOR='0;31' grep --color=always "TEST .* ERROR:" $logfile
@@ -81,4 +103,6 @@
     GREP_COLOR='0;31' grep --color=always "TEST .* ERROR:" $logfile
+    # this one is for odoo 13 for tests that produces errors (failures are matched with FAIL: )
+    GREP_COLOR='0;31' grep --color=always "ERROR \([[:alnum:]]\|_\)* .\+\.test_.\+ ERROR:" $logfile
 fi
 
 echo ''
@@ -88,6 +112,11 @@
     echo "[${RED}FATAL${RESET}] Tests failed (exit ${start_status})"
     exit $start_status
 fi
+if [[ $tests_failures -gt 0 ]] || [[ $tests_errors -gt 0 ]];
+then
+    echo "[${RED}FATAL${RESET}] Tests failed (found failure and errors in Ran N failures, M errors)"
+    exit 42
+fi
 if [[ $failed -gt 0 ]];
 then
     echo "[${RED}FATAL${RESET}] Tests failed (found failure with grep)"