rewrote expanding to allow blacklisting and add documentation

NEWS.rst

@@ -2,6 +2,11 @@
 History
 =======
 
+7.0.0
+-----
+
+Rewrote expanding to allow blacklisting. It should be simpler to add new options.
+
 6.0.0
 -----
 
@@ -10,6 +15,7 @@
 Add .expand to several options so that a super project can reuse other super projects.
 
 Fix --odoo-help in docker_dev_start to avoid requiring a database.
+
 5.0.0
 -----
 

README.rst

@@ -225,6 +225,24 @@
 postgresql_version
     version of postgresql to use. defaults to 9.6.
 
+It is possible to reuse another directory containing `setup.cfg`_. These type of keys can be used:
+
+.. code-block:: INI
+
+    # this key allows to define the path to the file to reuse (to expand on)
+    # name can be different from the directory name
+    # this key is mandatory
+    expand.name.path = path/to/directory
+    # this key indicates what keys to reuse from the subdirectories
+    # this is not required, but will default to all if not provided
+    # comma separated list of values.
+    expand.base.read = modules,module_list
+    # this key is to allow blacklisting some subdirectories
+    # the name is the same as the previous key
+    # if there is no blacklisting, no need to include it
+    # the value is a python regexp
+    expand.name.blacklist = regexp
+
 Completion files for zsh
 ========================
 

odoo_scripts/config.py

@@ -3,4 +3,5 @@
 import configparser
 import logging
 import os
+import re
 import sys
@@ -6,4 +7,5 @@
 import sys
+from collections import defaultdict
 from glob import glob
 from typing import List, Optional
 
@@ -7,6 +9,8 @@
 from glob import glob
 from typing import List, Optional
 
+SECTION = "odoo_scripts"
+
 _logger = logging.getLogger(__name__)
 
 
@@ -27,6 +31,8 @@
     At the moment, only read from `setup.cfg`.
     """
 
-    def __init__(self, base_path: str = None):
+    def __init__(
+        self, path: str = None, blacklist: str = None, read: str = None
+    ):
         self._expanded_configuration = dict()
         setup_path = "setup.cfg"
@@ -31,10 +37,9 @@
         self._expanded_configuration = dict()
         setup_path = "setup.cfg"
-        if base_path:
-            setup_path = os.path.join(base_path, setup_path)
-        section = "odoo_scripts"
+        if path:
+            setup_path = os.path.join(path, setup_path)
         config_parser = configparser.ConfigParser()
         if not os.path.exists(setup_path):
             _logger.warning("Missing %s", setup_path)
         else:
             config_parser.read(setup_path)
@@ -36,8 +41,36 @@
         config_parser = configparser.ConfigParser()
         if not os.path.exists(setup_path):
             _logger.warning("Missing %s", setup_path)
         else:
             config_parser.read(setup_path)
+        section = (
+            config_parser[SECTION] if SECTION in config_parser else dict()
+        )
+
+        # read expanded configurations information
+        expanded_info = defaultdict(dict)
+        for key in section:
+            if key.startswith("expand."):
+                remaining_key = key.split(".")[1:]
+                expanded_info[remaining_key[0]][
+                    remaining_key[1]
+                ] = section.get(key)
+
+        for name, value in expanded_info.items():
+            if "path" not in value:
+                _logger.error("Expansion without path for %s", name)
+                raise Exception("Expansion without path for %s" % name)
+            else:
+                if blacklist and re.match(blacklist, value["path"]):
+                    _logger.info(
+                        "Ignoring directory '%s' due to blacklisting",
+                        value["path"],
+                    )
+                else:
+                    if path:
+                        value["path"] = os.path.join(path, value["path"])
+                    # read expanded configurations
+                    self._expanded_configuration[name] = Configuration(**value)
 
         def set_from_glob_values(
             base: Optional[str], glob_values: List[str]
@@ -51,10 +84,9 @@
                         result_set.add(file)
             return result_set
 
-        def get_expanded_configuration(path_name: str):
-            if path_name not in self._expanded_configuration:
-                self._expanded_configuration[path_name] = Configuration(
-                    path_name
-                )
-            return self._expanded_configuration[path_name]
+        def toread(key: str) -> bool:
+            """Return true if value of key is to be read from the file"""
+            if read:
+                return key in read.split(",")
+            return True
 
@@ -60,2 +92,3 @@
 
+        # those keys are path to files (glob compatible)
         for key in ("modules", "dependencies", "other_sources"):
@@ -61,19 +94,16 @@
         for key in ("modules", "dependencies", "other_sources"):
-            # get the values from this configuration
-            values = config_parser.get(section, key, fallback="").split()
-            set_values = set_from_glob_values(base_path, values)
-            # handle expand, getting value from another configuration
-            expand_value = config_parser.get(
-                section, key + ".expand", fallback=None
-            )
-            if expand_value:
-                for element in expand_value.split():
-                    set_values.update(
-                        getattr(get_expanded_configuration(element), key)
-                    )
+            set_values = set()
+            if toread(key):
+                # add any expanded values
+                for config in self._expanded_configuration.values():
+                    set_values.update(getattr(config, key))
+                # get the values from this configuration
+                values = section.get(key, "").split()
+                # XXX eventually add option to not include path
+                set_values.update(set_from_glob_values(path, values))
             setattr(self, key, list(set_values))
 
         def key_format(a_key: str) -> str:
             """Replace invalid characters in an attribute"""
             return a_key.replace("-", "_").replace(".", "_")
 
@@ -74,9 +104,10 @@
             setattr(self, key, list(set_values))
 
         def key_format(a_key: str) -> str:
             """Replace invalid characters in an attribute"""
             return a_key.replace("-", "_").replace(".", "_")
 
+        # those keys are list of values
         for key in ("module_list", "module_list_tests", "pg.extensions"):
             value = list()
             formatted_key = key_format(key)
@@ -80,18 +111,11 @@
         for key in ("module_list", "module_list_tests", "pg.extensions"):
             value = list()
             formatted_key = key_format(key)
-            # add any expanded value
-            expand_value = config_parser.get(
-                section, key + ".expand", fallback=None
-            )
-            if expand_value:
-                for element in expand_value.split():
-                    value.extend(
-                        getattr(
-                            get_expanded_configuration(element), formatted_key
-                        )
-                    )
-            # add local values
-            value += config_parser.get(section, key, fallback="").split()
+            if toread(key):
+                # add any expanded values
+                for config in self._expanded_configuration.values():
+                    value.extend(getattr(config, formatted_key))
+                # add local values
+                value.extend(section.get(key, "").split())
             # then set it
             setattr(self, formatted_key, value)
@@ -96,9 +120,5 @@
             # then set it
             setattr(self, formatted_key, value)
-        for key in ("db_user", "db_password", "load-language"):
-            setattr(
-                self,
-                key_format(key),
-                config_parser.get(section, key, fallback=None),
-            )
+
+        # those keys are single string values, no expand
 
@@ -104,5 +124,6 @@
 
-        self.registry = config_parser.get(
-            section, "registry", fallback="registry.xcg.io"
-        )
+        for key in ("db_user", "db_password", "load-language"):
+            setattr(self, key_format(key), section.get(key, None))
+
+        self.registry = section.get("registry", "registry.xcg.io")
         project_path = os.path.realpath(".")
@@ -108,10 +129,6 @@
         project_path = os.path.realpath(".")
-        self.image = config_parser.get(
-            section, "image", fallback=os.path.basename(project_path)
-        )
-        self.odoo_type = config_parser.get(
-            section, "odoo_type", fallback="odoo7"
-        )
+        self.image = section.get("image", os.path.basename(project_path))
+        self.odoo_type = section.get("odoo_type", "odoo7")
         if self.odoo_type not in (
             "odoo7",
             "odoo8",
@@ -120,12 +137,8 @@
             "odoo13",
         ):
             _logger.warning("Unexpected odoo_type: %s", self.odoo_type)
-        self.postgresql_version = config_parser.get(
-            section, "postgresql_version", fallback="9.6"
-        )
-        self.start_py3o = config_parser.get(
-            section, "start_py3o", fallback="no"
-        ) in ("yes", "true")
+        self.postgresql_version = section.get("postgresql_version", "9.6")
+        self.start_py3o = section.get("start_py3o", "no") in ("yes", "true")
 
 
 def main(argv=None):