Skip to content
Snippets Groups Projects
Commit d7ff7f0b226f authored by Vincent Hatakeyama's avatar Vincent Hatakeyama
Browse files

:sparkles: docker_build_copy: Also cache compilation of modules

parent 0d6fc7636668
No related branches found
No related tags found
1 merge request!199✨ docker_build_copy: download module from forge if available, Multiprocess for checking/compiling/downloading
Hide whitespace changes
Inline Side-by-side
odoo_scripts/docker_build_copy.py
+ 99
− 63
View file @ d7ff7f0b
......@@ -263,6 +263,19 @@
def _compile_package(package_to_compile, target) -> List[str]:
return _compile_packages([package_to_compile], target)
versions = _find_packages_version(package_to_compile, tags_only=False)
content = []
requirements = []
if versions:
cachedirname = compiled_cache_dir(versions[0][0], versions[0][1])
if not os.path.exists(cachedirname):
os.makedirs(cachedirname)
content = os.listdir(cachedirname)
if not content:
with tempfile.TemporaryDirectory() as tmpdirname:
try:
# TODO pip wheel semble plus rapide
# build is not usable directly so use a subprocess.
# Disable flake8, this is just to test that the package is installed.
import build # type: ignore[import] # noqa: F401
......@@ -268,18 +281,27 @@
def _compile_packages(packages_to_compile, target) -> List[str]:
with tempfile.TemporaryDirectory() as tmpdirname:
try:
# TODO pip wheel semble plus rapide
# build is not usable directly so use a subprocess.
# Disable flake8, this is just to test that the package is installed.
import build # type: ignore[import] # noqa: F401
cmd = [
sys.executable,
"-m",
"build",
"--outdir",
tmpdirname,
] + packages_to_compile
cmd = [
sys.executable,
"-m",
"build",
"--outdir",
tmpdirname,
package_to_compile,
]
_logger.debug(" ".join(cmd))
check_call(cmd)
except ImportError:
# pip is not usable directly, it is indicated in its doc to use
# subprocess instead.
cmd = [
sys.executable,
"-m",
"pip",
"wheel",
"--no-deps",
"-w",
tmpdirname,
"--ignore-requires-python",
package_to_compile,
]
_logger.debug(" ".join(cmd))
check_call(cmd)
......@@ -284,30 +306,12 @@
_logger.debug(" ".join(cmd))
check_call(cmd)
except ImportError:
# pip is not usable directly, it is indicated in its doc to use subprocess
# instead.
cmd = [
sys.executable,
"-m",
"pip",
"wheel",
"--no-deps",
"-w",
tmpdirname,
"--ignore-requires-python",
] + packages_to_compile
_logger.debug(" ".join(cmd))
check_call(cmd)
content = os.listdir(tmpdirname)
try:
# Disable flake8, this is just to test that the package is installed
import twine # type: ignore[import] # noqa: F401
cmd = [sys.executable, "-m", "twine", "check", f"{target}/*"]
check_call(cmd)
except ImportError:
_logger.warning("No twine, no check done")
requirements = []
content = os.listdir(tmpdirname)
for file in content:
# LINUX
check_call(["cp", os.path.join(tmpdirname, file), target])
if versions:
# LINUX
check_call(["cp", os.path.join(target, file), cachedirname])
else:
for file in content:
# LINUX
......@@ -312,8 +316,18 @@
for file in content:
# LINUX
check_call(["mv", os.path.join(tmpdirname, file), target])
requirements.append(os.path.join(".", target, file))
check_call(["cp", os.path.join(cachedirname, file), target])
for file in content:
requirements.append(os.path.join(".", target, file))
try:
# Disable flake8, this is just to test that the package is installed
import twine # type: ignore[import] # noqa: F401
cmd = [sys.executable, "-m", "twine", "check", f"{target}/*"]
check_call(cmd)
except ImportError:
_logger.warning("No twine, no check done")
return requirements
def _search_and_download_package(package, orus_api_token, target) -> Union[None, Tuple]:
......@@ -316,8 +330,8 @@
return requirements
def _search_and_download_package(package, orus_api_token, target) -> Union[None, Tuple]:
potential_packages: List[Tuple[str, str]] = _find_packages_to_download(package)
potential_packages: List[Tuple[str, str]] = _find_packages_version(package)
if potential_packages:
return _download_package(package, orus_api_token, potential_packages, target)
return None
......@@ -366,7 +380,7 @@
return None
def _find_packages_to_download(package) -> List[Tuple[str, str]]:
def _find_packages_version(package, tags_only=True) -> List[Tuple[str, str]]:
# Try to download any package with a clear name
toml_file = os.path.join(package, "pyproject.toml")
if os.path.exists(toml_file):
......@@ -388,8 +402,14 @@
)
client = hglib.open(hg_root)
# TODO check if is dirty
tags = client.identify(tags=True).split()
package_list = [
(package_name, tag.decode("UTF-8")) for tag in tags if tag != b"tip"
]
if tags_only:
tags = client.identify(tags=True).split()
package_list = [
(package_name, tag.decode("UTF-8"))
for tag in tags
if tag != b"tip"
]
else:
version = client.identify(id=True).decode("UTF-8")[:-1]
package_list = [(package_name, version)]
if package_list:
......@@ -395,5 +415,5 @@
if package_list:
_logger.debug(f"Will try to download {package_list}")
_logger.debug("Matches for %s: %s", package, package_list)
return package_list
elif (
......@@ -405,14 +425,17 @@
if check_output(["git", "status"], cwd=package) == "":
_logger.debug("Repository is clean")
# 2. if clean, check tag
tag = check_output(
["git", "describe", "--exact-match", "HEAD"],
cwd=package,
)
if tag:
# Ignore local python version, this will be installed in an
# image.
# TODO changer pour prendre la version du python du super
# projet. Probablement pas renseigné où que ce soit.
return [(package_name, tag.decode("UTF-8"))]
if tags_only:
tag = check_output(
["git", "describe", "--exact-match", "HEAD"],
cwd=package,
)
if tag:
# Ignore local python version, this will be installed in an
# image.
# TODO changer pour prendre la version du python du super
# projet. Probablement pas renseigné où que ce soit.
return [(package_name, tag.decode("UTF-8"))]
else:
_logger.debug("No tag found")
else:
......@@ -418,5 +441,13 @@
else:
_logger.debug("No tag found")
return [
(
package_name,
check_output(
["git", "rev-parse", "HEAD"],
cwd=package,
).decode("UTF-8"),
)
]
else:
_logger.debug("Repository is dirty")
else:
......@@ -472,5 +503,10 @@
return os.path.join(base_cache_dir(), "packages", python_version, package, version)
def compiled_cache_dir(package: str, cvs_hash: str) -> str:
"""Return cache dir for package and version"""
return os.path.join(base_cache_dir(), "compiled-packages", package, cvs_hash)
if __name__ == "__main__":
sys.exit(main())
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment