From 468e4b0e3b04afae517e92cce4fa3bf91c1d432e Mon Sep 17 00:00:00 2001 From: Sergey Sharybin Date: Tue, 27 Feb 2024 20:02:15 +0100 Subject: [PATCH] Make update: Bring behavior closer to what it used to be The `make update` will now always update all initialized submodules, including the library ones. The `make_update.py --no-libraries` will neither initialize nor update pre-compiled libraries or tests. Pull Request: https://projects.blender.org/blender/blender/pulls/118812 --- build_files/utils/make_update.py | 92 ++++++++++++++++---------------- build_files/utils/make_utils.py | 37 +++++++++++-- 2 files changed, 79 insertions(+), 50 deletions(-) diff --git a/build_files/utils/make_update.py b/build_files/utils/make_update.py index 2eb2334d66a..152f30ef270 100755 --- a/build_files/utils/make_update.py +++ b/build_files/utils/make_update.py @@ -122,7 +122,7 @@ def get_submodule_directories(args: argparse.Namespace): submodule_directories_output = check_output( [args.git_command, "config", "--file", dot_modules, "--get-regexp", "path"]) - return (Path(line.split(' ', 1)[1]) for line in submodule_directories_output.strip().splitlines()) + return [Path(line.split(' ', 1)[1]) for line in submodule_directories_output.strip().splitlines()] def ensure_git_lfs(args: argparse.Namespace) -> None: @@ -131,9 +131,9 @@ def ensure_git_lfs(args: argparse.Namespace) -> None: call((args.git_command, "lfs", "install", "--skip-repo"), exit_on_error=True) -def update_precompiled_libraries(args: argparse.Namespace) -> str: +def initialize_precompiled_libraries(args: argparse.Namespace) -> str: """ - Configure and update submodule for precompiled libraries + Configure submodule for precompiled libraries This function detects the current host architecture and enables corresponding submodule, and updates the submodule. @@ -162,27 +162,23 @@ def update_precompiled_libraries(args: argparse.Namespace) -> str: if Path(submodule_dir) not in submodule_directories: return "Skipping libraries update: no configured submodule\n" - make_utils.git_enable_submodule(args.git_command, submodule_dir) - - if not make_utils.git_update_submodule(args.git_command, submodule_dir): - return "Error updating precompiled libraries\n" + print(f"* Enabling precompiled libraries at {submodule_dir}") + make_utils.git_enable_submodule(args.git_command, Path(submodule_dir)) return "" -def update_tests_data_files(args: argparse.Namespace) -> str: +def initialize_tests_data_files(args: argparse.Namespace) -> str: """ - Configure and update submodule with files used by regression tests + Configure submodule with files used by regression tests """ print_stage("Configuring Tests Data Files") submodule_dir = "tests/data" - make_utils.git_enable_submodule(args.git_command, submodule_dir) - - if not make_utils.git_update_submodule(args.git_command, submodule_dir): - return "Error updating test data\n" + print(f"* Enabling tests data at {submodule_dir}") + make_utils.git_enable_submodule(args.git_command, Path(submodule_dir)) return "" @@ -490,16 +486,6 @@ def external_scripts_update(args: argparse.Namespace, old_submodules_dir=Path("release") / "scripts" / directory_name) -def scripts_submodules_update(args: argparse.Namespace, branch: Optional[str]) -> str: - """Update working trees of addons and addons_contrib within the scripts/ directory""" - msg = "" - - msg += external_scripts_update(args, "blender-addons", "addons", branch) - msg += external_scripts_update(args, "blender-addons-contrib", "addons_contrib", branch) - - return msg - - def floating_libraries_update(args: argparse.Namespace, branch: Optional[str]) -> str: """Update libraries checkouts which are floating (not attached as Git submodules)""" msg = "" @@ -551,7 +537,39 @@ def add_submodule_push_url(args: argparse.Namespace): make_utils.git_set_config(args.git_command, "remote.origin.pushURL", push_url, str(config)) -def submodules_update(args: argparse.Namespace, branch: Optional[str]) -> str: +def submodules_lib_update(args: argparse.Namespace, branch: Optional[str]) -> str: + print_stage("Updating Libraries") + + msg = "" + msg += floating_libraries_update(args, branch) + + submodule_directories = get_submodule_directories(args) + for submodule_path in submodule_directories: + if not make_utils.is_git_submodule_enabled(args.git_command, submodule_path): + print(f"* Skipping {submodule_path}") + continue + + print(f"* Updating {submodule_path} ...") + + if not make_utils.git_update_submodule(args.git_command, submodule_path): + msg += f"Error updating Git submodule {submodule_path}\n" + + add_submodule_push_url(args) + + return msg + + +def scripts_submodules_update(args: argparse.Namespace, branch: Optional[str]) -> str: + """Update working trees of addons and addons_contrib within the scripts/ directory""" + msg = "" + + msg += external_scripts_update(args, "blender-addons", "addons", branch) + msg += external_scripts_update(args, "blender-addons-contrib", "addons_contrib", branch) + + return msg + + +def submodules_code_update(args: argparse.Namespace, branch: Optional[str]) -> str: """Update submodules or other externally tracked source trees""" print_stage("Updating Submodules") @@ -559,25 +577,6 @@ def submodules_update(args: argparse.Namespace, branch: Optional[str]) -> str: msg += scripts_submodules_update(args, branch) - msg += floating_libraries_update(args, branch) - - print("* Updating Git submodules") - - submodule_directories = get_submodule_directories(args) - for submodule_path in submodule_directories: - if submodule_path.parts[0] == "lib" and args.no_libraries: - print(f"Skipping library submodule {submodule_path}") - continue - - if submodule_path.parts[0] == "tests" and not args.use_tests: - print(f"Skipping tests submodule {submodule_path}") - continue - - if not make_utils.git_update_submodule(args.git_command, submodule_path): - msg += f"Error updating Git submodule {submodule_path}\n" - - add_submodule_push_url(args) - return msg @@ -606,12 +605,13 @@ if __name__ == "__main__": blender_skip_msg = "Blender repository skipped: " + blender_skip_msg + "\n" if not args.no_libraries: - libraries_skip_msg += update_precompiled_libraries(args) + libraries_skip_msg += initialize_precompiled_libraries(args) if args.use_tests: - libraries_skip_msg += update_tests_data_files(args) + libraries_skip_msg += initialize_tests_data_files(args) + libraries_skip_msg += submodules_lib_update(args, branch) if not args.no_submodules: - submodules_skip_msg = submodules_update(args, branch) + submodules_skip_msg += submodules_code_update(args, branch) # Report any skipped repositories at the end, so it's not as easy to miss. skip_msg = blender_skip_msg + libraries_skip_msg + submodules_skip_msg diff --git a/build_files/utils/make_utils.py b/build_files/utils/make_utils.py index 9fd8bad4a41..7b3b17619fc 100755 --- a/build_files/utils/make_utils.py +++ b/build_files/utils/make_utils.py @@ -130,17 +130,46 @@ def git_set_config(git_command: str, key: str, value: str, file: Optional[str] = return check_output([git_command, "config", key, value]) -def git_enable_submodule(git_command: str, submodule_dir: str): +def _git_submodule_config_key(submodule_dir: Path, key: str) -> str: + submodule_dir_str = submodule_dir.as_posix() + return f"submodule.{submodule_dir_str}.{key}" + + +def is_git_submodule_enabled(git_command: str, submodule_dir: Path): + """Check whether submodule denoted by its directory within the repository is enabled""" + + git_root = Path(check_output([git_command, "rev-parse", "--show-toplevel"])) + gitmodules = git_root / ".gitmodules" + + # Check whether the submodule actually exists. + # Request path of an unknown submodule will cause non-zero exit code. + path = git_get_config( + git_command, _git_submodule_config_key(submodule_dir, "path"), str(gitmodules)) + if not path: + return False + + # When the "update" strategy is not provided explicitly in the the local configuration + # `git config` returns a non-zero exit code. For those assume the default "checkout" + # strategy. + update = check_output( + (git_command, "config", "--local", _git_submodule_config_key(submodule_dir, "update")), + exit_on_error=False) + + return update.lower() != "none" + + +def git_enable_submodule(git_command: str, submodule_dir: Path): """Enable submodule denoted by its directory within the repository""" command = (git_command, "config", "--local", - f"submodule.{submodule_dir}.update", "checkout") - call(command, exit_on_error=True, silent=False) + _git_submodule_config_key(submodule_dir, "update"), + "checkout") + call(command, exit_on_error=True, silent=True) -def git_update_submodule(git_command: str, submodule_dir: str) -> bool: +def git_update_submodule(git_command: str, submodule_dir: Path) -> bool: """ Update the given submodule.