Make update: Bring behavior closer to what it used to be

The `make update` will now always update all initialized submodules,
including the library ones.

The `make_update.py --no-libraries` will neither initialize nor update
pre-compiled libraries or tests.

Pull Request: https://projects.blender.org/blender/blender/pulls/118812

build_files/utils/make_update.py

@@ -122,7 +122,7 @@ def get_submodule_directories(args: argparse.Namespace):
 
     submodule_directories_output = check_output(
         [args.git_command, "config", "--file", dot_modules, "--get-regexp", "path"])
-    return (Path(line.split(' ', 1)[1]) for line in submodule_directories_output.strip().splitlines())
+    return [Path(line.split(' ', 1)[1]) for line in submodule_directories_output.strip().splitlines()]
 
 
 def ensure_git_lfs(args: argparse.Namespace) -> None:
@@ -131,9 +131,9 @@ def ensure_git_lfs(args: argparse.Namespace) -> None:
     call((args.git_command, "lfs", "install", "--skip-repo"), exit_on_error=True)
 
 
-def update_precompiled_libraries(args: argparse.Namespace) -> str:
+def initialize_precompiled_libraries(args: argparse.Namespace) -> str:
     """
-    Configure and update submodule for precompiled libraries
+    Configure submodule for precompiled libraries
 
     This function detects the current host architecture and enables
     corresponding submodule, and updates the submodule.
@@ -162,27 +162,23 @@ def update_precompiled_libraries(args: argparse.Namespace) -> str:
     if Path(submodule_dir) not in submodule_directories:
         return "Skipping libraries update: no configured submodule\n"
 
-    make_utils.git_enable_submodule(args.git_command, submodule_dir)
+    print(f"* Enabling precompiled libraries at {submodule_dir}")
-
+    make_utils.git_enable_submodule(args.git_command, Path(submodule_dir))
-    if not make_utils.git_update_submodule(args.git_command, submodule_dir):
-        return "Error updating precompiled libraries\n"
 
     return ""
 
 
-def update_tests_data_files(args: argparse.Namespace) -> str:
+def initialize_tests_data_files(args: argparse.Namespace) -> str:
     """
-    Configure and update submodule with files used by regression tests
+    Configure submodule with files used by regression tests
     """
 
     print_stage("Configuring Tests Data Files")
 
     submodule_dir = "tests/data"
 
-    make_utils.git_enable_submodule(args.git_command, submodule_dir)
+    print(f"* Enabling tests data at {submodule_dir}")
-
+    make_utils.git_enable_submodule(args.git_command, Path(submodule_dir))
-    if not make_utils.git_update_submodule(args.git_command, submodule_dir):
-        return "Error updating test data\n"
 
     return ""
 
@@ -490,16 +486,6 @@ def external_scripts_update(args: argparse.Namespace,
                                     old_submodules_dir=Path("release") / "scripts" / directory_name)
 
 
-def scripts_submodules_update(args: argparse.Namespace, branch: Optional[str]) -> str:
-    """Update working trees of addons and addons_contrib within the scripts/ directory"""
-    msg = ""
-
-    msg += external_scripts_update(args, "blender-addons", "addons", branch)
-    msg += external_scripts_update(args, "blender-addons-contrib", "addons_contrib", branch)
-
-    return msg
-
-
 def floating_libraries_update(args: argparse.Namespace, branch: Optional[str]) -> str:
     """Update libraries checkouts which are floating (not attached as Git submodules)"""
     msg = ""
@@ -551,7 +537,39 @@ def add_submodule_push_url(args: argparse.Namespace):
         make_utils.git_set_config(args.git_command, "remote.origin.pushURL", push_url, str(config))
 
 
-def submodules_update(args: argparse.Namespace, branch: Optional[str]) -> str:
+def submodules_lib_update(args: argparse.Namespace, branch: Optional[str]) -> str:
+    print_stage("Updating Libraries")
+
+    msg = ""
+    msg += floating_libraries_update(args, branch)
+
+    submodule_directories = get_submodule_directories(args)
+    for submodule_path in submodule_directories:
+        if not make_utils.is_git_submodule_enabled(args.git_command, submodule_path):
+            print(f"* Skipping {submodule_path}")
+            continue
+
+        print(f"* Updating {submodule_path} ...")
+
+        if not make_utils.git_update_submodule(args.git_command, submodule_path):
+            msg += f"Error updating Git submodule {submodule_path}\n"
+
+    add_submodule_push_url(args)
+
+    return msg
+
+
+def scripts_submodules_update(args: argparse.Namespace, branch: Optional[str]) -> str:
+    """Update working trees of addons and addons_contrib within the scripts/ directory"""
+    msg = ""
+
+    msg += external_scripts_update(args, "blender-addons", "addons", branch)
+    msg += external_scripts_update(args, "blender-addons-contrib", "addons_contrib", branch)
+
+    return msg
+
+
+def submodules_code_update(args: argparse.Namespace, branch: Optional[str]) -> str:
     """Update submodules or other externally tracked source trees"""
     print_stage("Updating Submodules")
 
@@ -559,25 +577,6 @@ def submodules_update(args: argparse.Namespace, branch: Optional[str]) -> str:
 
     msg += scripts_submodules_update(args, branch)
 
-    msg += floating_libraries_update(args, branch)
-
-    print("* Updating Git submodules")
-
-    submodule_directories = get_submodule_directories(args)
-    for submodule_path in submodule_directories:
-        if submodule_path.parts[0] == "lib" and args.no_libraries:
-            print(f"Skipping library submodule {submodule_path}")
-            continue
-
-        if submodule_path.parts[0] == "tests" and not args.use_tests:
-            print(f"Skipping tests submodule {submodule_path}")
-            continue
-
-        if not make_utils.git_update_submodule(args.git_command, submodule_path):
-            msg += f"Error updating Git submodule {submodule_path}\n"
-
-    add_submodule_push_url(args)
-
     return msg
 
 
@@ -606,12 +605,13 @@ if __name__ == "__main__":
             blender_skip_msg = "Blender repository skipped: " + blender_skip_msg + "\n"
 
     if not args.no_libraries:
-        libraries_skip_msg += update_precompiled_libraries(args)
+        libraries_skip_msg += initialize_precompiled_libraries(args)
         if args.use_tests:
-            libraries_skip_msg += update_tests_data_files(args)
+            libraries_skip_msg += initialize_tests_data_files(args)
+        libraries_skip_msg += submodules_lib_update(args, branch)
 
     if not args.no_submodules:
-        submodules_skip_msg = submodules_update(args, branch)
+        submodules_skip_msg += submodules_code_update(args, branch)
 
     # Report any skipped repositories at the end, so it's not as easy to miss.
     skip_msg = blender_skip_msg + libraries_skip_msg + submodules_skip_msg

build_files/utils/make_utils.py

@@ -130,17 +130,46 @@ def git_set_config(git_command: str, key: str, value: str, file: Optional[str] =
     return check_output([git_command, "config", key, value])
 
 
-def git_enable_submodule(git_command: str, submodule_dir: str):
+def _git_submodule_config_key(submodule_dir: Path, key: str) -> str:
+    submodule_dir_str = submodule_dir.as_posix()
+    return f"submodule.{submodule_dir_str}.{key}"
+
+
+def is_git_submodule_enabled(git_command: str, submodule_dir: Path):
+    """Check whether submodule denoted by its directory within the repository is enabled"""
+
+    git_root = Path(check_output([git_command, "rev-parse", "--show-toplevel"]))
+    gitmodules = git_root / ".gitmodules"
+
+    # Check whether the submodule actually exists.
+    # Request path of an unknown submodule will cause non-zero exit code.
+    path = git_get_config(
+        git_command, _git_submodule_config_key(submodule_dir, "path"), str(gitmodules))
+    if not path:
+        return False
+
+    # When the "update" strategy is not provided explicitly in the the local configuration
+    # `git config` returns a non-zero exit code. For those assume the default "checkout"
+    # strategy.
+    update = check_output(
+        (git_command, "config", "--local", _git_submodule_config_key(submodule_dir, "update")),
+        exit_on_error=False)
+
+    return update.lower() != "none"
+
+
+def git_enable_submodule(git_command: str, submodule_dir: Path):
     """Enable submodule denoted by its directory within the repository"""
 
     command = (git_command,
                "config",
                "--local",
-               f"submodule.{submodule_dir}.update", "checkout")
+               _git_submodule_config_key(submodule_dir, "update"),
-    call(command, exit_on_error=True, silent=False)
+               "checkout")
+    call(command, exit_on_error=True, silent=True)
 
 
-def git_update_submodule(git_command: str, submodule_dir: str) -> bool:
+def git_update_submodule(git_command: str, submodule_dir: Path) -> bool:
     """
     Update the given submodule.