Merge branch 'blender-v4.1-release'

build_files/utils/make_update.py

@@ -131,7 +131,7 @@ def ensure_git_lfs(args: argparse.Namespace) -> None:
     call((args.git_command, "lfs", "install", "--skip-repo"), exit_on_error=True)
 
 
-def update_precompiled_libraries(args: argparse.Namespace) -> None:
+def update_precompiled_libraries(args: argparse.Namespace) -> str:
     """
     Configure and update submodule for precompiled libraries
 
@@ -153,21 +153,24 @@ def update_precompiled_libraries(args: argparse.Namespace) -> None:
 
     if sys.platform == "linux" and not args.use_linux_libraries:
         print("Skipping Linux libraries configuration")
-        return
+        return ""
 
     submodule_dir = f"lib/{platform}_{arch}"
 
     submodule_directories = get_submodule_directories(args)
 
     if Path(submodule_dir) not in submodule_directories:
-        print("Skipping libraries update: no configured submodule")
-        return
+        return "Skipping libraries update: no configured submodule\n"
 
     make_utils.git_enable_submodule(args.git_command, submodule_dir)
-    make_utils.git_update_submodule(args.git_command, submodule_dir)
+
+    if not make_utils.git_update_submodule(args.git_command, submodule_dir):
+        return "Error updating precompiled libraries\n"
+
+    return ""
 
 
-def update_tests_data_files(args: argparse.Namespace) -> None:
+def update_tests_data_files(args: argparse.Namespace) -> str:
     """
     Configure and update submodule with files used by regression tests
     """
@@ -177,7 +180,11 @@ def update_tests_data_files(args: argparse.Namespace) -> None:
     submodule_dir = "tests/data"
 
     make_utils.git_enable_submodule(args.git_command, submodule_dir)
-    make_utils.git_update_submodule(args.git_command, submodule_dir)
+
+    if not make_utils.git_update_submodule(args.git_command, submodule_dir):
+        return "Error updating test data\n"
+
+    return ""
 
 
 def git_update_skip(args: argparse.Namespace, check_remote_exists: bool = True) -> str:
@@ -566,9 +573,7 @@ def submodules_update(args: argparse.Namespace, branch: Optional[str]) -> str:
             print(f"Skipping tests submodule {submodule_path}")
             continue
 
-        exitcode = call((args.git_command, "submodule", "update", "--init", submodule_path),
-                        exit_on_error=False)
-        if exitcode != 0:
+        if not make_utils.git_update_submodule(args.git_command, submodule_path):
             msg += f"Error updating Git submodule {submodule_path}\n"
 
     add_submodule_push_url(args)
@@ -579,6 +584,7 @@ def submodules_update(args: argparse.Namespace, branch: Optional[str]) -> str:
 if __name__ == "__main__":
     args = parse_arguments()
     blender_skip_msg = ""
+    libraries_skip_msg = ""
     submodules_skip_msg = ""
 
     blender_version = make_utils. parse_blender_version()
@@ -600,19 +606,18 @@ if __name__ == "__main__":
             blender_skip_msg = "Blender repository skipped: " + blender_skip_msg + "\n"
 
     if not args.no_libraries:
-        update_precompiled_libraries(args)
+        libraries_skip_msg += update_precompiled_libraries(args)
         if args.use_tests:
-            update_tests_data_files(args)
+            libraries_skip_msg += update_tests_data_files(args)
 
     if not args.no_submodules:
         submodules_skip_msg = submodules_update(args, branch)
 
     # Report any skipped repositories at the end, so it's not as easy to miss.
-    skip_msg = blender_skip_msg + submodules_skip_msg
+    skip_msg = blender_skip_msg + libraries_skip_msg + submodules_skip_msg
     if skip_msg:
-        print()
+        print_stage("Update finished with the following messages")
         print(skip_msg.strip())
-        print()
 
     # For failed submodule update we throw an error, since not having correct
     # submodules can make Blender throw errors.

build_files/utils/make_utils.py

@@ -8,6 +8,7 @@ Utility functions for make update and make tests.
 """
 
 import re
+import os
 import shutil
 import subprocess
 import sys
@@ -19,18 +20,30 @@ from typing import (
 )
 
 
-def call(cmd: Sequence[str], exit_on_error: bool = True, silent: bool = False) -> int:
+def call(cmd: Sequence[str], exit_on_error: bool = True, silent: bool = False, env=None) -> int:
     if not silent:
-        print(" ".join([str(x) for x in cmd]))
+        cmd_str = ""
+        if env:
+            cmd_str += " ".join([f"{item[0]}={item[1]}" for item in env.items()])
+            cmd_str += " "
+        cmd_str += " ".join([str(x) for x in cmd])
+        print(cmd_str)
+
+    env_full = None
+    if env:
+        env_full = os.environ.copy()
+        for key, value in env.items():
+            env_full[key] = value
 
     # Flush to ensure correct order output on Windows.
     sys.stdout.flush()
     sys.stderr.flush()
 
     if silent:
-        retcode = subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        retcode = subprocess.call(
+            cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env_full)
     else:
-        retcode = subprocess.call(cmd)
+        retcode = subprocess.call(cmd, env=env_full)
 
     if exit_on_error and retcode != 0:
         sys.exit(retcode)
@@ -127,15 +140,43 @@ def git_enable_submodule(git_command: str, submodule_dir: str):
     call(command, exit_on_error=True, silent=False)
 
 
-def git_update_submodule(git_command: str, submodule_dir: str):
+def git_update_submodule(git_command: str, submodule_dir: str) -> bool:
     """
     Update the given submodule.
 
     The submodule is denoted by its path within the repository.
     This function will initialize the submodule if it has not been initialized.
+
+    Returns true if the update succeeded
     """
 
-    call((git_command, "submodule", "update", "--init", submodule_dir))
+    # Use the two stage update process:
+    # - Step 1: checkout the submodule to the desired (by the parent repository) hash, but
+    #           skip the LFS smudging.
+    # - Step 2: Fetch LFS files, if needed.
+    #
+    # This allows to show download progress, potentially allowing resuming the download
+    # progress, and even recovering from partial/corrupted checkout of submodules.
+    #
+    # This bypasses the limitation of submodules which are configured as "update=checkout"
+    # with regular `git submodule update` which, depending on the Git version will not report
+    # any progress. This is because submodule--helper.c configures Git checkout process with
+    # the "quiet" flag, so that there is no detached head information printed after submodule
+    # update, and since Git 2.33 the LFS messages "Filtering contents..." is suppressed by
+    #
+    #   https://github.com/git/git/commit/7a132c628e57b9bceeb88832ea051395c0637b16
+    #
+    # Doing "git lfs pull" after checkout with GIT_LFS_SKIP_SMUDGE=true seems to be the
+    # valid process. For example, https://www.mankier.com/7/git-lfs-faq
+
+    env = {"GIT_LFS_SKIP_SMUDGE": "1"}
+
+    if call((git_command, "submodule", "update", "--init", "--progress", submodule_dir),
+            exit_on_error=False, env=env) != 0:
+        return False
+
+    return call((git_command, "-C", submodule_dir, "lfs", "pull"),
+                exit_on_error=False) == 0
 
 
 def command_missing(command: str) -> bool: