Add `make source_archive_complete` target
Add a `source_archive_complete` target for `make` that creates a source
archive including the source packages of Blender's dependencies.
This expands `make_source_archive.py` to include files from
`${BUILD_DIR}/source_archive/packages/` as well.
Reviewed By: dfelinto
Maniphest Tasks: T86124
Differential Revision: https://developer.blender.org/D10727
This commit is contained in:
Sybren A. Stüvel
parent
1f2a801044
commit
b8479a70c9
10
GNUmakefile
10
GNUmakefile
|
|
@ -128,6 +128,9 @@ Utilities
|
|||
* source_archive:
|
||||
Create a compressed archive of the source code.
|
||||
|
||||
* source_archive_complete:
|
||||
Create a compressed archive of the source code and all the libraries of dependencies.
|
||||
|
||||
* update:
|
||||
Updates git and all submodules and svn.
|
||||
|
||||
|
|
@ -511,6 +514,13 @@ check_descriptions: .FORCE
|
|||
source_archive: .FORCE
|
||||
python3 ./build_files/utils/make_source_archive.py
|
||||
|
||||
source_archive_complete: .FORCE
|
||||
cmake -S "$(BLENDER_DIR)/build_files/build_environment" -B"$(BUILD_DIR)/source_archive" \
|
||||
-DCMAKE_BUILD_TYPE_INIT:STRING=$(BUILD_TYPE) -DPACKAGE_USE_UPSTREAM_SOURCES=OFF
|
||||
# This assumes CMake is still using a default `PACKAGE_DIR` variable:
|
||||
python3 ./build_files/utils/make_source_archive.py --include-packages "$(BUILD_DIR)/source_archive/packages"
|
||||
|
||||
|
||||
INKSCAPE_BIN?="inkscape"
|
||||
icons: .FORCE
|
||||
BLENDER_BIN=$(BLENDER_BIN) INKSCAPE_BIN=$(INKSCAPE_BIN) \
|
||||
|
|
|
|||
|
|
@ -37,14 +37,8 @@ else(BUILD_MODE STREQUAL "Debug")
|
|||
endif()
|
||||
|
||||
set(DOWNLOAD_DIR "${CMAKE_CURRENT_BINARY_DIR}/downloads" CACHE STRING "Path for downloaded files")
|
||||
# look in blenders source folder for packages directory, if that exists
|
||||
# it will our package folder, otherwise it will be in the build folder
|
||||
if(EXISTS "${CMAKE_SOURCE_DIR}/../../packages")
|
||||
set(PACKAGE_DIR_DEFAULT "${CMAKE_SOURCE_DIR}/../../packages")
|
||||
else()
|
||||
set(PACKAGE_DIR_DEFAULT "${CMAKE_CURRENT_BINARY_DIR}/packages")
|
||||
endif()
|
||||
set(PACKAGE_DIR ${PACKAGE_DIR_DEFAULT} CACHE STRING "Path for downloaded source files")
|
||||
# This path must be hard-coded like this, so that the GNUmakefile knows where it is and can pass it to make_source_archive.py:
|
||||
set(PACKAGE_DIR "${CMAKE_CURRENT_BINARY_DIR}/packages")
|
||||
option(PACKAGE_USE_UPSTREAM_SOURCES "Use soures upstream to download the package sources, when OFF the blender mirror will be used" ON)
|
||||
|
||||
file(TO_CMAKE_PATH ${DOWNLOAD_DIR} DOWNLOAD_DIR)
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import dataclasses
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Iterable, TextIO
|
||||
from typing import Iterable, TextIO, Optional, Any
|
||||
|
||||
# This script can run from any location,
|
||||
# output is created in the $CWD
|
||||
|
|
@ -18,21 +19,43 @@ SKIP_NAMES = {
|
|||
".gitignore",
|
||||
".gitmodules",
|
||||
".arcconfig",
|
||||
".svn",
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
output_dir = Path(".").absolute()
|
||||
blender_srcdir = Path(__file__).absolute().parent.parent.parent
|
||||
|
||||
cli_parser = argparse.ArgumentParser(
|
||||
description=f"Create a tarball of the Blender sources, optionally including sources of dependencies.",
|
||||
epilog="This script is intended to be run by `make source_archive_complete`.",
|
||||
)
|
||||
cli_parser.add_argument(
|
||||
"-p",
|
||||
"--include-packages",
|
||||
type=Path,
|
||||
default=None,
|
||||
metavar="PACKAGE_PATH",
|
||||
help="Include all source files from the given package directory as well.",
|
||||
)
|
||||
|
||||
cli_args = cli_parser.parse_args()
|
||||
|
||||
print(f"Source dir: {blender_srcdir}")
|
||||
|
||||
version = parse_blender_version(blender_srcdir)
|
||||
manifest = output_dir / f"blender-{version}-manifest.txt"
|
||||
tarball = output_dir / f"blender-{version}.tar.xz"
|
||||
curdir = blender_srcdir.parent
|
||||
os.chdir(curdir)
|
||||
blender_srcdir = blender_srcdir.relative_to(curdir)
|
||||
|
||||
os.chdir(blender_srcdir)
|
||||
create_manifest(version, manifest)
|
||||
create_tarball(version, tarball, manifest)
|
||||
print(f"Output dir: {curdir}")
|
||||
|
||||
version = parse_blender_version(blender_srcdir)
|
||||
tarball = tarball_path(curdir, version, cli_args)
|
||||
manifest = manifest_path(tarball)
|
||||
packages_dir = packages_path(curdir, cli_args)
|
||||
|
||||
create_manifest(version, manifest, blender_srcdir, packages_dir)
|
||||
create_tarball(version, tarball, manifest, blender_srcdir, packages_dir)
|
||||
create_checksum_file(tarball)
|
||||
cleanup(manifest)
|
||||
print("Done!")
|
||||
|
|
@ -84,43 +107,109 @@ def parse_blender_version(blender_srcdir: Path) -> BlenderVersion:
|
|||
)
|
||||
|
||||
|
||||
def tarball_path(output_dir: Path, version: BlenderVersion, cli_args: Any) -> Path:
|
||||
extra = ""
|
||||
if cli_args.include_packages:
|
||||
extra = "-with-libraries"
|
||||
|
||||
return output_dir / f"blender{extra}-{version}.tar.xz"
|
||||
|
||||
|
||||
def manifest_path(tarball: Path) -> Path:
|
||||
"""Return the manifest path for the given tarball path.
|
||||
|
||||
>>> from pathlib import Path
|
||||
>>> tarball = Path("/home/sybren/workspace/blender-git/blender-test.tar.gz")
|
||||
>>> manifest_path(tarball).as_posix()
|
||||
'/home/sybren/workspace/blender-git/blender-test-manifest.txt'
|
||||
"""
|
||||
# ".tar.gz" is seen as two suffixes.
|
||||
without_suffix = tarball.with_suffix("").with_suffix("")
|
||||
name = without_suffix.name
|
||||
return without_suffix.with_name(f"{name}-manifest.txt")
|
||||
|
||||
|
||||
def packages_path(current_directory: Path, cli_args: Any) -> Optional[Path]:
|
||||
if not cli_args.include_packages:
|
||||
return None
|
||||
|
||||
abspath = cli_args.include_packages.absolute()
|
||||
|
||||
# os.path.relpath() can return paths like "../../packages", where
|
||||
# Path.relative_to() will not go up directories (so its return value never
|
||||
# has "../" in there).
|
||||
relpath = os.path.relpath(abspath, current_directory)
|
||||
|
||||
return Path(relpath)
|
||||
|
||||
|
||||
### Manifest creation
|
||||
|
||||
|
||||
def create_manifest(version: BlenderVersion, outpath: Path) -> None:
|
||||
def create_manifest(
|
||||
version: BlenderVersion,
|
||||
outpath: Path,
|
||||
blender_srcdir: Path,
|
||||
packages_dir: Optional[Path],
|
||||
) -> None:
|
||||
print(f'Building manifest of files: "{outpath}"...', end="", flush=True)
|
||||
with outpath.open("w", encoding="utf-8") as outfile:
|
||||
main_files_to_manifest(outfile)
|
||||
submodules_to_manifest(version, outfile)
|
||||
main_files_to_manifest(blender_srcdir, outfile)
|
||||
submodules_to_manifest(blender_srcdir, version, outfile)
|
||||
|
||||
if packages_dir:
|
||||
packages_to_manifest(outfile, packages_dir)
|
||||
print("OK")
|
||||
|
||||
|
||||
def main_files_to_manifest(outfile: TextIO) -> None:
|
||||
for path in git_ls_files():
|
||||
def main_files_to_manifest(blender_srcdir: Path, outfile: TextIO) -> None:
|
||||
assert not blender_srcdir.is_absolute()
|
||||
for path in git_ls_files(blender_srcdir):
|
||||
print(path, file=outfile)
|
||||
|
||||
|
||||
def submodules_to_manifest(version: BlenderVersion, outfile: TextIO) -> None:
|
||||
def submodules_to_manifest(
|
||||
blender_srcdir: Path, version: BlenderVersion, outfile: TextIO
|
||||
) -> None:
|
||||
skip_addon_contrib = version.is_release
|
||||
assert not blender_srcdir.is_absolute()
|
||||
|
||||
for line in git_command("submodule"):
|
||||
for line in git_command("-C", blender_srcdir, "submodule"):
|
||||
submodule = line.split()[1]
|
||||
|
||||
# Don't use native slashes as GIT for MS-Windows outputs forward slashes.
|
||||
if skip_addon_contrib and submodule == "release/scripts/addons_contrib":
|
||||
continue
|
||||
|
||||
for path in git_ls_files(Path(submodule)):
|
||||
for path in git_ls_files(blender_srcdir / submodule):
|
||||
print(path, file=outfile)
|
||||
|
||||
|
||||
def create_tarball(version: BlenderVersion, tarball: Path, manifest: Path) -> None:
|
||||
def packages_to_manifest(outfile: TextIO, packages_dir: Path) -> None:
|
||||
for path in packages_dir.glob("*"):
|
||||
if not path.is_file():
|
||||
continue
|
||||
if path.name in SKIP_NAMES:
|
||||
continue
|
||||
print(path, file=outfile)
|
||||
|
||||
|
||||
### Higher-level functions
|
||||
|
||||
|
||||
def create_tarball(
|
||||
version: BlenderVersion, tarball: Path, manifest: Path, blender_srcdir: Path, packages_dir: Optional[Path]
|
||||
) -> None:
|
||||
print(f'Creating archive: "{tarball}" ...', end="", flush=True)
|
||||
command = ["tar"]
|
||||
|
||||
# Requires GNU `tar`, since `--transform` is used.
|
||||
command = [
|
||||
"tar",
|
||||
if packages_dir:
|
||||
command += ["--transform", f"s,{packages_dir}/,packages/,g"]
|
||||
|
||||
command += [
|
||||
"--transform",
|
||||
f"s,^,blender-{version}/,g",
|
||||
f"s,^{blender_srcdir.name}/,blender-{version}/,g",
|
||||
"--use-compress-program=xz -9",
|
||||
"--create",
|
||||
f"--file={tarball}",
|
||||
|
|
@ -130,7 +219,8 @@ def create_tarball(version: BlenderVersion, tarball: Path, manifest: Path) -> No
|
|||
"--owner=0",
|
||||
"--group=0",
|
||||
]
|
||||
subprocess.run(command, check=True, timeout=300)
|
||||
|
||||
subprocess.run(command, check=True, timeout=3600)
|
||||
print("OK")
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue