Python version of `make_source_archive.sh`
This is a Python version of the existing `make_source_archive.sh` script. IMO it's easier to read, and it'll also be easier to extend with the necessary functionality for D10598. The number of lines of code is larger than `make_source_archive.sh`, but it has considerably less invocations of `awk` ;-) And also the filtering is integrated, instead of forking out to Python to prevent certain files to be included in the tarball. Reviewed By: dfelinto, campbellbarton Differential Revision: https://developer.blender.org/D10629
This commit is contained in:
Sybren A. Stüvel
parent
9ce950daab
commit
8771f015f5
|
|
@ -506,7 +506,7 @@ check_descriptions: .FORCE
|
|||
#
|
||||
|
||||
source_archive: .FORCE
|
||||
./build_files/utils/make_source_archive.sh
|
||||
python3 ./build_files/utils/make_source_archive.py
|
||||
|
||||
INKSCAPE_BIN?="inkscape"
|
||||
icons: .FORCE
|
||||
|
|
|
|||
|
|
@ -0,0 +1,191 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import dataclasses
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Iterable, TextIO
|
||||
|
||||
# This script can run from any location,
|
||||
# output is created in the $CWD
|
||||
|
||||
SKIP_NAMES = {
|
||||
".gitignore",
|
||||
".gitmodules",
|
||||
".arcconfig",
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
output_dir = Path(".").absolute()
|
||||
blender_srcdir = Path(__file__).absolute().parent.parent.parent
|
||||
print(f"Source dir: {blender_srcdir}")
|
||||
|
||||
version = parse_blender_version(blender_srcdir)
|
||||
manifest = output_dir / f"blender-{version}-manifest.txt"
|
||||
tarball = output_dir / f"blender-{version}.tar.xz"
|
||||
|
||||
os.chdir(blender_srcdir)
|
||||
create_manifest(version, manifest)
|
||||
create_tarball(version, tarball, manifest)
|
||||
create_checksum_file(tarball)
|
||||
cleanup(manifest)
|
||||
print("Done!")
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class BlenderVersion:
|
||||
version: int # 293 for 2.93.1
|
||||
patch: int # 1 for 2.93.1
|
||||
cycle: str # 'alpha', 'beta', 'release', maybe others.
|
||||
|
||||
@property
|
||||
def is_release(self) -> bool:
|
||||
return self.cycle == "release"
|
||||
|
||||
def __str__(self) -> str:
|
||||
"""Convert to version string.
|
||||
|
||||
>>> str(BlenderVersion(293, 1, "alpha"))
|
||||
'2.93.1-alpha'
|
||||
>>> str(BlenderVersion(327, 0, "release"))
|
||||
'3.27.0'
|
||||
"""
|
||||
|
||||
as_string = f"{self.version/100:.2f}.{self.patch}"
|
||||
if self.is_release:
|
||||
return as_string
|
||||
return f"{as_string}-{self.cycle}"
|
||||
|
||||
|
||||
def parse_blender_version(blender_srcdir: Path) -> BlenderVersion:
|
||||
version_path = blender_srcdir / "source/blender/blenkernel/BKE_blender_version.h"
|
||||
|
||||
version_info = {}
|
||||
line_re = re.compile(r"^#define (BLENDER_VERSION[A-Z_]*)\s+([0-9a-z]+)$")
|
||||
|
||||
with version_path.open(encoding="utf-8") as version_file:
|
||||
for line in version_file:
|
||||
match = line_re.match(line.strip())
|
||||
if not match:
|
||||
continue
|
||||
version_info[match.group(1)] = match.group(2)
|
||||
|
||||
return BlenderVersion(
|
||||
int(version_info["BLENDER_VERSION"]),
|
||||
int(version_info["BLENDER_VERSION_PATCH"]),
|
||||
version_info["BLENDER_VERSION_CYCLE"],
|
||||
)
|
||||
|
||||
|
||||
### Manifest creation
|
||||
|
||||
|
||||
def create_manifest(version: BlenderVersion, outpath: Path) -> None:
|
||||
print(f'Building manifest of files: "{outpath}"...', end="", flush=True)
|
||||
with outpath.open("w", encoding="utf-8") as outfile:
|
||||
main_files_to_manifest(outfile)
|
||||
submodules_to_manifest(version, outfile)
|
||||
print("OK")
|
||||
|
||||
|
||||
def main_files_to_manifest(outfile: TextIO) -> None:
|
||||
for path in git_ls_files():
|
||||
print(path, file=outfile)
|
||||
|
||||
|
||||
def submodules_to_manifest(version: BlenderVersion, outfile: TextIO) -> None:
|
||||
skip_addon_contrib = version.is_release
|
||||
|
||||
for line in git_command("submodule"):
|
||||
submodule = line.split()[1]
|
||||
|
||||
if skip_addon_contrib and submodule == "release/scripts/addons_contrib":
|
||||
continue
|
||||
|
||||
for path in git_ls_files(Path(submodule)):
|
||||
print(path, file=outfile)
|
||||
|
||||
|
||||
def create_tarball(version: BlenderVersion, tarball: Path, manifest: Path) -> None:
|
||||
print(f'Creating archive: "{tarball}" ...', end="", flush=True)
|
||||
command = [
|
||||
"tar",
|
||||
"--transform",
|
||||
f"s,^,blender-{version}/,g",
|
||||
"--use-compress-program=xz -9",
|
||||
"--create",
|
||||
f"--file={tarball}",
|
||||
f"--files-from={manifest}",
|
||||
# Without owner/group args, extracting the files as root will
|
||||
# use ownership from the tar archive:
|
||||
"--owner=0",
|
||||
"--group=0",
|
||||
]
|
||||
subprocess.run(command, check=True, timeout=300)
|
||||
print("OK")
|
||||
|
||||
|
||||
def create_checksum_file(tarball: Path) -> None:
|
||||
md5_path = tarball.with_name(tarball.name + ".md5sum")
|
||||
print(f'Creating checksum: "{md5_path}" ...', end="", flush=True)
|
||||
command = [
|
||||
"md5sum",
|
||||
# The name is enough, as the tarball resides in the same dir as the MD5
|
||||
# file, and that's the current working directory.
|
||||
tarball.name,
|
||||
]
|
||||
md5_cmd = subprocess.run(
|
||||
command, stdout=subprocess.PIPE, check=True, text=True, timeout=300
|
||||
)
|
||||
with md5_path.open("w") as outfile:
|
||||
outfile.write(md5_cmd.stdout)
|
||||
print("OK")
|
||||
|
||||
|
||||
def cleanup(manifest: Path) -> None:
|
||||
print("Cleaning up ...", end="", flush=True)
|
||||
if manifest.exists():
|
||||
manifest.unlink()
|
||||
print("OK")
|
||||
|
||||
|
||||
## Low-level commands
|
||||
|
||||
|
||||
def git_ls_files(directory: Path = Path(".")) -> Iterable[Path]:
|
||||
"""Generator, yields lines of output from 'git ls-files'.
|
||||
|
||||
Only lines that are actually files (so no directories, sockets, etc.) are
|
||||
returned, and never one from SKIP_NAMES.
|
||||
"""
|
||||
for line in git_command("-C", str(directory), "ls-files"):
|
||||
path = directory / line
|
||||
if not path.is_file() or path.name in SKIP_NAMES:
|
||||
continue
|
||||
yield path
|
||||
|
||||
|
||||
def git_command(*cli_args) -> Iterable[str]:
|
||||
"""Generator, yields lines of output from a Git command."""
|
||||
command = ("git", *cli_args)
|
||||
|
||||
# import shlex
|
||||
# print(">", " ".join(shlex.quote(arg) for arg in command))
|
||||
|
||||
git = subprocess.run(
|
||||
command, stdout=subprocess.PIPE, check=True, text=True, timeout=30
|
||||
)
|
||||
for line in git.stdout.split("\n"):
|
||||
if line:
|
||||
yield line
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import doctest
|
||||
|
||||
if doctest.testmod().failed:
|
||||
raise SystemExit("ERROR: Self-test failed, refusing to run")
|
||||
|
||||
main()
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
#!/bin/sh
|
||||
|
||||
# This script can run from any location,
|
||||
# output is created in the $CWD
|
||||
|
||||
BASE_DIR="$PWD"
|
||||
|
||||
blender_srcdir=$(dirname -- $0)/../..
|
||||
blender_version=$(grep "BLENDER_VERSION\s" "$blender_srcdir/source/blender/blenkernel/BKE_blender_version.h" | awk '{print $3}')
|
||||
blender_version_patch=$(grep "BLENDER_VERSION_PATCH\s" "$blender_srcdir/source/blender/blenkernel/BKE_blender_version.h" | awk '{print $3}')
|
||||
blender_version_cycle=$(grep "BLENDER_VERSION_CYCLE\s" "$blender_srcdir/source/blender/blenkernel/BKE_blender_version.h" | awk '{print $3}')
|
||||
|
||||
VERSION=$(expr $blender_version / 100).$(expr $blender_version % 100).$blender_version_patch
|
||||
if [ "$blender_version_cycle" = "release" ] ; then
|
||||
SUBMODULE_EXCLUDE="^\(release/scripts/addons_contrib\)$"
|
||||
else
|
||||
VERSION=$VERSION-$blender_version_cycle
|
||||
SUBMODULE_EXCLUDE="^$" # dummy regex
|
||||
fi
|
||||
|
||||
MANIFEST="blender-$VERSION-manifest.txt"
|
||||
TARBALL="blender-$VERSION.tar.xz"
|
||||
|
||||
cd "$blender_srcdir"
|
||||
|
||||
# not so nice, but works
|
||||
FILTER_FILES_PY=\
|
||||
"import os, sys; "\
|
||||
"[print(l[:-1]) for l in sys.stdin.readlines() "\
|
||||
"if os.path.isfile(l[:-1]) "\
|
||||
"if os.path.basename(l[:-1]) not in {"\
|
||||
"'.gitignore', "\
|
||||
"'.gitmodules', "\
|
||||
"'.arcconfig', "\
|
||||
"}"\
|
||||
"]"
|
||||
|
||||
# Build master list
|
||||
echo -n "Building manifest of files: \"$BASE_DIR/$MANIFEST\" ..."
|
||||
git ls-files | python3 -c "$FILTER_FILES_PY" > $BASE_DIR/$MANIFEST
|
||||
|
||||
# Enumerate submodules
|
||||
for lcv in $(git submodule | awk '{print $2}' | grep -v "$SUBMODULE_EXCLUDE"); do
|
||||
cd "$BASE_DIR"
|
||||
cd "$blender_srcdir/$lcv"
|
||||
git ls-files | python3 -c "$FILTER_FILES_PY" | awk '$0="'"$lcv"/'"$0' >> $BASE_DIR/$MANIFEST
|
||||
cd "$BASE_DIR"
|
||||
done
|
||||
echo "OK"
|
||||
|
||||
|
||||
# Create the tarball
|
||||
#
|
||||
# Without owner/group args, extracting the files as root will
|
||||
# use ownership from the tar archive.
|
||||
cd "$blender_srcdir"
|
||||
echo -n "Creating archive: \"$BASE_DIR/$TARBALL\" ..."
|
||||
tar \
|
||||
--transform "s,^,blender-$VERSION/,g" \
|
||||
--use-compress-program="xz -9" \
|
||||
--create \
|
||||
--file="$BASE_DIR/$TARBALL" \
|
||||
--files-from="$BASE_DIR/$MANIFEST" \
|
||||
--owner=0 \
|
||||
--group=0
|
||||
|
||||
echo "OK"
|
||||
|
||||
|
||||
# Create checksum file
|
||||
cd "$BASE_DIR"
|
||||
echo -n "Creating checksum: \"$BASE_DIR/$TARBALL.md5sum\" ..."
|
||||
md5sum "$TARBALL" > "$TARBALL.md5sum"
|
||||
echo "OK"
|
||||
|
||||
|
||||
# Cleanup
|
||||
echo -n "Cleaning up ..."
|
||||
rm "$BASE_DIR/$MANIFEST"
|
||||
echo "OK"
|
||||
|
||||
echo "Done!"
|
||||
Loading…
Reference in New Issue