Tests: speed up render tests by running multiple in the same process
Blender startup time and shader compilation is a big factor when running hundreds of tests, so now all renders in the same ctest run in the same process. This was previously reverted due to skipping other tests when one test crashed. Now if a test crashes, Blender is re-run with the remaining tests so we get results from them still.
This commit is contained in:
Brecht Van Lommel
parent
219a10e46a
commit
6f516fcc63
|
|
@ -9,14 +9,13 @@ import subprocess
|
|||
import sys
|
||||
|
||||
|
||||
def render_file(filepath, output_filepath):
|
||||
def get_arguments(filepath, output_filepath):
|
||||
dirname = os.path.dirname(filepath)
|
||||
basedir = os.path.dirname(dirname)
|
||||
subject = os.path.basename(dirname)
|
||||
|
||||
frame_filepath = output_filepath + '0001.png'
|
||||
|
||||
common_args = [
|
||||
args = [
|
||||
"--background",
|
||||
"-noaudio",
|
||||
"--factory-startup",
|
||||
"--enable-autoexec",
|
||||
|
|
@ -29,57 +28,17 @@ def render_file(filepath, output_filepath):
|
|||
# custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True"]
|
||||
# custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.device = 'GPU'"]
|
||||
custom_args = os.getenv('CYCLESTEST_ARGS')
|
||||
custom_args = shlex.split(custom_args) if custom_args else []
|
||||
common_args += custom_args
|
||||
if custom_args:
|
||||
args.extend(shlex.split(custom_args))
|
||||
|
||||
if subject == 'opengl':
|
||||
command = [BLENDER, "--window-geometry", "0", "0", "1", "1"]
|
||||
command += common_args
|
||||
command += ['--python', os.path.join(basedir, "util", "render_opengl.py")]
|
||||
elif subject == 'bake':
|
||||
command = [BLENDER, "--background"]
|
||||
command += common_args
|
||||
command += ['--python', os.path.join(basedir, "util", "render_bake.py")]
|
||||
if subject == 'bake':
|
||||
args.extend(['--python', os.path.join(basedir, "util", "render_bake.py")])
|
||||
elif subject == 'denoise_animation':
|
||||
command = [BLENDER, "--background"]
|
||||
command += common_args
|
||||
command += ['--python', os.path.join(basedir, "util", "render_denoise.py")]
|
||||
args.extend(['--python', os.path.join(basedir, "util", "render_denoise.py")])
|
||||
else:
|
||||
command = [BLENDER, "--background"]
|
||||
command += common_args
|
||||
command += ["-f", "1"]
|
||||
|
||||
try:
|
||||
# Success
|
||||
output = subprocess.check_output(command)
|
||||
if os.path.exists(frame_filepath):
|
||||
shutil.copy(frame_filepath, output_filepath)
|
||||
os.remove(frame_filepath)
|
||||
if VERBOSE:
|
||||
print(" ".join(command))
|
||||
print(output.decode("utf-8"))
|
||||
return None
|
||||
except subprocess.CalledProcessError as e:
|
||||
# Error
|
||||
if os.path.exists(frame_filepath):
|
||||
os.remove(frame_filepath)
|
||||
if VERBOSE:
|
||||
print(" ".join(command))
|
||||
print(e.output.decode("utf-8"))
|
||||
if b"Error: engine not found" in e.output:
|
||||
return "NO_ENGINE"
|
||||
elif b"blender probably wont start" in e.output:
|
||||
return "NO_START"
|
||||
return "CRASH"
|
||||
except BaseException as e:
|
||||
# Crash
|
||||
if os.path.exists(frame_filepath):
|
||||
os.remove(frame_filepath)
|
||||
if VERBOSE:
|
||||
print(" ".join(command))
|
||||
print(e)
|
||||
return "CRASH"
|
||||
args.extend(["-f", "1"])
|
||||
|
||||
return args
|
||||
|
||||
def create_argparse():
|
||||
parser = argparse.ArgumentParser()
|
||||
|
|
@ -94,11 +53,7 @@ def main():
|
|||
parser = create_argparse()
|
||||
args = parser.parse_args()
|
||||
|
||||
global BLENDER, VERBOSE
|
||||
|
||||
BLENDER = args.blender[0]
|
||||
VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
|
||||
|
||||
blender = args.blender[0]
|
||||
test_dir = args.testdir[0]
|
||||
idiff = args.idiff[0]
|
||||
output_dir = args.outdir[0]
|
||||
|
|
@ -108,7 +63,7 @@ def main():
|
|||
report.set_pixelated(True)
|
||||
report.set_reference_dir("cycles_renders")
|
||||
report.set_compare_engines('cycles', 'eevee')
|
||||
ok = report.run(test_dir, render_file)
|
||||
ok = report.run(test_dir, blender, get_arguments, batch=True)
|
||||
|
||||
sys.exit(not ok)
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import shutil
|
|||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def setup():
|
||||
import bpy
|
||||
|
||||
|
|
@ -46,15 +45,8 @@ if inside_blender:
|
|||
sys.exit(1)
|
||||
|
||||
|
||||
def render_file(filepath, output_filepath):
|
||||
dirname = os.path.dirname(filepath)
|
||||
basedir = os.path.dirname(dirname)
|
||||
subject = os.path.basename(dirname)
|
||||
|
||||
frame_filepath = output_filepath + '0001.png'
|
||||
|
||||
command = [
|
||||
BLENDER,
|
||||
def get_arguments(filepath, output_filepath):
|
||||
return [
|
||||
"--background",
|
||||
"-noaudio",
|
||||
"--factory-startup",
|
||||
|
|
@ -67,37 +59,6 @@ def render_file(filepath, output_filepath):
|
|||
"-F", "PNG",
|
||||
"-f", "1"]
|
||||
|
||||
try:
|
||||
# Success
|
||||
output = subprocess.check_output(command)
|
||||
if os.path.exists(frame_filepath):
|
||||
shutil.copy(frame_filepath, output_filepath)
|
||||
os.remove(frame_filepath)
|
||||
if VERBOSE:
|
||||
print(" ".join(command))
|
||||
print(output.decode("utf-8"))
|
||||
return None
|
||||
except subprocess.CalledProcessError as e:
|
||||
# Error
|
||||
if os.path.exists(frame_filepath):
|
||||
os.remove(frame_filepath)
|
||||
if VERBOSE:
|
||||
print(" ".join(command))
|
||||
print(e.output.decode("utf-8"))
|
||||
if b"Error: engine not found" in e.output:
|
||||
return "NO_ENGINE"
|
||||
elif b"blender probably wont start" in e.output:
|
||||
return "NO_START"
|
||||
return "CRASH"
|
||||
except BaseException as e:
|
||||
# Crash
|
||||
if os.path.exists(frame_filepath):
|
||||
os.remove(frame_filepath)
|
||||
if VERBOSE:
|
||||
print(" ".join(command))
|
||||
print(e)
|
||||
return "CRASH"
|
||||
|
||||
|
||||
def create_argparse():
|
||||
parser = argparse.ArgumentParser()
|
||||
|
|
@ -112,11 +73,7 @@ def main():
|
|||
parser = create_argparse()
|
||||
args = parser.parse_args()
|
||||
|
||||
global BLENDER, VERBOSE
|
||||
|
||||
BLENDER = args.blender[0]
|
||||
VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
|
||||
|
||||
blender = args.blender[0]
|
||||
test_dir = args.testdir[0]
|
||||
idiff = args.idiff[0]
|
||||
output_dir = args.outdir[0]
|
||||
|
|
@ -126,7 +83,7 @@ def main():
|
|||
report.set_pixelated(True)
|
||||
report.set_reference_dir("eevee_renders")
|
||||
report.set_compare_engines('eevee', 'cycles')
|
||||
ok = report.run(test_dir, render_file)
|
||||
ok = report.run(test_dir, blender, get_arguments, batch=True)
|
||||
|
||||
sys.exit(not ok)
|
||||
|
||||
|
|
|
|||
|
|
@ -134,10 +134,10 @@ class Report:
|
|||
def set_compare_engines(self, engine, other_engine):
|
||||
self.compare_engines = (engine, other_engine)
|
||||
|
||||
def run(self, dirpath, render_cb):
|
||||
def run(self, dirpath, blender, arguments_cb, batch=False):
|
||||
# Run tests and output report.
|
||||
dirname = os.path.basename(dirpath)
|
||||
ok = self._run_all_tests(dirname, dirpath, render_cb)
|
||||
ok = self._run_all_tests(dirname, dirpath, blender, arguments_cb, batch)
|
||||
self._write_data(dirname)
|
||||
self._write_html()
|
||||
if self.compare_engines:
|
||||
|
|
@ -399,43 +399,81 @@ class Report:
|
|||
|
||||
return not failed
|
||||
|
||||
def _run_test(self, filepath, render_cb):
|
||||
testname = test_get_name(filepath)
|
||||
print_message(testname, 'SUCCESS', 'RUN')
|
||||
time_start = time.time()
|
||||
tmp_filepath = os.path.join(self.output_dir, "tmp_" + testname)
|
||||
def _run_tests(self, filepaths, blender, arguments_cb, batch):
|
||||
# Run multiple tests in a single Blender process since startup can be
|
||||
# a significant factor. In case of crashes, re-run the remaining tests.
|
||||
verbose = os.environ.get("BLENDER_VERBOSE") is not None
|
||||
|
||||
error = render_cb(filepath, tmp_filepath)
|
||||
status = "FAIL"
|
||||
if not error:
|
||||
if not self._diff_output(filepath, tmp_filepath):
|
||||
error = "VERIFY"
|
||||
remaining_filepaths = filepaths[:]
|
||||
errors = []
|
||||
|
||||
if os.path.exists(tmp_filepath):
|
||||
os.remove(tmp_filepath)
|
||||
while len(remaining_filepaths) > 0:
|
||||
command = [blender]
|
||||
output_filepaths = []
|
||||
|
||||
time_end = time.time()
|
||||
elapsed_ms = int((time_end - time_start) * 1000)
|
||||
if not error:
|
||||
print_message("{} ({} ms)" . format(testname, elapsed_ms),
|
||||
'SUCCESS', 'OK')
|
||||
else:
|
||||
if error == "NO_ENGINE":
|
||||
print_message("Can't perform tests because the render engine failed to load!")
|
||||
return error
|
||||
elif error == "NO_START":
|
||||
print_message('Can not perform tests because blender fails to start.',
|
||||
'Make sure INSTALL target was run.')
|
||||
return error
|
||||
elif error == 'VERIFY':
|
||||
print_message("Rendered result is different from reference image")
|
||||
else:
|
||||
print_message("Unknown error %r" % error)
|
||||
print_message("{} ({} ms)" . format(testname, elapsed_ms),
|
||||
'FAILURE', 'FAILED')
|
||||
return error
|
||||
# Construct output filepaths and command to run
|
||||
for filepath in remaining_filepaths:
|
||||
testname = test_get_name(filepath)
|
||||
print_message(testname, 'SUCCESS', 'RUN')
|
||||
|
||||
def _run_all_tests(self, dirname, dirpath, render_cb):
|
||||
base_output_filepath = os.path.join(self.output_dir, "tmp_" + testname)
|
||||
output_filepath = base_output_filepath + '0001.png'
|
||||
output_filepaths.append(output_filepath)
|
||||
|
||||
if os.path.exists(output_filepath):
|
||||
os.remove(output_filepath)
|
||||
|
||||
command.extend(arguments_cb(filepath, base_output_filepath))
|
||||
|
||||
# Only chain multiple commands for batch
|
||||
if not batch:
|
||||
break
|
||||
|
||||
# Run process
|
||||
crash = False
|
||||
try:
|
||||
output = subprocess.check_output(command)
|
||||
except subprocess.CalledProcessError as e:
|
||||
crash = True
|
||||
except BaseException as e:
|
||||
crash = True
|
||||
|
||||
if verbose:
|
||||
print(" ".join(command))
|
||||
print(output.decode("utf-8"))
|
||||
|
||||
# Detect missing filepaths and consider those errors
|
||||
for filepath, output_filepath in zip(remaining_filepaths[:], output_filepaths):
|
||||
remaining_filepaths.pop(0)
|
||||
|
||||
if crash:
|
||||
# In case of crash, stop after missing files and re-render remaing
|
||||
if not os.path.exists(output_filepath):
|
||||
errors.append("CRASH")
|
||||
print_message("Crash running Blender")
|
||||
print_message(testname, 'FAILURE', 'FAILED')
|
||||
break
|
||||
|
||||
testname = test_get_name(filepath)
|
||||
|
||||
if not os.path.exists(output_filepath) or os.path.getsize(output_filepath) == 0:
|
||||
errors.append("NO OUTPUT")
|
||||
print_message("No render result file found")
|
||||
print_message(testname, 'FAILURE', 'FAILED')
|
||||
elif not self._diff_output(filepath, output_filepath):
|
||||
errors.append("VERIFY")
|
||||
print_message("Render result is different from reference image")
|
||||
print_message(testname, 'FAILURE', 'FAILED')
|
||||
else:
|
||||
errors.append(None)
|
||||
print_message(testname, 'SUCCESS', 'OK')
|
||||
|
||||
if os.path.exists(output_filepath):
|
||||
os.remove(output_filepath)
|
||||
|
||||
return errors
|
||||
|
||||
def _run_all_tests(self, dirname, dirpath, blender, arguments_cb, batch):
|
||||
passed_tests = []
|
||||
failed_tests = []
|
||||
all_files = list(blend_list(dirpath))
|
||||
|
|
@ -444,8 +482,8 @@ class Report:
|
|||
format(len(all_files)),
|
||||
'SUCCESS', "==========")
|
||||
time_start = time.time()
|
||||
for filepath in all_files:
|
||||
error = self._run_test(filepath, render_cb)
|
||||
errors = self._run_tests(all_files, blender, arguments_cb, batch)
|
||||
for filepath, error in zip(all_files, errors):
|
||||
testname = test_get_name(filepath)
|
||||
if error:
|
||||
if error == "NO_ENGINE":
|
||||
|
|
|
|||
|
|
@ -31,9 +31,8 @@ if inside_blender:
|
|||
sys.exit(0)
|
||||
|
||||
|
||||
def render_file(filepath, output_filepath):
|
||||
command = (
|
||||
BLENDER,
|
||||
def get_arguments(filepath, output_filepath):
|
||||
return [
|
||||
"--no-window-focus",
|
||||
"--window-geometry",
|
||||
"0", "0", "1024", "768",
|
||||
|
|
@ -44,28 +43,7 @@ def render_file(filepath, output_filepath):
|
|||
"-P",
|
||||
os.path.realpath(__file__),
|
||||
"--",
|
||||
output_filepath)
|
||||
|
||||
try:
|
||||
# Success
|
||||
output = subprocess.check_output(command)
|
||||
if VERBOSE:
|
||||
print(output.decode("utf-8"))
|
||||
return None
|
||||
except subprocess.CalledProcessError as e:
|
||||
# Error
|
||||
if os.path.exists(output_filepath):
|
||||
os.remove(output_filepath)
|
||||
if VERBOSE:
|
||||
print(e.output.decode("utf-8"))
|
||||
return "CRASH"
|
||||
except BaseException as e:
|
||||
# Crash
|
||||
if os.path.exists(output_filepath):
|
||||
os.remove(output_filepath)
|
||||
if VERBOSE:
|
||||
print(e)
|
||||
return "CRASH"
|
||||
output_filepath + '0001.png']
|
||||
|
||||
|
||||
def create_argparse():
|
||||
|
|
@ -81,18 +59,14 @@ def main():
|
|||
parser = create_argparse()
|
||||
args = parser.parse_args()
|
||||
|
||||
global BLENDER, VERBOSE
|
||||
|
||||
BLENDER = args.blender[0]
|
||||
VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
|
||||
|
||||
blender = args.blender[0]
|
||||
test_dir = args.testdir[0]
|
||||
idiff = args.idiff[0]
|
||||
output_dir = args.outdir[0]
|
||||
|
||||
from modules import render_report
|
||||
report = render_report.Report("OpenGL Draw", output_dir, idiff)
|
||||
ok = report.run(test_dir, render_file)
|
||||
ok = report.run(test_dir, blender, get_arguments)
|
||||
|
||||
sys.exit(not ok)
|
||||
|
||||
|
|
|
|||
|
|
@ -33,15 +33,8 @@ if inside_blender:
|
|||
sys.exit(1)
|
||||
|
||||
|
||||
def render_file(filepath, output_filepath):
|
||||
dirname = os.path.dirname(filepath)
|
||||
basedir = os.path.dirname(dirname)
|
||||
subject = os.path.basename(dirname)
|
||||
|
||||
frame_filepath = output_filepath + '0001.png'
|
||||
|
||||
command = [
|
||||
BLENDER,
|
||||
def get_arguments(filepath, output_filepath):
|
||||
return [
|
||||
"--background",
|
||||
"-noaudio",
|
||||
"--factory-startup",
|
||||
|
|
@ -54,37 +47,6 @@ def render_file(filepath, output_filepath):
|
|||
"-F", "PNG",
|
||||
"-f", "1"]
|
||||
|
||||
try:
|
||||
# Success
|
||||
output = subprocess.check_output(command)
|
||||
if os.path.exists(frame_filepath):
|
||||
shutil.copy(frame_filepath, output_filepath)
|
||||
os.remove(frame_filepath)
|
||||
if VERBOSE:
|
||||
print(" ".join(command))
|
||||
print(output.decode("utf-8"))
|
||||
return None
|
||||
except subprocess.CalledProcessError as e:
|
||||
# Error
|
||||
if os.path.exists(frame_filepath):
|
||||
os.remove(frame_filepath)
|
||||
if VERBOSE:
|
||||
print(" ".join(command))
|
||||
print(e.output.decode("utf-8"))
|
||||
if b"Error: engine not found" in e.output:
|
||||
return "NO_ENGINE"
|
||||
elif b"blender probably wont start" in e.output:
|
||||
return "NO_START"
|
||||
return "CRASH"
|
||||
except BaseException as e:
|
||||
# Crash
|
||||
if os.path.exists(frame_filepath):
|
||||
os.remove(frame_filepath)
|
||||
if VERBOSE:
|
||||
print(" ".join(command))
|
||||
print(e)
|
||||
return "CRASH"
|
||||
|
||||
|
||||
def create_argparse():
|
||||
parser = argparse.ArgumentParser()
|
||||
|
|
@ -99,11 +61,7 @@ def main():
|
|||
parser = create_argparse()
|
||||
args = parser.parse_args()
|
||||
|
||||
global BLENDER, VERBOSE
|
||||
|
||||
BLENDER = args.blender[0]
|
||||
VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
|
||||
|
||||
blender = args.blender[0]
|
||||
test_dir = args.testdir[0]
|
||||
idiff = args.idiff[0]
|
||||
output_dir = args.outdir[0]
|
||||
|
|
@ -113,7 +71,7 @@ def main():
|
|||
report.set_pixelated(True)
|
||||
report.set_reference_dir("workbench_renders")
|
||||
report.set_compare_engines('workbench', 'eevee')
|
||||
ok = report.run(test_dir, render_file)
|
||||
ok = report.run(test_dir, blender, get_arguments, batch=True)
|
||||
|
||||
sys.exit(not ok)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue