tmaq
/
tornavis
1
0
Fork 0
Code Issues 2 Pull Requests 29 Packages Projects 24 Releases Wiki Activity

Tests: add WIITH_TESTS_BATCHED option to execute Blender once per test 

Some tests like cycles, sequencer and compositor batch together multiple
tests in a single Blender invocation. This makes them run faster, but
makes debugging harder. This is an option to disable that batching.

Pull Request: https://projects.blender.org/blender/blender/pulls/114603
This commit is contained in:
Brecht Van Lommel 2023-11-08 18:41:33 +01:00 committed by Brecht Van Lommel
parent 8d0323e76e
commit dccbc27eab
11 changed files with 44 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CMakeLists.txt
View File

@ -773,6 +773,12 @@ else()
set(WITH_UI_TESTS OFF)
endif()
# Enabled by default for typical use cases to speed up development cycles. However, when looking
# into threading or memory related issues (in dependency graph, out-of-bounds, etc) forcing single
# test per Blender instance could give much better clues about the root of the problem.
option(WITH_TESTS_BATCHED "Run multiple tests in a single Blender invocation, for faster test execution" ON)
mark_as_advanced(WITH_TESTS_BATCHED)
# NOTE: All callers of this must add `TEST_PYTHON_EXE_EXTRA_ARGS` before any other arguments.
set(TEST_PYTHON_EXE "" CACHE PATH "Python executable to run unit tests")
mark_as_advanced(TEST_PYTHON_EXE)

53
tests/python/CMakeLists.txt
View File

@ -127,6 +127,15 @@ function(add_python_test testname testscript)
endif()
endfunction()
# Run Python render test.
function(add_render_test testname testscript)
set(_args ${ARGN} -blender "${TEST_BLENDER_EXE}" -idiff "${OPENIMAGEIO_IDIFF}")
if(WITH_TESTS_BATCHED)
list(APPEND _args --batch)
endif()
add_python_test(${testname} ${testscript} ${_args})
endfunction()
# ------------------------------------------------------------------------------
# GENERAL PYTHON CORRECTNESS TESTS
add_blender_test(
@ -629,12 +638,10 @@ if(TRUE)
set(_svg_render_tests complex path)
foreach(render_test ${_svg_render_tests})
add_python_test(
add_render_test(
io_curve_svg_${render_test}
${CMAKE_CURRENT_LIST_DIR}/bl_io_curve_svg_test.py
-blender "${TEST_BLENDER_EXE}"
-testdir "${TEST_SRC_DIR}/io_tests/svg/${render_test}"
-idiff "${OPENIMAGEIO_IDIFF}"
-outdir "${TEST_OUT_DIR}/io_curve_svg"
)
endforeach()
@ -716,12 +723,10 @@ if(WITH_CYCLES OR WITH_OPENGL_RENDER_TESTS)
foreach(render_test ${_cycles_render_tests})
set(_cycles_test_name "cycles_${render_test}_${_cycles_device_lower}")
add_python_test(
add_render_test(
${_cycles_test_name}
${CMAKE_CURRENT_LIST_DIR}/cycles_render_tests.py
-blender "${TEST_BLENDER_EXE}"
-testdir "${TEST_SRC_DIR}/render/${render_test}"
-idiff "${OPENIMAGEIO_IDIFF}"
-outdir "${TEST_OUT_DIR}/cycles"
-device ${_cycles_device}
-blacklist ${_cycles_blacklist}
@ -739,36 +744,30 @@ if(WITH_CYCLES OR WITH_OPENGL_RENDER_TESTS)
if(WITH_OPENGL_RENDER_TESTS)
# Eevee
foreach(render_test ${render_tests})
add_python_test(
add_render_test(
eevee_${render_test}_test
${CMAKE_CURRENT_LIST_DIR}/eevee_render_tests.py
-blender "${TEST_BLENDER_EXE}"
-testdir "${TEST_SRC_DIR}/render/${render_test}"
-idiff "${OPENIMAGEIO_IDIFF}"
-outdir "${TEST_OUT_DIR}/eevee"
)
endforeach()
# Eevee Next
foreach(render_test ${render_tests})
add_python_test(
add_render_test(
eevee_next_${render_test}_test
${CMAKE_CURRENT_LIST_DIR}/eevee_next_render_tests.py
-blender "${TEST_BLENDER_EXE}"
-testdir "${TEST_SRC_DIR}/render/${render_test}"
-idiff "${OPENIMAGEIO_IDIFF}"
-outdir "${TEST_OUT_DIR}/eevee_next"
)
endforeach()
foreach(render_test ${render_tests})
# Workbench
add_python_test(
add_render_test(
workbench_${render_test}_test
${CMAKE_CURRENT_LIST_DIR}/workbench_render_tests.py
-blender "${TEST_BLENDER_EXE}"
-testdir "${TEST_SRC_DIR}/render/${render_test}"
-idiff "${OPENIMAGEIO_IDIFF}"
-outdir "${TEST_OUT_DIR}/workbench"
)
endforeach()
@ -776,24 +775,20 @@ if(WITH_CYCLES OR WITH_OPENGL_RENDER_TESTS)
if(WITH_HYDRA)
# Hydra Storm
foreach(render_test ${render_tests})
add_python_test(
add_render_test(
storm_hydra_${render_test}_test
${CMAKE_CURRENT_LIST_DIR}/storm_render_tests.py
-blender "${TEST_BLENDER_EXE}"
-testdir "${TEST_SRC_DIR}/render/${render_test}"
-idiff "${OPENIMAGEIO_IDIFF}"
-outdir "${TEST_OUT_DIR}/storm_hydra"
-export_method "HYDRA"
)
endforeach()
foreach(render_test ${render_tests})
add_python_test(
add_render_test(
storm_usd_${render_test}_test
${CMAKE_CURRENT_LIST_DIR}/storm_render_tests.py
-blender "${TEST_BLENDER_EXE}"
-testdir "${TEST_SRC_DIR}/render/${render_test}"
-idiff "${OPENIMAGEIO_IDIFF}"
-outdir "${TEST_OUT_DIR}/storm_usd"
-export_method "USD"
)
@ -823,12 +818,10 @@ if(WITH_COMPOSITOR_CPU)
endif()
foreach(comp_test ${compositor_tests})
add_python_test(
add_render_test(
compositor_${comp_test}_cpu_test
${CMAKE_CURRENT_LIST_DIR}/compositor_cpu_render_tests.py
-blender "${TEST_BLENDER_EXE}"
-testdir "${TEST_SRC_DIR}/compositor/${comp_test}"
-idiff "${OPENIMAGEIO_IDIFF}"
-outdir "${TEST_OUT_DIR}/compositor_cpu"
)
endforeach()
@ -857,12 +850,10 @@ if(WITH_COMPOSITOR_REALTIME_TESTS AND WITH_COMPOSITOR_CPU)
endif()
foreach(comp_test ${compositor_tests})
add_python_test(
add_render_test(
compositor_${comp_test}_realtime_test
${CMAKE_CURRENT_LIST_DIR}/compositor_realtime_render_tests.py
-blender "${TEST_BLENDER_EXE}"
-testdir "${TEST_SRC_DIR}/compositor/${comp_test}"
-idiff "${OPENIMAGEIO_IDIFF}"
-outdir "${TEST_OUT_DIR}/compositor_realtime"
)
endforeach()
@ -945,12 +936,10 @@ if(WITH_OPENGL_DRAW_TESTS)
if(IS_DIRECTORY ${child_path})
file(GLOB_RECURSE blends "${child_path}/*.blend")
if(blends)
add_python_test(
add_render_test(
opengl_draw_${child}
${CMAKE_CURRENT_LIST_DIR}/opengl_draw_tests.py
-blender "${TEST_BLENDER_EXE}"
-testdir "${child_path}"
-idiff "${OPENIMAGEIO_IDIFF}"
-outdir "${TEST_OUT_DIR}/opengl_draw"
)
endif()
@ -1057,12 +1046,10 @@ else()
)
foreach(render_test ${render_tests})
add_python_test(
add_render_test(
sequencer_render_${render_test}
${CMAKE_CURRENT_LIST_DIR}/sequencer_render_tests.py
-blender "${TEST_BLENDER_EXE}"
-testdir "${TEST_SRC_DIR}/sequence_editing/${render_test}"
-idiff "${OPENIMAGEIO_IDIFF}"
-outdir "${TEST_OUT_DIR}/sequence_editing"
)
endforeach()

3
tests/python/bl_io_curve_svg_test.py
View File

@ -37,6 +37,7 @@ def create_argparse():
parser.add_argument("-testdir", nargs=1)
parser.add_argument("-outdir", nargs=1)
parser.add_argument("-idiff", nargs=1)
parser.add_argument('--batch', default=False, action='store_true')
return parser
@ -57,7 +58,7 @@ def main():
if test_dir_name == 'complex':
report.set_fail_percent(0.01)
ok = report.run(test_dir, blender, get_arguments, batch=True)
ok = report.run(test_dir, blender, get_arguments, batch=args.batch)
sys.exit(not ok)

3
tests/python/compositor_cpu_render_tests.py
View File

@ -38,6 +38,7 @@ def create_argparse():
parser.add_argument("-testdir", nargs=1)
parser.add_argument("-outdir", nargs=1)
parser.add_argument("-idiff", nargs=1)
parser.add_argument('--batch', default=False, action='store_true')
return parser
@ -66,7 +67,7 @@ def main():
report.set_fail_threshold(0.06)
report.set_fail_percent(2)
ok = report.run(test_dir, blender, get_arguments, batch=True)
ok = report.run(test_dir, blender, get_arguments, batch=args.batch)
sys.exit(not ok)

3
tests/python/compositor_realtime_render_tests.py
View File

@ -49,6 +49,7 @@ def create_argparse():
parser.add_argument("-testdir", nargs=1)
parser.add_argument("-outdir", nargs=1)
parser.add_argument("-idiff", nargs=1)
parser.add_argument('--batch', default=False, action='store_true')
return parser
@ -66,7 +67,7 @@ def main():
report = render_report.Report("Compositor Realtime", output_dir, idiff, blacklist=blacklist_all)
report.set_reference_dir("compositor_realtime_renders")
ok = report.run(test_dir, blender, get_arguments, batch=True)
ok = report.run(test_dir, blender, get_arguments, batch=args.batch)
sys.exit(not ok)

3
tests/python/cycles_render_tests.py
View File

@ -107,6 +107,7 @@ def create_argparse():
parser.add_argument("-idiff", nargs=1)
parser.add_argument("-device", nargs=1)
parser.add_argument("-blacklist", nargs="*")
parser.add_argument('--batch', default=False, action='store_true')
return parser
@ -147,7 +148,7 @@ def main():
if test_dir_name in ('motion_blur', 'integrator', ):
report.set_fail_threshold(0.032)
ok = report.run(test_dir, blender, get_arguments, batch=True)
ok = report.run(test_dir, blender, get_arguments, batch=args.batch)
sys.exit(not ok)

3
tests/python/eevee_next_render_tests.py
View File

@ -110,6 +110,7 @@ def create_argparse():
parser.add_argument("-testdir", nargs=1)
parser.add_argument("-outdir", nargs=1)
parser.add_argument("-idiff", nargs=1)
parser.add_argument('--batch', default=False, action='store_true')
return parser
@ -138,7 +139,7 @@ def main():
if test_dir_name.startswith('image'):
report.set_fail_threshold(0.051)
ok = report.run(test_dir, blender, get_arguments, batch=True)
ok = report.run(test_dir, blender, get_arguments, batch=args.batch)
sys.exit(not ok)

3
tests/python/eevee_render_tests.py
View File

@ -143,6 +143,7 @@ def create_argparse():
parser.add_argument("-testdir", nargs=1)
parser.add_argument("-outdir", nargs=1)
parser.add_argument("-idiff", nargs=1)
parser.add_argument('--batch', default=False, action='store_true')
return parser
@ -171,7 +172,7 @@ def main():
if test_dir_name.startswith('image'):
report.set_fail_threshold(0.051)
ok = report.run(test_dir, blender, get_arguments, batch=True)
ok = report.run(test_dir, blender, get_arguments, batch=args.batch)
sys.exit(not ok)

3
tests/python/sequencer_render_tests.py
View File

@ -34,6 +34,7 @@ def create_argparse():
parser.add_argument("-testdir", nargs=1)
parser.add_argument("-outdir", nargs=1)
parser.add_argument("-idiff", nargs=1)
parser.add_argument('--batch', default=False, action='store_true')
return parser
@ -52,7 +53,7 @@ def main():
report.set_reference_dir("reference")
test_dir_name = Path(test_dir).name
ok = report.run(test_dir, blender, get_arguments, batch=True)
ok = report.run(test_dir, blender, get_arguments, batch=args.batch)
sys.exit(not ok)

3
tests/python/storm_render_tests.py
View File

@ -62,6 +62,7 @@ def create_argparse():
parser.add_argument("-outdir", nargs=1)
parser.add_argument("-idiff", nargs=1)
parser.add_argument("-export_method", nargs=1)
parser.add_argument('--batch', default=False, action='store_true')
return parser
@ -92,7 +93,7 @@ def main():
os.environ['BLENDER_HYDRA_EXPORT_METHOD'] = export_method
ok = report.run(test_dir, blender, get_arguments, batch=True)
ok = report.run(test_dir, blender, get_arguments, batch=args.batch)
sys.exit(not ok)

3
tests/python/workbench_render_tests.py
View File

@ -57,6 +57,7 @@ def create_argparse():
parser.add_argument("-testdir", nargs=1)
parser.add_argument("-outdir", nargs=1)
parser.add_argument("-idiff", nargs=1)
parser.add_argument('--batch', default=False, action='store_true')
return parser
@ -79,7 +80,7 @@ def main():
if test_dir_name.startswith('hair') and platform.system() == "Darwin":
report.set_fail_threshold(0.050)
ok = report.run(test_dir, blender, get_arguments, batch=True)
ok = report.run(test_dir, blender, get_arguments, batch.args=batch)
sys.exit(not ok)