From 9d08be1eec62ec4028372ed4e4175bc711c1afd3 Mon Sep 17 00:00:00 2001 From: Habib Gahbiche Date: Sat, 19 Aug 2023 15:00:54 +0200 Subject: [PATCH] Realtime Compositor: add regression tests Tests must be enabled manually using the CMake flag `WITH_COMPOSITOR_REALTIME_TEST`. Reasons are F12 for realtime compositor is experimental and buildbots have no GPU. Failing tests are excluded. Pull Request: https://projects.blender.org/blender/blender/pulls/109878 --- CMakeLists.txt | 1 + tests/python/CMakeLists.txt | 40 ++++++++- ...ests.py => compositor_cpu_render_tests.py} | 4 +- .../compositor_realtime_render_tests.py | 81 +++++++++++++++++++ 4 files changed, 121 insertions(+), 5 deletions(-) rename tests/python/{compositor_render_tests.py => compositor_cpu_render_tests.py} (93%) create mode 100644 tests/python/compositor_realtime_render_tests.py diff --git a/CMakeLists.txt b/CMakeLists.txt index cda591a8b4d..28f576820ae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -761,6 +761,7 @@ endif() option(WITH_GTESTS "Enable GTest unit testing" OFF) option(WITH_OPENGL_RENDER_TESTS "Enable OpenGL render related unit testing (Experimental)" OFF) option(WITH_OPENGL_DRAW_TESTS "Enable OpenGL UI drawing related unit testing (Experimental)" OFF) +option(WITH_COMPOSITOR_REALTIME_TEST "Enable regression testing for realtime compositor" OFF) # NOTE: All callers of this must add `TEST_PYTHON_EXE_EXTRA_ARGS` before any other arguments. set(TEST_PYTHON_EXE "" CACHE PATH "Python executable to run unit tests") mark_as_advanced(TEST_PYTHON_EXE) diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt index 4168a73f274..95d8848cc9f 100644 --- a/tests/python/CMakeLists.txt +++ b/tests/python/CMakeLists.txt @@ -717,14 +717,48 @@ if(WITH_COMPOSITOR_CPU) foreach(comp_test ${compositor_tests}) add_python_test( - compositor_${comp_test}_test - ${CMAKE_CURRENT_LIST_DIR}/compositor_render_tests.py + compositor_${comp_test}_cpu_test + ${CMAKE_CURRENT_LIST_DIR}/compositor_cpu_render_tests.py -blender "${TEST_BLENDER_EXE}" -testdir "${TEST_SRC_DIR}/compositor/${comp_test}" -idiff "${OPENIMAGEIO_IDIFF}" - -outdir "${TEST_OUT_DIR}/compositor" + -outdir "${TEST_OUT_DIR}/compositor_cpu" ) endforeach() + + endif() +endif() + +# NOTE: WITH_COMPOSITOR_CPU is needed for rendering. +if(WITH_COMPOSITOR_REALTIME_TEST AND WITH_COMPOSITOR_CPU) + if(NOT OPENIMAGEIO_IDIFF) + message(WARNING "Disabling realtime compositor tests because OIIO idiff does not exist") + else() + set(compositor_tests + color + converter + filter + input + output + vector + + multiple_node_setups + ) + + if(WITH_LIBMV) + list(APPEND compositor_tests distort matte) + endif() + + foreach(comp_test ${compositor_tests}) + add_python_test( + compositor_${comp_test}_realtime_test + ${CMAKE_CURRENT_LIST_DIR}/compositor_realtime_render_tests.py + -blender "${TEST_BLENDER_EXE}" + -testdir "${TEST_SRC_DIR}/compositor/${comp_test}" + -idiff "${OPENIMAGEIO_IDIFF}" + -outdir "${TEST_OUT_DIR}/compositor_realtime" + ) + endforeach() endif() endif() diff --git a/tests/python/compositor_render_tests.py b/tests/python/compositor_cpu_render_tests.py similarity index 93% rename from tests/python/compositor_render_tests.py rename to tests/python/compositor_cpu_render_tests.py index 6b879a600c4..df87c4fac9e 100644 --- a/tests/python/compositor_render_tests.py +++ b/tests/python/compositor_cpu_render_tests.py @@ -51,9 +51,9 @@ def main(): output_dir = args.outdir[0] from modules import render_report - report = render_report.Report("Compositor", output_dir, idiff) + report = render_report.Report("Compositor CPU", output_dir, idiff) report.set_pixelated(True) - report.set_reference_dir("compositor_renders") + report.set_reference_dir("compositor_cpu_renders") if os.path.basename(test_dir) == 'filter': # Temporary change to pass OpenImageDenoise test with both 1.3 and 1.4. diff --git a/tests/python/compositor_realtime_render_tests.py b/tests/python/compositor_realtime_render_tests.py new file mode 100644 index 00000000000..942e5626da6 --- /dev/null +++ b/tests/python/compositor_realtime_render_tests.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2015-2023 Blender Foundation +# +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import os +import sys + + +# When run from inside Blender, render and exit. +try: + import bpy + inside_blender = True +except ImportError: + inside_blender = False + +BLACKLIST_UNSUPPORTED_RENDER_TESTS = [ + 'node_cryptomatte.blend', + 'node_cryptomatte_legacy.blend', + 'node_keying_screen.blend' +] + +BLACKLIST_CRASHING_TESTS = [ + 'node_keying.blend', + 'node_keying_edge.blend', + 'node_keying_matte.blend' +] + +ENABLE_REALTIME_COMPOSITOR_SCRIPT = "import bpy; " \ + "bpy.context.preferences.experimental.use_experimental_compositors = True; " \ + "bpy.data.scenes[0].node_tree.execution_mode = 'REALTIME'" + + +def get_arguments(filepath, output_filepath): + return [ + "--background", + "-noaudio", + "--factory-startup", + "--enable-autoexec", + "--debug-memory", + "--debug-exit-on-error", + filepath, + "-P", os.path.realpath(__file__), + "--python-expr", ENABLE_REALTIME_COMPOSITOR_SCRIPT, + "-o", output_filepath, + "-F", "PNG", + "-f", "1" + ] + + +def create_argparse(): + parser = argparse.ArgumentParser() + parser.add_argument("-blender", nargs="+") + parser.add_argument("-testdir", nargs=1) + parser.add_argument("-outdir", nargs=1) + parser.add_argument("-idiff", nargs=1) + return parser + + +def main(): + parser = create_argparse() + args = parser.parse_args() + + blender = args.blender[0] + test_dir = args.testdir[0] + idiff = args.idiff[0] + output_dir = args.outdir[0] + + blacklist_all = BLACKLIST_CRASHING_TESTS + BLACKLIST_UNSUPPORTED_RENDER_TESTS + from modules import render_report + report = render_report.Report("Compositor Realtime", output_dir, idiff, blacklist=blacklist_all) + report.set_reference_dir("compositor_realtime_renders") + + ok = report.run(test_dir, blender, get_arguments, batch=True) + + sys.exit(not ok) + + +if not inside_blender and __name__ == "__main__": + main()