Realtime Compositor: add regression tests

Tests must be enabled manually using the CMake flag `WITH_COMPOSITOR_REALTIME_TEST`. Reasons are F12 for realtime compositor is experimental and buildbots have no GPU. Failing tests are excluded.

Pull Request: https://projects.blender.org/blender/blender/pulls/109878

CMakeLists.txt

@@ -761,6 +761,7 @@ endif()
 option(WITH_GTESTS "Enable GTest unit testing" OFF)
 option(WITH_OPENGL_RENDER_TESTS "Enable OpenGL render related unit testing (Experimental)" OFF)
 option(WITH_OPENGL_DRAW_TESTS "Enable OpenGL UI drawing related unit testing (Experimental)" OFF)
+option(WITH_COMPOSITOR_REALTIME_TEST "Enable regression testing for realtime compositor" OFF)
 # NOTE: All callers of this must add `TEST_PYTHON_EXE_EXTRA_ARGS` before any other arguments.
 set(TEST_PYTHON_EXE "" CACHE PATH "Python executable to run unit tests")
 mark_as_advanced(TEST_PYTHON_EXE)

tests/python/CMakeLists.txt

@@ -717,14 +717,48 @@ if(WITH_COMPOSITOR_CPU)
 
     foreach(comp_test ${compositor_tests})
       add_python_test(
-        compositor_${comp_test}_test
-        ${CMAKE_CURRENT_LIST_DIR}/compositor_render_tests.py
+        compositor_${comp_test}_cpu_test
+        ${CMAKE_CURRENT_LIST_DIR}/compositor_cpu_render_tests.py
         -blender "${TEST_BLENDER_EXE}"
         -testdir "${TEST_SRC_DIR}/compositor/${comp_test}"
         -idiff "${OPENIMAGEIO_IDIFF}"
-        -outdir "${TEST_OUT_DIR}/compositor"
+        -outdir "${TEST_OUT_DIR}/compositor_cpu"
       )
     endforeach()
+
+  endif()
+endif()
+
+# NOTE: WITH_COMPOSITOR_CPU is needed for rendering. 
+if(WITH_COMPOSITOR_REALTIME_TEST AND WITH_COMPOSITOR_CPU)
+  if(NOT OPENIMAGEIO_IDIFF)
+      message(WARNING "Disabling realtime compositor tests because OIIO idiff does not exist")
+    else()
+      set(compositor_tests
+          color
+          converter
+          filter
+          input
+          output
+          vector
+
+          multiple_node_setups
+        )
+      
+      if(WITH_LIBMV)
+        list(APPEND compositor_tests distort matte)
+      endif()
+
+      foreach(comp_test ${compositor_tests})
+        add_python_test(
+          compositor_${comp_test}_realtime_test
+          ${CMAKE_CURRENT_LIST_DIR}/compositor_realtime_render_tests.py
+          -blender "${TEST_BLENDER_EXE}"
+          -testdir "${TEST_SRC_DIR}/compositor/${comp_test}"
+          -idiff "${OPENIMAGEIO_IDIFF}"
+          -outdir "${TEST_OUT_DIR}/compositor_realtime"
+        )
+      endforeach()
   endif()
 endif()
 

tests/python/compositor_cpu_render_tests.py

@@ -51,9 +51,9 @@ def main():
     output_dir = args.outdir[0]
 
     from modules import render_report
-    report = render_report.Report("Compositor", output_dir, idiff)
+    report = render_report.Report("Compositor CPU", output_dir, idiff)
     report.set_pixelated(True)
-    report.set_reference_dir("compositor_renders")
+    report.set_reference_dir("compositor_cpu_renders")
 
     if os.path.basename(test_dir) == 'filter':
         # Temporary change to pass OpenImageDenoise test with both 1.3 and 1.4.

tests/python/compositor_realtime_render_tests.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: 2015-2023 Blender Foundation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import argparse
+import os
+import sys
+
+
+# When run from inside Blender, render and exit.
+try:
+    import bpy
+    inside_blender = True
+except ImportError:
+    inside_blender = False
+
+BLACKLIST_UNSUPPORTED_RENDER_TESTS = [
+    'node_cryptomatte.blend',
+    'node_cryptomatte_legacy.blend',
+    'node_keying_screen.blend'
+]
+
+BLACKLIST_CRASHING_TESTS = [
+    'node_keying.blend',
+    'node_keying_edge.blend',
+    'node_keying_matte.blend'
+]
+
+ENABLE_REALTIME_COMPOSITOR_SCRIPT = "import bpy; " \
+    "bpy.context.preferences.experimental.use_experimental_compositors = True; " \
+    "bpy.data.scenes[0].node_tree.execution_mode = 'REALTIME'"
+
+
+def get_arguments(filepath, output_filepath):
+    return [
+        "--background",
+        "-noaudio",
+        "--factory-startup",
+        "--enable-autoexec",
+        "--debug-memory",
+        "--debug-exit-on-error",
+        filepath,
+        "-P", os.path.realpath(__file__),
+        "--python-expr", ENABLE_REALTIME_COMPOSITOR_SCRIPT,
+        "-o", output_filepath,
+        "-F", "PNG",
+        "-f", "1"
+    ]
+
+
+def create_argparse():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-blender", nargs="+")
+    parser.add_argument("-testdir", nargs=1)
+    parser.add_argument("-outdir", nargs=1)
+    parser.add_argument("-idiff", nargs=1)
+    return parser
+
+
+def main():
+    parser = create_argparse()
+    args = parser.parse_args()
+
+    blender = args.blender[0]
+    test_dir = args.testdir[0]
+    idiff = args.idiff[0]
+    output_dir = args.outdir[0]
+
+    blacklist_all = BLACKLIST_CRASHING_TESTS + BLACKLIST_UNSUPPORTED_RENDER_TESTS
+    from modules import render_report
+    report = render_report.Report("Compositor Realtime", output_dir, idiff, blacklist=blacklist_all)
+    report.set_reference_dir("compositor_realtime_renders")
+
+    ok = report.run(test_dir, blender, get_arguments, batch=True)
+
+    sys.exit(not ok)
+
+
+if not inside_blender and __name__ == "__main__":
+    main()