Realtime Compositor: Support Viewer nodes

This patch adds support for Viewer and File Output nodes to the realtime
compositor. The experimental render GPU compositor was also extended to
support viewers. While support for File Output nodes was added, it
remains unimplemented.

This is just an experimental implementation, the logic for viewers will
probably be changed once #108656 is agreed upon. Furthermore, the recalc
NODE_DO_OUTPUT_RECALC flags need to be taken into account to avoid
superfluous computations.

Pull Request: https://projects.blender.org/blender/blender/pulls/108804

source/blender/compositor/realtime_compositor/COM_context.hh

@@ -45,9 +45,14 @@ class Context {
   /* Get the node tree used for compositing. */
   virtual const bNodeTree &get_node_tree() const = 0;
 
-  /* True if compositor should do write file outputs, false if only running for viewing. */
+  /* True if the compositor should write file outputs, false otherwise. */
   virtual bool use_file_output() const = 0;
 
+  /* True if the compositor should write the composite output, otherwise, the compositor is assumed
+   * to not support the composite output and just displays its viewer output. In that case, the
+   * composite output will be used as a fallback viewer if no other viewer exists */
+  virtual bool use_composite_output() const = 0;
+
   /* True if color management should be used for texture evaluation. */
   virtual bool use_texture_color_management() const = 0;
 
@@ -66,10 +71,14 @@ class Context {
    * region. */
   virtual rcti get_compositing_region() const = 0;
 
-  /* Get the texture representing the output where the result of the compositor should be
-   * written. This should be called by output nodes to get their target texture. */
+  /* Get the texture where the result of the compositor should be written. This should be called by
+   * the composite output node to get its target texture. */
   virtual GPUTexture *get_output_texture() = 0;
 
+  /* Get the texture where the result of the compositor viewer should be written. This should be
+   * called by viewer output nodes to get their target texture. */
+  virtual GPUTexture *get_viewer_output_texture() = 0;
+
   /* Get the texture where the given render pass is stored. This should be called by the Render
    * Layer node to populate its outputs. */
   virtual GPUTexture *get_input_texture(int view_layer, const char *pass_name) = 0;

source/blender/compositor/realtime_compositor/COM_scheduler.hh

@@ -8,6 +8,8 @@
 
 #include "NOD_derived_node_tree.hh"
 
+#include "COM_context.hh"
+
 namespace blender::realtime_compositor {
 
 using namespace nodes::derived_node_tree_types;
@@ -18,6 +20,6 @@ using Schedule = VectorSet<DNode>;
 /* Computes the execution schedule of the node tree. This is essentially a post-order depth first
  * traversal of the node tree from the output node to the leaf input nodes, with informed order of
  * traversal of dependencies based on a heuristic estimation of the number of needed buffers. */
-Schedule compute_schedule(const DerivedNodeTree &tree);
+Schedule compute_schedule(const Context &context, const DerivedNodeTree &tree);
 
 }  // namespace blender::realtime_compositor

source/blender/compositor/realtime_compositor/intern/evaluator.cc

@@ -72,7 +72,7 @@ void Evaluator::compile_and_evaluate()
     return;
   }
 
-  const Schedule schedule = compute_schedule(*derived_node_tree_);
+  const Schedule schedule = compute_schedule(context_, *derived_node_tree_);
 
   CompileState compile_state(schedule);
 

source/blender/compositor/realtime_compositor/intern/scheduler.cc

@@ -13,6 +13,7 @@
 #include "BKE_node.hh"
 #include "BKE_node_runtime.hh"
 
+#include "COM_context.hh"
 #include "COM_scheduler.hh"
 #include "COM_utilities.hh"
 
@@ -72,55 +73,88 @@ static const DTreeContext *find_active_context(const DerivedNodeTree &tree)
   return find_active_context_recursive(&tree.root_context(), NODE_INSTANCE_KEY_BASE);
 }
 
-/* Return the output node which is marked as NODE_DO_OUTPUT. If multiple types of output nodes are
- * marked, then the preference will be CMP_NODE_VIEWER > CMP_NODE_SPLITVIEWER > CMP_NODE_COMPOSITE.
- * If no output node exists, a null node will be returned. */
-static DNode find_output_in_context(const DTreeContext *context)
+/* Add the viewer node which is marked as NODE_DO_OUTPUT in the given context to the given stack.
+ * If multiple types of viewer nodes are marked, then the preference will be CMP_NODE_VIEWER >
+ * CMP_NODE_SPLITVIEWER. If no viewer nodes were found, composite nodes can be added as a fallback
+ * viewer node. */
+static bool add_viewer_nodes_in_context(const DTreeContext *context, Stack<DNode> &node_stack)
 {
-  const bNodeTree &tree = context->btree();
-
-  for (const bNode *node : tree.nodes_by_type("CompositorNodeViewer")) {
+  for (const bNode *node : context->btree().nodes_by_type("CompositorNodeViewer")) {
     if (node->flag & NODE_DO_OUTPUT) {
-      return DNode(context, node);
+      node_stack.push(DNode(context, node));
+      return true;
     }
   }
 
-  for (const bNode *node : tree.nodes_by_type("CompositorNodeSplitViewer")) {
+  for (const bNode *node : context->btree().nodes_by_type("CompositorNodeSplitViewer")) {
     if (node->flag & NODE_DO_OUTPUT) {
-      return DNode(context, node);
+      node_stack.push(DNode(context, node));
+      return true;
     }
   }
 
-  for (const bNode *node : tree.nodes_by_type("CompositorNodeComposite")) {
+  /* The active Composite node was already added, no need to add it again, see the next block. */
+  if (!node_stack.is_empty() && node_stack.peek()->type == CMP_NODE_COMPOSITE) {
+    return false;
+  }
+
+  /* No active viewers exist in this context, try to add the Composite node as a fallback viewer if
+   * it was not already added. */
+  for (const bNode *node : context->btree().nodes_by_type("CompositorNodeComposite")) {
     if (node->flag & NODE_DO_OUTPUT) {
-      return DNode(context, node);
+      node_stack.push(DNode(context, node));
+      return true;
     }
   }
 
-  return DNode();
+  return false;
 }
 
-/* Compute the output node whose result should be computed. This node is the output node that
- * satisfies the requirements in the find_output_in_context function. First, the active context is
- * searched for an output node, if non was found, the root context is search. For more information
- * on what contexts mean here, see the find_active_context function. */
-static DNode compute_output_node(const DerivedNodeTree &tree)
+/* Add the output nodes whose result should be computed to the given stack. This includes File
+ * Output, Composite, and Viewer nodes. Viewer nodes are a special case, as only the nodes that
+ * satisfies the requirements in the add_viewer_nodes_in_context function are added. First, the
+ * active context is searched for viewer nodes, if non were found, the root context is searched.
+ * For more information on what contexts mean here, see the find_active_context function. */
+static void add_output_nodes(const Context &context,
+                             const DerivedNodeTree &tree,
+                             Stack<DNode> &node_stack)
 {
+  const DTreeContext &root_context = tree.root_context();
+
+  /* Only add File Output nodes if the context supports them. */
+  if (context.use_file_output()) {
+    for (const bNode *node : root_context.btree().nodes_by_type("CompositorNodeOutputFile")) {
+      node_stack.push(DNode(&root_context, node));
+    }
+  }
+
+  /* Only add the Composite output node if the context supports composite outputs. The active
+   * Composite node may still be added as a fallback viewer output below. */
+  if (context.use_composite_output()) {
+    for (const bNode *node : root_context.btree().nodes_by_type("CompositorNodeComposite")) {
+      if (node->flag & NODE_DO_OUTPUT) {
+        node_stack.push(DNode(&root_context, node));
+        break;
+      }
+    }
+  }
+
   const DTreeContext *active_context = find_active_context(tree);
+  const bool viewer_was_added = add_viewer_nodes_in_context(active_context, node_stack);
 
-  const DNode node = find_output_in_context(active_context);
-  if (node) {
-    return node;
+  /* An active viewer was added, no need to search further. */
+  if (viewer_was_added) {
+    return;
   }
 
-  /* If the active context is the root one and no output node was found, we consider this node tree
-   * to have no output node, even if one of the non-active descendants have an output node. */
+  /* If the active context is the root one and no viewer nodes were found, we consider this node
+   * tree to have no viewer nodes, even if one of the non-active descendants have viewer nodes. */
   if (active_context->is_root()) {
-    return DNode();
+    return;
   }
 
-  /* The active context doesn't have an output node, search in the root context as a fallback. */
-  return find_output_in_context(&tree.root_context());
+  /* The active context doesn't have a viewer node, search in the root context as a fallback. */
+  add_viewer_nodes_in_context(&tree.root_context(), node_stack);
 }
 
 /* A type representing a mapping that associates each node with a heuristic estimation of the
@@ -177,12 +211,12 @@ using NeededBuffers = Map<DNode, int>;
  *   implementation because it rarely affects the output and is done by very few nodes.
  * - The compiler may decide to compiler the schedule differently depending on runtime information
  *   which we can merely speculate at scheduling-time as described above. */
-static NeededBuffers compute_number_of_needed_buffers(DNode output_node)
+static NeededBuffers compute_number_of_needed_buffers(Stack<DNode> &output_nodes)
 {
   NeededBuffers needed_buffers;
 
-  /* A stack of nodes used to traverse the node tree starting from the output node. */
-  Stack<DNode> node_stack = {output_node};
+  /* A stack of nodes used to traverse the node tree starting from the output nodes. */
+  Stack<DNode> node_stack = output_nodes;
 
   /* Traverse the node tree in a post order depth first manner and compute the number of needed
    * buffers for each node. Post order traversal guarantee that all the node dependencies of each
@@ -301,23 +335,23 @@ static NeededBuffers compute_number_of_needed_buffers(DNode output_node)
  * doesn't always guarantee an optimal evaluation order, as the optimal evaluation order is very
  * difficult to compute, however, this method works well in most cases. Moreover it assumes that
  * all buffers will have roughly the same size, which may not always be the case. */
-Schedule compute_schedule(const DerivedNodeTree &tree)
+Schedule compute_schedule(const Context &context, const DerivedNodeTree &tree)
 {
   Schedule schedule;
 
-  /* Compute the output node whose result should be computed. */
-  const DNode output_node = compute_output_node(tree);
+  /* A stack of nodes used to traverse the node tree starting from the output nodes. */
+  Stack<DNode> node_stack;
 
-  /* No output node, the node tree has no effect, return an empty schedule. */
-  if (!output_node) {
+  /* Add the output nodes whose result should be computed to the stack. */
+  add_output_nodes(context, tree, node_stack);
+
+  /* No output nodes, the node tree has no effect, return an empty schedule. */
+  if (node_stack.is_empty()) {
     return schedule;
   }
 
-  /* Compute the number of buffers needed by each node connected to the output. */
-  const NeededBuffers needed_buffers = compute_number_of_needed_buffers(output_node);
-
-  /* A stack of nodes used to traverse the node tree starting from the output node. */
-  Stack<DNode> node_stack = {output_node};
+  /* Compute the number of buffers needed by each node connected to the outputs. */
+  const NeededBuffers needed_buffers = compute_number_of_needed_buffers(node_stack);
 
   /* Traverse the node tree in a post order depth first manner, scheduling the nodes in an order
    * informed by the number of buffers needed by each node. Post order traversal guarantee that all
@@ -360,7 +394,8 @@ Schedule compute_schedule(const DerivedNodeTree &tree)
       int insertion_position = 0;
       for (int i = 0; i < sorted_dependency_nodes.size(); i++) {
         if (needed_buffers.lookup(doutput.node()) >
-            needed_buffers.lookup(sorted_dependency_nodes[i])) {
+            needed_buffers.lookup(sorted_dependency_nodes[i]))
+        {
           insertion_position++;
         }
         else {

source/blender/draw/engines/compositor/compositor_engine.cc

@@ -68,6 +68,14 @@ class Context : public realtime_compositor::Context {
     return false;
   }
 
+  /* The viewport compositor doesn't really support the composite output, it only displays the
+   * viewer output in the viewport. Settings this to false will make the compositor use the
+   * composite output as fallback viewer if no other viewer exists. */
+  bool use_composite_output() const override
+  {
+    return false;
+  }
+
   bool use_texture_color_management() const override
   {
     return BKE_scene_check_color_management_enabled(DRW_context_state_get()->scene);
@@ -145,6 +153,11 @@ class Context : public realtime_compositor::Context {
     return DRW_viewport_texture_list_get()->color;
   }
 
+  GPUTexture *get_viewer_output_texture() override
+  {
+    return DRW_viewport_texture_list_get()->color;
+  }
+
   GPUTexture *get_input_texture(int view_layer, const char *pass_name) override
   {
     if (view_layer == 0 && STREQ(pass_name, RE_PASSNAME_COMBINED)) {

source/blender/nodes/composite/nodes/node_composite_split_viewer.cc

@@ -77,7 +77,7 @@ class ViewerOperation : public NodeOperation {
     const Result &second_image = get_input("Image_001");
     second_image.bind_as_texture(shader, "second_image_tx");
 
-    GPUTexture *output_texture = context().get_output_texture();
+    GPUTexture *output_texture = context().get_viewer_output_texture();
     const int image_unit = GPU_shader_get_sampler_binding(shader, "output_img");
     GPU_texture_image_bind(output_texture, image_unit);
 

source/blender/nodes/composite/nodes/node_composite_viewer.cc

@@ -105,7 +105,7 @@ class ViewerOperation : public NodeOperation {
       color.w = alpha.get_float_value();
     }
 
-    GPU_texture_clear(context().get_output_texture(), GPU_DATA_FLOAT, color);
+    GPU_texture_clear(context().get_viewer_output_texture(), GPU_DATA_FLOAT, color);
   }
 
   /* Executes when the alpha channel of the image is ignored. */
@@ -123,7 +123,7 @@ class ViewerOperation : public NodeOperation {
     const Result &image = get_input("Image");
     image.bind_as_texture(shader, "input_tx");
 
-    GPUTexture *output_texture = context().get_output_texture();
+    GPUTexture *output_texture = context().get_viewer_output_texture();
     const int image_unit = GPU_shader_get_sampler_binding(shader, "output_img");
     GPU_texture_image_bind(output_texture, image_unit);
 
@@ -151,7 +151,7 @@ class ViewerOperation : public NodeOperation {
     const Result &image = get_input("Image");
     image.bind_as_texture(shader, "input_tx");
 
-    GPUTexture *output_texture = context().get_output_texture();
+    GPUTexture *output_texture = context().get_viewer_output_texture();
     const int image_unit = GPU_shader_get_sampler_binding(shader, "output_img");
     GPU_texture_image_bind(output_texture, image_unit);
 
@@ -181,7 +181,7 @@ class ViewerOperation : public NodeOperation {
     const Result &alpha = get_input("Alpha");
     alpha.bind_as_texture(shader, "alpha_tx");
 
-    GPUTexture *output_texture = context().get_output_texture();
+    GPUTexture *output_texture = context().get_viewer_output_texture();
     const int image_unit = GPU_shader_get_sampler_binding(shader, "output_img");
     GPU_texture_image_bind(output_texture, image_unit);
 

source/blender/render/intern/compositor.cc

@@ -2,9 +2,13 @@
  *
  * SPDX-License-Identifier: GPL-2.0-or-later */
 
+#include <cstring>
+
 #include "BLI_threads.h"
 #include "BLI_vector.hh"
 
+#include "MEM_guardedalloc.h"
+
 #include "BKE_global.h"
 #include "BKE_image.h"
 #include "BKE_node.hh"
@@ -12,6 +16,9 @@
 
 #include "DRW_engine.h"
 
+#include "IMB_colormanagement.h"
+#include "IMB_imbuf.h"
+
 #include "COM_context.hh"
 #include "COM_evaluator.hh"
 
@@ -63,6 +70,9 @@ class Context : public realtime_compositor::Context {
   /* Output combined texture. */
   GPUTexture *output_texture_ = nullptr;
 
+  /* Viewer output texture. */
+  GPUTexture *viewer_output_texture_ = nullptr;
+
  public:
   Context(const Scene &scene,
           const RenderData &render_data,
@@ -81,7 +91,8 @@ class Context : public realtime_compositor::Context {
 
   virtual ~Context()
   {
-    GPU_texture_free(output_texture_);
+    GPU_TEXTURE_FREE_SAFE(output_texture_);
+    GPU_TEXTURE_FREE_SAFE(viewer_output_texture_);
   }
 
   const bNodeTree &get_node_tree() const override
@@ -94,6 +105,11 @@ class Context : public realtime_compositor::Context {
     return use_file_output_;
   }
 
+  bool use_composite_output() const override
+  {
+    return true;
+  }
+
   bool use_texture_color_management() const override
   {
     return BKE_scene_check_color_management_enabled(&scene_);
@@ -121,7 +137,7 @@ class Context : public realtime_compositor::Context {
 
   GPUTexture *get_output_texture() override
   {
-    /* TODO: support outputting for viewers and previews.
+    /* TODO: support outputting for previews.
      * TODO: just a temporary hack, needs to get stored in RenderResult,
      * once that supports GPU buffers. */
     if (output_texture_ == nullptr) {
@@ -138,6 +154,25 @@ class Context : public realtime_compositor::Context {
     return output_texture_;
   }
 
+  GPUTexture *get_viewer_output_texture() override
+  {
+    /* TODO: support outputting previews.
+     * TODO: just a temporary hack, needs to get stored in RenderResult,
+     * once that supports GPU buffers. */
+    if (viewer_output_texture_ == nullptr) {
+      const int2 size = get_render_size();
+      viewer_output_texture_ = GPU_texture_create_2d("compositor_viewer_output_texture",
+                                                     size.x,
+                                                     size.y,
+                                                     1,
+                                                     GPU_RGBA16F,
+                                                     GPU_TEXTURE_USAGE_GENERAL,
+                                                     NULL);
+    }
+
+    return viewer_output_texture_;
+  }
+
   GPUTexture *get_input_texture(int view_layer_id, const char *pass_name) override
   {
     /* TODO: eventually this should get cached on the RenderResult itself when
@@ -224,6 +259,10 @@ class Context : public realtime_compositor::Context {
 
   void output_to_render_result()
   {
+    if (!output_texture_) {
+      return;
+    }
+
     Render *re = RE_GetSceneRender(&scene_);
     RenderResult *rr = RE_AcquireResultWrite(re);
 
@@ -253,6 +292,55 @@ class Context : public realtime_compositor::Context {
     BKE_image_signal(G.main, image, nullptr, IMA_SIGNAL_FREE);
     BLI_thread_unlock(LOCK_DRAW_IMAGE);
   }
+
+  void viewer_output_to_viewer_image()
+  {
+    if (!viewer_output_texture_) {
+      return;
+    }
+
+    Image *image = BKE_image_ensure_viewer(G.main, IMA_TYPE_COMPOSITE, "Viewer Node");
+
+    ImageUser image_user = {0};
+    image_user.multi_index = BKE_scene_multiview_view_id_get(&render_data_, view_name_);
+
+    if (BKE_scene_multiview_is_render_view_first(&render_data_, view_name_)) {
+      BKE_image_ensure_viewer_views(&render_data_, image, &image_user);
+    }
+
+    BLI_thread_lock(LOCK_DRAW_IMAGE);
+
+    void *lock;
+    ImBuf *image_buffer = BKE_image_acquire_ibuf(image, &image_user, &lock);
+
+    const int2 render_size = get_render_size();
+    if (image_buffer->x != render_size.x || image_buffer->y != render_size.y) {
+      imb_freerectImBuf(image_buffer);
+      imb_freerectfloatImBuf(image_buffer);
+      IMB_freezbuffloatImBuf(image_buffer);
+      image_buffer->x = render_size.x;
+      image_buffer->y = render_size.y;
+      imb_addrectfloatImBuf(image_buffer, 4);
+      image_buffer->userflags |= IB_DISPLAY_BUFFER_INVALID;
+    }
+
+    BKE_image_release_ibuf(image, image_buffer, lock);
+    BLI_thread_unlock(LOCK_DRAW_IMAGE);
+
+    GPU_memory_barrier(GPU_BARRIER_TEXTURE_UPDATE);
+    float *output_buffer = (float *)GPU_texture_read(viewer_output_texture_, GPU_DATA_FLOAT, 0);
+
+    std::memcpy(image_buffer->float_buffer.data,
+                output_buffer,
+                render_size.x * render_size.y * 4 * sizeof(float));
+
+    MEM_freeN(output_buffer);
+
+    BKE_image_partial_update_mark_full_update(image);
+    if (node_tree_.runtime->update_draw) {
+      node_tree_.runtime->update_draw(node_tree_.runtime->udh);
+    }
+  }
 };
 
 /* Render Realtime Compositor */
@@ -289,6 +377,7 @@ void RealtimeCompositor::execute()
   DRW_render_context_enable(&render_);
   evaluator_->evaluate();
   context_->output_to_render_result();
+  context_->viewer_output_to_viewer_image();
   DRW_render_context_disable(&render_);
 }