Fix T75936: Alembic, allow exporting of invisible objects

Add a new depsgraph builder class that includes invisible objects and
use that in the Alembic exporter.

Alembic supports three options for visibility, "visible", "inherited",
and "hidden". This means that parents can be hidden and still have
visible children (contrary to USD, where invisibility is used to prune
an entire scene graph subtree). Because of this, the visibility is
stored on the transform node, as that represents the Object in Blender
and thus keeps the Alembic file as close to Blender's own structure as
possible.

Reviewed By: Sergey

Differential Revision: https://developer.blender.org/D8595

source/blender/depsgraph/CMakeLists.txt

@@ -55,6 +55,7 @@ set(SRC
   intern/builder/deg_builder_rna.cc
   intern/builder/deg_builder_transitive.cc
   intern/builder/pipeline.cc
+  intern/builder/pipeline_all_objects.cc
   intern/builder/pipeline_compositor.cc
   intern/builder/pipeline_from_ids.cc
   intern/builder/pipeline_render.cc
@@ -116,6 +117,7 @@ set(SRC
   intern/builder/deg_builder_rna.h
   intern/builder/deg_builder_transitive.h
   intern/builder/pipeline.h
+  intern/builder/pipeline_all_objects.h
   intern/builder/pipeline_compositor.h
   intern/builder/pipeline_from_ids.h
   intern/builder/pipeline_render.h

source/blender/depsgraph/DEG_depsgraph_build.h

@@ -56,6 +56,12 @@ void DEG_graph_build_from_view_layer(struct Depsgraph *graph,
                                      struct Scene *scene,
                                      struct ViewLayer *view_layer);
 
+/* Build depsgraph for all objects (so also invisible ones) in the given view layer. */
+void DEG_graph_build_for_all_objects(struct Depsgraph *graph,
+                                     struct Main *bmain,
+                                     struct Scene *scene,
+                                     struct ViewLayer *view_layer);
+
 /* Special version of builder which produces dependency graph suitable for the render pipeline.
  * It will contain sequencer and compositor (if needed) and all their dependencies. */
 void DEG_graph_build_for_render_pipeline(struct Depsgraph *graph,

source/blender/depsgraph/intern/builder/pipeline_all_objects.cc

@@ -0,0 +1,80 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2020 Blender Foundation.
+ * All rights reserved.
+ */
+
+#include "pipeline_all_objects.h"
+
+#include "intern/builder/deg_builder_nodes.h"
+#include "intern/builder/deg_builder_relations.h"
+#include "intern/depsgraph.h"
+
+#include "DNA_layer_types.h"
+
+namespace blender {
+namespace deg {
+
+namespace {
+
+class AllObjectsNodeBuilder : public DepsgraphNodeBuilder {
+ public:
+  AllObjectsNodeBuilder(Main *bmain, Depsgraph *graph, DepsgraphBuilderCache *cache)
+      : DepsgraphNodeBuilder(bmain, graph, cache)
+  {
+  }
+
+  virtual bool need_pull_base_into_graph(Base * /*base*/) override
+  {
+    return true;
+  }
+};
+
+class AllObjectsRelationBuilder : public DepsgraphRelationBuilder {
+ public:
+  AllObjectsRelationBuilder(Main *bmain, Depsgraph *graph, DepsgraphBuilderCache *cache)
+      : DepsgraphRelationBuilder(bmain, graph, cache)
+  {
+  }
+
+  virtual bool need_pull_base_into_graph(Base * /*base*/) override
+  {
+    return true;
+  }
+};
+
+}  // namespace
+
+AllObjectsBuilderPipeline::AllObjectsBuilderPipeline(::Depsgraph *graph,
+                                                     Main *bmain,
+                                                     Scene *scene,
+                                                     ViewLayer *view_layer)
+    : ViewLayerBuilderPipeline(graph, bmain, scene, view_layer)
+{
+}
+
+unique_ptr<DepsgraphNodeBuilder> AllObjectsBuilderPipeline::construct_node_builder()
+{
+  return std::make_unique<AllObjectsNodeBuilder>(bmain_, deg_graph_, &builder_cache_);
+}
+
+unique_ptr<DepsgraphRelationBuilder> AllObjectsBuilderPipeline::construct_relation_builder()
+{
+  return std::make_unique<AllObjectsRelationBuilder>(bmain_, deg_graph_, &builder_cache_);
+}
+
+}  // namespace deg
+}  // namespace blender

source/blender/depsgraph/intern/builder/pipeline_all_objects.h

@@ -0,0 +1,44 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2020 Blender Foundation.
+ * All rights reserved.
+ */
+
+/** \file
+ * \ingroup depsgraph
+ */
+
+#pragma once
+
+#include "pipeline_view_layer.h"
+
+namespace blender {
+namespace deg {
+
+/* Builds a dependency graph that contains all objects in the view layer.
+ * This is contrary to the regular ViewLayerBuilderPipeline, which is limited to visible objects
+ * (and their dependencies). */
+class AllObjectsBuilderPipeline : public ViewLayerBuilderPipeline {
+ public:
+  AllObjectsBuilderPipeline(::Depsgraph *graph, Main *bmain, Scene *scene, ViewLayer *view_layer);
+
+ protected:
+  virtual unique_ptr<DepsgraphNodeBuilder> construct_node_builder() override;
+  virtual unique_ptr<DepsgraphRelationBuilder> construct_relation_builder() override;
+};
+
+}  // namespace deg
+}  // namespace blender

source/blender/depsgraph/intern/depsgraph_build.cc

@@ -44,6 +44,7 @@
 #include "DEG_depsgraph_debug.h"
 
 #include "builder/deg_builder_relations.h"
+#include "builder/pipeline_all_objects.h"
 #include "builder/pipeline_compositor.h"
 #include "builder/pipeline_from_ids.h"
 #include "builder/pipeline_render.h"
@@ -218,6 +219,15 @@ void DEG_graph_build_from_view_layer(Depsgraph *graph,
   builder.build();
 }
 
+void DEG_graph_build_for_all_objects(struct Depsgraph *graph,
+                                     struct Main *bmain,
+                                     struct Scene *scene,
+                                     struct ViewLayer *view_layer)
+{
+  deg::AllObjectsBuilderPipeline builder(graph, bmain, scene, view_layer);
+  builder.build();
+}
+
 void DEG_graph_build_for_render_pipeline(Depsgraph *graph,
                                          Main *bmain,
                                          Scene *scene,

source/blender/io/alembic/exporter/abc_export_capi.cc

@@ -67,11 +67,17 @@ namespace io {
 namespace alembic {
 
 // Construct the depsgraph for exporting.
-static void build_depsgraph(Depsgraph *depsgraph, Main *bmain)
+static void build_depsgraph(Depsgraph *depsgraph, Main *bmain, const bool visible_objects_only)
 {
   Scene *scene = DEG_get_input_scene(depsgraph);
   ViewLayer *view_layer = DEG_get_input_view_layer(depsgraph);
-  DEG_graph_build_from_view_layer(depsgraph, bmain, scene, view_layer);
+
+  if (visible_objects_only) {
+    DEG_graph_build_from_view_layer(depsgraph, bmain, scene, view_layer);
+  }
+  else {
+    DEG_graph_build_for_all_objects(depsgraph, bmain, scene, view_layer);
+  }
 }
 
 static void export_startjob(void *customdata,
@@ -91,7 +97,7 @@ static void export_startjob(void *customdata,
   *progress = 0.0f;
   *do_update = true;
 
-  build_depsgraph(data->depsgraph, data->bmain);
+  build_depsgraph(data->depsgraph, data->bmain, data->params.visible_objects_only);
   SubdivModifierDisabler subdiv_disabler(data->depsgraph);
   if (!data->params.apply_subdiv) {
     subdiv_disabler.disable_modifiers();

source/blender/io/alembic/exporter/abc_writer_abstract.cc

@@ -25,6 +25,10 @@
 
 #include "DNA_modifier_types.h"
 
+#include "DEG_depsgraph.h"
+
+#include <Alembic/AbcGeom/Visibility.h>
+
 #include "CLG_log.h"
 static CLG_LogRef LOG = {"io.alembic"};
 
@@ -96,6 +100,18 @@ void ABCAbstractWriter::update_bounding_box(Object *object)
   bounding_box_.max.z = -bb->vec[0][1];
 }
 
+void ABCAbstractWriter::write_visibility(const HierarchyContext &context)
+{
+  const bool is_visible = context.is_object_visible(DAG_EVAL_RENDER);
+  Alembic::Abc::OObject abc_object = get_alembic_object();
+
+  if (!abc_visibility_.valid()) {
+    abc_visibility_ = Alembic::AbcGeom::CreateVisibilityProperty(abc_object, timesample_index_);
+  }
+  abc_visibility_.set(is_visible ? Alembic::AbcGeom::kVisibilityVisible :
+                                   Alembic::AbcGeom::kVisibilityHidden);
+}
+
 }  // namespace alembic
 }  // namespace io
 }  // namespace blender

source/blender/io/alembic/exporter/abc_writer_abstract.h

@@ -43,6 +43,9 @@ class ABCAbstractWriter : public AbstractHierarchyWriter {
   uint32_t timesample_index_;
   Imath::Box3d bounding_box_;
 
+  /* Visibility of this writer's data in Alembic. */
+  Alembic::Abc::OCharProperty abc_visibility_;
+
  public:
   explicit ABCAbstractWriter(const ABCWriterConstructorArgs &args);
   virtual ~ABCAbstractWriter();
@@ -70,6 +73,8 @@ class ABCAbstractWriter : public AbstractHierarchyWriter {
   virtual void do_write(HierarchyContext &context) = 0;
 
   virtual void update_bounding_box(Object *object);
+
+  void write_visibility(const HierarchyContext &context);
 };
 
 }  // namespace alembic

source/blender/io/alembic/exporter/abc_writer_mesh.cc

@@ -159,7 +159,10 @@ ModifierData *ABCGenericMeshWriter::get_liquid_sim_modifier(Scene *scene, Object
 
 bool ABCGenericMeshWriter::is_supported(const HierarchyContext *context) const
 {
-  return context->is_object_visible(DAG_EVAL_RENDER);
+  if (args_.export_params->visible_objects_only) {
+    return context->is_object_visible(DAG_EVAL_RENDER);
+  }
+  return true;
 }
 
 void ABCGenericMeshWriter::do_write(HierarchyContext &context)

source/blender/io/alembic/exporter/abc_writer_transform.cc

@@ -92,6 +92,8 @@ void ABCTransformWriter::do_write(HierarchyContext &context)
   xform_sample.setMatrix(convert_matrix_datatype(parent_relative_matrix));
   xform_sample.setInheritsXforms(true);
   abc_xform_schema_.set(xform_sample);
+
+  write_visibility(context);
 }
 
 const OObject ABCTransformWriter::get_alembic_object() const

source/blender/io/common/intern/abstract_hierarchy_iterator.cc

@@ -80,7 +80,7 @@ void HierarchyContext::mark_as_not_instanced()
 
 bool HierarchyContext::is_object_visible(const enum eEvaluationMode evaluation_mode) const
 {
-  bool is_dupli = duplicator != nullptr;
+  const bool is_dupli = duplicator != nullptr;
   int base_flag;
 
   if (is_dupli) {
@@ -92,7 +92,7 @@ bool HierarchyContext::is_object_visible(const enum eEvaluationMode evaluation_m
     object->base_flag = duplicator->base_flag | BASE_FROM_DUPLI;
   }
 
-  int visibility = BKE_object_visibility(object, evaluation_mode);
+  const int visibility = BKE_object_visibility(object, evaluation_mode);
 
   if (is_dupli) {
     object->base_flag = base_flag;

tests/python/alembic_export_tests.py

@@ -32,6 +32,7 @@ import pathlib
 import subprocess
 import sys
 import unittest
+from typing import Tuple
 
 from modules.test_utils import (
     with_tempdir,
@@ -59,19 +60,13 @@ class AbstractAlembicTest(AbstractBlenderRunnerTest):
         # 'abcls' array notation, like "name[16]"
         cls.abcls_array = re.compile(r'^(?P<name>[^\[]+)(\[(?P<arraysize>\d+)\])?$')
 
-    def abcprop(self, filepath: pathlib.Path, proppath: str) -> dict:
-        """Uses abcls to obtain compound property values from an Alembic object.
+    def abcls(self, *arguments) -> Tuple[int, str]:
+        """Uses abcls and return its output.
 
-        A dict of subproperties is returned, where the values are Python values.
-
-        The Python bindings for Alembic are old, and only compatible with Python 2.x,
-        so that's why we can't use them here, and have to rely on other tooling.
+        :return: tuple (process exit status code, stdout)
         """
-        import collections
 
-        abcls = self.alembic_root / 'bin' / 'abcls'
-
-        command = (str(abcls), '-vl', '%s%s' % (filepath, proppath))
+        command = (self.alembic_root / 'bin' / 'abcls', *arguments)
         proc = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                               timeout=30)
 
@@ -84,7 +79,25 @@ class AbstractAlembicTest(AbstractBlenderRunnerTest):
         output = output.replace('\r\n', '\n').replace('\r', '\n')
 
         if proc.returncode:
-            raise AbcPropError('Error %d running %s:\n%s' % (proc.returncode, ' '.join(command), output))
+            str_command = " ".join(str(c) for c in command)
+            print(f'command {str_command} failed with status {proc.returncode}')
+
+        return (proc.returncode, output)
+
+    def abcprop(self, filepath: pathlib.Path, proppath: str) -> dict:
+        """Uses abcls to obtain compound property values from an Alembic object.
+
+        A dict of subproperties is returned, where the values are Python values.
+
+        The Python bindings for Alembic are old, and only compatible with Python 2.x,
+        so that's why we can't use them here, and have to rely on other tooling.
+        """
+        import collections
+
+        command = ('-vl', '%s%s' % (filepath, proppath))
+        returncode, output = self.abcls(*command)
+        if returncode:
+            raise AbcPropError('Error %d running abcls:\n%s' % (returncode, output))
 
         # Mapping from value type to callable that can convert a string to Python values.
         converters = {
@@ -536,6 +549,41 @@ class LongNamesExportTest(AbstractAlembicTest):
         self.assertIn('.faceCounts', abcprop)
 
 
+class InvisibleObjectExportTest(AbstractAlembicTest):
+    """Export an object which is invisible.
+
+    This test only tests a small subset of the functionality that is required to
+    export invisible objects. It just tests that the visibility property is
+    written, and that it has the correct initial value. This is a limitation
+    caused by these tests relying on `abcls`.
+    """
+
+    @with_tempdir
+    def test_hierarchical_export(self, tempdir: pathlib.Path):
+        abc = tempdir / 'visibility.abc'
+        script = "import bpy; bpy.ops.wm.alembic_export(filepath='%s', start=1, end=2, " \
+                 "renderable_only=False, visible_objects_only=False)" % abc.as_posix()
+        self.run_blender('visibility.blend', script)
+
+        def test(cube_name: str, expect_visible: bool):
+            returncode, output = self.abcls('-va', f'{abc}/{cube_name}')
+            if returncode:
+                self.fail(f"abcls failed: {output}")
+            output = output.strip()
+            self.assertEqual(f'Cube   .xform   visible   {int(expect_visible)}', output)
+
+        # This cube is always visible.
+        test('VisibleCube', True)
+
+        # This cube is never visible, and thus will not be pulled into the
+        # depsgraph by the standard builder, only by the all-objects builder.
+        test('InvisibleCube', False)
+
+        # This cube has animated visibility, and thus will be pulled into the
+        # depsgraph by the standard builder as well as the all-objects builder.
+        test('InvisibleAnimatedCube', False)
+
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
     parser.add_argument('--blender', required=True)