Geometry Nodes: Support top-level instance meshes in boolean node

During the 2.93 to 3.0 transition, instance handling was made more
explicit in general. However, we forgot to change the boolean node,
which still implicitly gathered all the instanced meshes and fed them
to the boolean algorithm separately. We waited for the next breaking
release, 4.0 to "correct" this, and did it in fc06a471f1.
However, in that commit it was assumed that the "Self Intersection"
mode would be able to address the use case. The idea was also to push
some complexity outside of the boolean code, which is already one of
the more complex areas in Blender. Though it's possible to have a
"Group ID" or "Shape ID" input in the future as well, it's also
reasonable to expect some instances to be processed by the node,
even though it isn't quite consistent.

This commit makes a compromise by processing meshes contained by
top-level instances. We do it at this stage of the release to avoid the
breaking change.

Pull Request: https://projects.blender.org/blender/blender/pulls/114632

source/blender/blenloader/intern/versioning_400.cc

@@ -455,15 +455,6 @@ static void version_movieclips_legacy_camera_object(Main *bmain)
   }
 }
 
-static void version_geometry_nodes_add_realize_instance_nodes(bNodeTree *ntree)
-{
-  LISTBASE_FOREACH_MUTABLE (bNode *, node, &ntree->nodes) {
-    if (STREQ(node->idname, "GeometryNodeMeshBoolean")) {
-      add_realize_instances_before_socket(ntree, node, nodeFindSocket(node, SOCK_IN, "Mesh 2"));
-    }
-  }
-}
-
 /* Version VertexWeightEdit modifier to make existing weights exclusive of the threshold. */
 static void version_vertex_weight_edit_preserve_threshold_exclusivity(Main *bmain)
 {
@@ -1300,14 +1291,6 @@ void blo_do_versions_400(FileData *fd, Library * /*lib*/, Main *bmain)
     }
   }
 
-  if (!MAIN_VERSION_FILE_ATLEAST(bmain, 400, 3)) {
-    LISTBASE_FOREACH (bNodeTree *, ntree, &bmain->nodetrees) {
-      if (ntree->type == NTREE_GEOMETRY) {
-        version_geometry_nodes_add_realize_instance_nodes(ntree);
-      }
-    }
-  }
-
   /* 400 4 did not require any do_version here. */
 
   if (!MAIN_VERSION_FILE_ATLEAST(bmain, 400, 5)) {

source/blender/nodes/geometry/nodes/node_geo_boolean.cc

@@ -3,6 +3,7 @@
  * SPDX-License-Identifier: GPL-2.0-or-later */
 
 #include "BKE_geometry_set_instances.hh"
+#include "BKE_instances.hh"
 #include "BKE_mesh_boolean_convert.hh"
 
 #include "DNA_mesh_types.h"
@@ -65,6 +66,16 @@ static void node_init(bNodeTree * /*tree*/, bNode *node)
   node->custom1 = GEO_NODE_BOOLEAN_DIFFERENCE;
 }
 
+static Array<short> calc_mesh_material_map(const Mesh &mesh, VectorSet<Material *> &all_materials)
+{
+  Array<short> map(mesh.totcol);
+  for (const int i : IndexRange(mesh.totcol)) {
+    Material *material = mesh.mat[i];
+    map[i] = material ? all_materials.index_of_or_add(material) : -1;
+  }
+  return map;
+}
+
 static void node_geo_exec(GeoNodeExecParams params)
 {
 #ifdef WITH_GMP
@@ -73,6 +84,7 @@ static void node_geo_exec(GeoNodeExecParams params)
   const bool hole_tolerant = params.get_input<bool>("Hole Tolerant");
 
   Vector<const Mesh *> meshes;
+  Vector<float4x4> transforms;
   VectorSet<Material *> materials;
   Vector<Array<short>> material_remaps;
 
@@ -84,6 +96,7 @@ static void node_geo_exec(GeoNodeExecParams params)
      * to be a single mesh. */
     if (const Mesh *mesh_in_a = set_a.get_mesh()) {
       meshes.append(mesh_in_a);
+      transforms.append(float4x4::identity());
       if (mesh_in_a->totcol == 0) {
         /* Necessary for faces using the default material when there are no material slots. */
         materials.add(nullptr);
@@ -100,12 +113,39 @@ static void node_geo_exec(GeoNodeExecParams params)
   for (const GeometrySet &geometry : geometry_sets) {
     if (const Mesh *mesh = geometry.get_mesh()) {
       meshes.append(mesh);
-      Array<short> map(mesh->totcol);
-      for (const int i : IndexRange(mesh->totcol)) {
-        Material *material = mesh->mat[i];
-        map[i] = material ? materials.index_of_or_add(material) : -1;
+      transforms.append(float4x4::identity());
+      material_remaps.append(calc_mesh_material_map(*mesh, materials));
+    }
+    if (const bke::Instances *instances = geometry.get_instances()) {
+      const Span<bke::InstanceReference> references = instances->references();
+      const Span<int> handles = instances->reference_handles();
+      const Span<float4x4> instance_transforms = instances->transforms();
+      for (const int i : handles.index_range()) {
+        const bke::InstanceReference &reference = references[handles[i]];
+        switch (reference.type()) {
+          case bke::InstanceReference::Type::Object: {
+            const GeometrySet object_geometry = bke::object_get_evaluated_geometry_set(
+                reference.object());
+            if (const Mesh *mesh = object_geometry.get_mesh()) {
+              meshes.append(mesh);
+              transforms.append(instance_transforms[i]);
+              material_remaps.append(calc_mesh_material_map(*mesh, materials));
+            }
+            break;
+          }
+          case bke::InstanceReference::Type::GeometrySet: {
+            if (const Mesh *mesh = reference.geometry_set().get_mesh()) {
+              meshes.append(mesh);
+              transforms.append(instance_transforms[i]);
+              material_remaps.append(calc_mesh_material_map(*mesh, materials));
+            }
+            break;
+          }
+          case bke::InstanceReference::Type::None:
+          case bke::InstanceReference::Type::Collection:
+            break;
+        }
       }
-      material_remaps.append(std::move(map));
     }
   }
 
@@ -116,7 +156,7 @@ static void node_geo_exec(GeoNodeExecParams params)
   Vector<int> intersecting_edges;
   Mesh *result = blender::meshintersect::direct_mesh_boolean(
       meshes,
-      {},
+      transforms,
       float4x4::identity(),
       material_remaps,
       use_self,