Sculpt: Implement face set brush for dyntopo

The implementation follows the logic of the face set brush for the
faces and grids type of PBVH, with the similar weak part of iterating
over entire mesh at the start of a stroke.

The difference in the behavior is that face needs to be fully covered
by the brush in order to have face set assigned to it (while for the
other types of the PBVH face gets assigned its face set if any of its
vertices are covered). The main reason for this is that this seems to
avoid boundaries being too wiggly.

The auto-masking is not fully integrated into this brush yet. Doing so
is possible, but seems to require deeper re-considerations of the way
how automasking accesses original data and how it is fetched from an
undo node. It worth noting that auto-masking is something that needs
to be looked into for all brushes to make it supported in the dynamic
topology mode.

Pull Request: https://projects.blender.org/blender/blender/pulls/113357

source/blender/editors/sculpt_paint/sculpt.cc

@@ -794,8 +794,24 @@ int SCULPT_face_set_next_available_get(SculptSession *ss)
       next_face_set++;
       return next_face_set;
     }
-    case PBVH_BMESH:
-      return 0;
+    case PBVH_BMESH: {
+      const int cd_offset = CustomData_get_offset_named(
+          &ss->bm->pdata, CD_PROP_INT32, ".sculpt_face_set");
+      if (cd_offset == -1) {
+        return 0;
+      }
+
+      int next_face_set = 0;
+      BMIter iter;
+      BMFace *f;
+      BM_ITER_MESH (f, &iter, ss->bm, BM_FACES_OF_MESH) {
+        const int fset = *static_cast<const int *>(POINTER_OFFSET(f->head.data, cd_offset));
+        next_face_set = blender::math::max(next_face_set, fset);
+      }
+
+      next_face_set++;
+      return next_face_set;
+    }
   }
   return 0;
 }

source/blender/editors/sculpt_paint/sculpt_face_set.cc

@@ -113,62 +113,47 @@ int ED_sculpt_face_sets_active_update_and_get(bContext *C, Object *ob, const flo
 
 /* Draw Face Sets Brush. */
 
-static void do_draw_face_sets_brush_task(Object *ob, const Brush *brush, PBVHNode *node)
+constexpr float FACE_SET_BRUSH_MIN_FADE = 0.05f;
+
+static void do_draw_face_sets_brush_faces(Object *ob, const Brush *brush, PBVHNode *node)
 {
   using namespace blender;
-  SculptSession *ss = ob->sculpt;
-  const float bstrength = ss->cache->bstrength;
 
-  PBVHVertexIter vd;
+  SculptSession *ss = ob->sculpt;
+
+  BLI_assert(BKE_pbvh_type(ss->pbvh) == PBVH_FACES);
+
+  const float bstrength = ss->cache->bstrength;
+  const int thread_id = BLI_task_parallel_thread_id(nullptr);
 
   SculptBrushTest test;
   SculptBrushTestFn sculpt_brush_test_sq_fn = SCULPT_brush_test_init_with_falloff_shape(
       ss, &test, brush->falloff_shape);
-  const int thread_id = BLI_task_parallel_thread_id(nullptr);
 
   const Span<float3> positions(
       reinterpret_cast<const float3 *>(SCULPT_mesh_deformed_positions_get(ss)),
       SCULPT_vertex_count_get(ss));
+
   AutomaskingNodeData automask_data;
   SCULPT_automasking_node_begin(ob, ss, ss->cache->automasking, &automask_data, node);
 
   bool changed = false;
+
+  PBVHVertexIter vd;
   BKE_pbvh_vertex_iter_begin (ss->pbvh, node, vd, PBVH_ITER_UNIQUE) {
     SCULPT_automasking_node_update(ss, &automask_data, &vd);
 
-    if (BKE_pbvh_type(ss->pbvh) == PBVH_FACES) {
-      for (const int face_i : ss->pmap[vd.index]) {
-        const blender::IndexRange face = ss->faces[face_i];
+    for (const int face_i : ss->pmap[vd.index]) {
+      const blender::IndexRange face = ss->faces[face_i];
 
-        const float3 poly_center = bke::mesh::face_center_calc(positions,
-                                                               ss->corner_verts.slice(face));
+      const float3 poly_center = bke::mesh::face_center_calc(positions,
+                                                             ss->corner_verts.slice(face));
 
-        if (!sculpt_brush_test_sq_fn(&test, poly_center)) {
-          continue;
-        }
-        const bool face_hidden = ss->hide_poly && ss->hide_poly[face_i];
-        if (face_hidden) {
-          continue;
-        }
-        const float fade = bstrength * SCULPT_brush_strength_factor(ss,
-                                                                    brush,
-                                                                    vd.co,
-                                                                    sqrtf(test.dist),
-                                                                    vd.no,
-                                                                    vd.fno,
-                                                                    vd.mask,
-                                                                    vd.vertex,
-                                                                    thread_id,
-                                                                    &automask_data);
-
-        if (fade > 0.05f) {
-          ss->face_sets[face_i] = ss->cache->paint_face_set;
-          changed = true;
-        }
+      if (!sculpt_brush_test_sq_fn(&test, poly_center)) {
+        continue;
       }
-    }
-    else if (BKE_pbvh_type(ss->pbvh) == PBVH_GRIDS) {
-      if (!sculpt_brush_test_sq_fn(&test, vd.co)) {
+      const bool face_hidden = ss->hide_poly && ss->hide_poly[face_i];
+      if (face_hidden) {
         continue;
       }
       const float fade = bstrength * SCULPT_brush_strength_factor(ss,
@@ -182,8 +167,8 @@ static void do_draw_face_sets_brush_task(Object *ob, const Brush *brush, PBVHNod
                                                                   thread_id,
                                                                   &automask_data);
 
-      if (fade > 0.05f) {
-        SCULPT_vertex_face_set_set(ss, vd.vertex, ss->cache->paint_face_set);
+      if (fade > FACE_SET_BRUSH_MIN_FADE) {
+        ss->face_sets[face_i] = ss->cache->paint_face_set;
         changed = true;
       }
     }
@@ -195,6 +180,167 @@ static void do_draw_face_sets_brush_task(Object *ob, const Brush *brush, PBVHNod
   }
 }
 
+static void do_draw_face_sets_brush_grids(Object *ob, const Brush *brush, PBVHNode *node)
+{
+  using namespace blender;
+
+  SculptSession *ss = ob->sculpt;
+
+  BLI_assert(BKE_pbvh_type(ss->pbvh) == PBVH_GRIDS);
+
+  const float bstrength = ss->cache->bstrength;
+  const int thread_id = BLI_task_parallel_thread_id(nullptr);
+
+  SculptBrushTest test;
+  SculptBrushTestFn sculpt_brush_test_sq_fn = SCULPT_brush_test_init_with_falloff_shape(
+      ss, &test, brush->falloff_shape);
+
+  AutomaskingNodeData automask_data;
+  SCULPT_automasking_node_begin(ob, ss, ss->cache->automasking, &automask_data, node);
+
+  bool changed = false;
+
+  PBVHVertexIter vd;
+  BKE_pbvh_vertex_iter_begin (ss->pbvh, node, vd, PBVH_ITER_UNIQUE) {
+    SCULPT_automasking_node_update(ss, &automask_data, &vd);
+
+    if (!sculpt_brush_test_sq_fn(&test, vd.co)) {
+      continue;
+    }
+    const float fade = bstrength * SCULPT_brush_strength_factor(ss,
+                                                                brush,
+                                                                vd.co,
+                                                                sqrtf(test.dist),
+                                                                vd.no,
+                                                                vd.fno,
+                                                                vd.mask,
+                                                                vd.vertex,
+                                                                thread_id,
+                                                                &automask_data);
+
+    if (fade > FACE_SET_BRUSH_MIN_FADE) {
+      SCULPT_vertex_face_set_set(ss, vd.vertex, ss->cache->paint_face_set);
+      changed = true;
+    }
+  }
+  BKE_pbvh_vertex_iter_end;
+
+  if (changed) {
+    SCULPT_undo_push_node(ob, node, SCULPT_UNDO_FACE_SETS);
+  }
+}
+
+static void do_draw_face_sets_brush_bmesh(Object *ob, const Brush *brush, PBVHNode *node)
+{
+  using namespace blender;
+
+  SculptSession *ss = ob->sculpt;
+
+  BLI_assert(BKE_pbvh_type(ss->pbvh) == PBVH_BMESH);
+
+  const float bstrength = ss->cache->bstrength;
+  const int thread_id = BLI_task_parallel_thread_id(nullptr);
+
+  SculptBrushTest test;
+  SculptBrushTestFn sculpt_brush_test_sq_fn = SCULPT_brush_test_init_with_falloff_shape(
+      ss, &test, brush->falloff_shape);
+
+  /* Disable auto-masking code path which rely on an undo step to access original data.
+   *
+   * This is because the dynamic topology uses BMesh Log based undo system, which creates a single
+   * node for the undo step, and its type could be different for the needs of the brush undo and
+   * the original data access.
+   *
+   * For the brushes like Draw the ss->cache->automasking is set to nullptr at the first step of
+   * the brush, as there is an explicit check there for the brushes which support dynamic topology.
+   * Do it locally here for the Draw Face Set brush here, to mimic the behavior of the other
+   * brushes but without marking the brush as supporting dynamic topology. */
+  AutomaskingNodeData automask_data;
+  SCULPT_automasking_node_begin(ob, ss, nullptr, &automask_data, node);
+
+  bool changed = false;
+
+  const int cd_offset = CustomData_get_offset_named(
+      &ss->bm->pdata, CD_PROP_INT32, ".sculpt_face_set");
+
+  for (BMFace *f : BKE_pbvh_bmesh_node_faces(node)) {
+    if (BM_elem_flag_test(f, BM_ELEM_HIDDEN)) {
+      continue;
+    }
+
+    bool is_face_fully_covered = true;
+
+    float3 face_center;
+    BM_face_calc_center_median(f, face_center);
+
+    const BMLoop *l_iter = f->l_first = BM_FACE_FIRST_LOOP(f);
+    do {
+      if (!sculpt_brush_test_sq_fn(&test, l_iter->v->co)) {
+        is_face_fully_covered = false;
+        break;
+      }
+
+      BMVert *vert = l_iter->v;
+
+      /* There is no need to update the automasking data as it is disabled above. Additionally,
+       * there is no access to the PBVHVertexIter as iteration happens over faces.
+       *
+       * The full auto-masking support would be very good to be implemented here, so keeping the
+       * typical code flow for it here for the reference, and ease of looking at what needs to be
+       * done for such integration.
+       *
+       * SCULPT_automasking_node_update(ss, &automask_data, &vd); */
+
+      const float fade = bstrength *
+                         SCULPT_brush_strength_factor(ss,
+                                                      brush,
+                                                      face_center,
+                                                      sqrtf(test.dist),
+                                                      f->no,
+                                                      f->no,
+                                                      0.0f,
+                                                      BKE_pbvh_make_vref(intptr_t(vert)),
+                                                      thread_id,
+                                                      &automask_data);
+
+      if (fade <= FACE_SET_BRUSH_MIN_FADE) {
+        is_face_fully_covered = false;
+        break;
+      }
+
+    } while ((l_iter = l_iter->next) != f->l_first);
+
+    if (is_face_fully_covered) {
+      int &fset = *static_cast<int *>(POINTER_OFFSET(f->head.data, cd_offset));
+      fset = ss->cache->paint_face_set;
+      changed = true;
+    }
+  }
+
+  if (changed) {
+    SCULPT_undo_push_node(ob, node, SCULPT_UNDO_FACE_SETS);
+  }
+}
+
+static void do_draw_face_sets_brush_task(Object *ob, const Brush *brush, PBVHNode *node)
+{
+  const SculptSession *ss = ob->sculpt;
+
+  switch (BKE_pbvh_type(ss->pbvh)) {
+    case PBVH_FACES:
+      do_draw_face_sets_brush_faces(ob, brush, node);
+      break;
+
+    case PBVH_GRIDS:
+      do_draw_face_sets_brush_grids(ob, brush, node);
+      break;
+
+    case PBVH_BMESH:
+      do_draw_face_sets_brush_bmesh(ob, brush, node);
+      break;
+  }
+}
+
 static void do_relax_face_sets_brush_task(Object *ob,
                                           const Brush *brush,
                                           const int iteration,