Sculpt: Fix #104631: Tip Roundness on Paint brush causes jitering

I cleaned up the cube brush tip code quite a bit; more remains
to be done.  There is a new function to initialize cube
tip matrices, SCULPT_cube_tip_init.  It's currently only
used by the paint brush, I'll need to do a bit of testing
before using it for clay strips and multiplane scrape.

Note: SCULPT_cube_tip_init uses the brush local matrix code
to avoid code duplication (and to take advantage of the debouncing
that is done there).

source/blender/blenkernel/BKE_brush.h

@@ -11,6 +11,8 @@
 #include "DNA_color_types.h"
 #include "DNA_object_enums.h"
 
+#include "BKE_paint.h" /* for ePaintMode */
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -185,6 +187,11 @@ void BKE_brush_scale_size(int *r_brush_size,
                           float new_unprojected_radius,
                           float old_unprojected_radius);
 
+/* Returns true if a brush requires a cube
+ * (often presented to the user as a square) tip inside a specific paint mode.
+ */
+bool BKE_brush_has_cube_tip(const struct Brush *brush, ePaintMode paint_mode);
+
 /* Accessors */
 #define BKE_brush_tool_get(brush, p) \
   (CHECK_TYPE_ANY(brush, struct Brush *, const struct Brush *), \

source/blender/blenkernel/BKE_paint.h

@@ -238,7 +238,8 @@ void BKE_paint_face_set_overlay_color_get(int face_set, int seed, uchar r_color[
 
 bool paint_calculate_rake_rotation(struct UnifiedPaintSettings *ups,
                                    struct Brush *brush,
-                                   const float mouse_pos[2]);
+                                   const float mouse_pos[2],
+                                   ePaintMode paint_mode);
 void paint_update_brush_rake_rotation(struct UnifiedPaintSettings *ups,
                                       struct Brush *brush,
                                       float rotation);

source/blender/blenkernel/intern/brush.cc

@@ -2576,3 +2576,22 @@ struct ImBuf *BKE_brush_gen_radial_control_imbuf(Brush *br, bool secondary, bool
 
   return im;
 }
+
+bool BKE_brush_has_cube_tip(const Brush *brush, ePaintMode paint_mode)
+{
+  switch (paint_mode) {
+    case PAINT_MODE_SCULPT:
+      if (brush->sculpt_tool == SCULPT_TOOL_MULTIPLANE_SCRAPE) {
+        return true;
+      }
+
+      if (ELEM(brush->sculpt_tool, SCULPT_TOOL_CLAY_STRIPS, SCULPT_TOOL_PAINT) &&
+          brush->tip_roundness < 1.0f) {
+        return true;
+      }
+
+      break;
+  }
+
+  return false;
+}

source/blender/blenkernel/intern/paint.cc

@@ -1302,12 +1302,7 @@ float paint_grid_paint_mask(const GridPaintMask *gpm, uint level, uint x, uint y
 
 void paint_update_brush_rake_rotation(UnifiedPaintSettings *ups, Brush *brush, float rotation)
 {
-  if (brush->mtex.brush_angle_mode & MTEX_ANGLE_RAKE) {
-    ups->brush_rotation = rotation;
-  }
-  else {
-    ups->brush_rotation = 0.0f;
-  }
+  ups->brush_rotation = rotation;
 
   if (brush->mask_mtex.brush_angle_mode & MTEX_ANGLE_RAKE) {
     ups->brush_rotation_sec = rotation;
@@ -1322,17 +1317,19 @@ static bool paint_rake_rotation_active(const MTex &mtex)
   return mtex.tex && mtex.brush_angle_mode & MTEX_ANGLE_RAKE;
 }
 
-static bool paint_rake_rotation_active(const Brush &brush)
+const bool paint_rake_rotation_active(const Brush &brush, ePaintMode paint_mode)
 {
-  return paint_rake_rotation_active(brush.mtex) || paint_rake_rotation_active(brush.mask_mtex);
+  return paint_rake_rotation_active(brush.mtex) || paint_rake_rotation_active(brush.mask_mtex) ||
+         BKE_brush_has_cube_tip(&brush, paint_mode);
 }
 
 bool paint_calculate_rake_rotation(UnifiedPaintSettings *ups,
                                    Brush *brush,
-                                   const float mouse_pos[2])
+                                   const float mouse_pos[2],
+                                   ePaintMode paint_mode)
 {
   bool ok = false;
-  if (paint_rake_rotation_active(*brush)) {
+  if (paint_rake_rotation_active(*brush, paint_mode)) {
     const float r = RAKE_THRESHHOLD;
     float rotation;
 

source/blender/editors/sculpt_paint/paint_cursor.cc

@@ -1860,7 +1860,8 @@ static void paint_cursor_update_rake_rotation(PaintCursorContext *pcontext)
    * and we may get interference with the stroke itself.
    * For line strokes, such interference is visible. */
   if (!pcontext->ups->stroke_active) {
-    paint_calculate_rake_rotation(pcontext->ups, pcontext->brush, pcontext->translation);
+    paint_calculate_rake_rotation(
+        pcontext->ups, pcontext->brush, pcontext->translation, pcontext->mode);
   }
 }
 

source/blender/editors/sculpt_paint/paint_stroke.cc

@@ -440,7 +440,7 @@ static bool paint_brush_update(bContext *C,
     }
     /* curve strokes do their own rake calculation */
     else if (!(brush->flag & BRUSH_CURVE)) {
-      if (!paint_calculate_rake_rotation(ups, brush, mouse_init)) {
+      if (!paint_calculate_rake_rotation(ups, brush, mouse_init, mode)) {
         /* Not enough motion to define an angle. */
         if (!stroke->rake_started) {
           is_dry_run = true;
@@ -1562,7 +1562,7 @@ int paint_stroke_modal(bContext *C, wmOperator *op, const wmEvent *event, PaintS
           (br->mask_mtex.brush_angle_mode & MTEX_ANGLE_RAKE)) {
         copy_v2_v2(stroke->ups->last_rake, stroke->last_mouse_position);
       }
-      paint_calculate_rake_rotation(stroke->ups, br, mouse);
+      paint_calculate_rake_rotation(stroke->ups, br, mouse, mode);
     }
   }
   else if (first_modal ||

source/blender/editors/sculpt_paint/sculpt.cc

@@ -1307,12 +1307,6 @@ void SCULPT_floodfill_free(SculptFloodFill *flood)
 
 /** \} */
 
-static bool sculpt_tool_has_cube_tip(const char sculpt_tool)
-{
-  return ELEM(
-      sculpt_tool, SCULPT_TOOL_CLAY_STRIPS, SCULPT_TOOL_PAINT, SCULPT_TOOL_MULTIPLANE_SCRAPE);
-}
-
 /* -------------------------------------------------------------------- */
 /** \name Tool Capabilities
  *
@@ -1376,7 +1370,8 @@ static int sculpt_brush_needs_normal(const SculptSession *ss, Sculpt *sd, const
                SCULPT_TOOL_THUMB) ||
 
           (mask_tex->brush_map_mode == MTEX_MAP_MODE_AREA)) ||
-         sculpt_brush_use_topology_rake(ss, brush);
+         sculpt_brush_use_topology_rake(ss, brush) ||
+         BKE_brush_has_cube_tip(brush, PAINT_MODE_SCULPT);
 }
 
 static bool sculpt_brush_needs_rake_rotation(const Brush *brush)
@@ -3018,7 +3013,7 @@ static void calc_local_y(ViewContext *vc, const float center[3], float y[3])
   mul_m4_v3(ob->world_to_object, y);
 }
 
-static void calc_brush_local_mat(const MTex *mtex,
+static void calc_brush_local_mat(const float rotation,
                                  Object *ob,
                                  float local_mat[4][4],
                                  float local_mat_inv[4][4])
@@ -3045,7 +3040,7 @@ static void calc_brush_local_mat(const MTex *mtex,
   /* Calculate the X axis of the local matrix. */
   cross_v3_v3v3(v, up, cache->sculpt_normal);
   /* Apply rotation (user angle, rake, etc.) to X axis. */
-  angle = mtex->rot - cache->special_rotation;
+  angle = rotation - cache->special_rotation;
   rotate_v3_v3v3fl(mat[0], v, cache->sculpt_normal, angle);
 
   /* Get other axes. */
@@ -3059,7 +3054,7 @@ static void calc_brush_local_mat(const MTex *mtex,
   float radius = cache->radius;
 
   /* Square tips should scale by square root of 2. */
-  if (sculpt_tool_has_cube_tip(cache->brush->sculpt_tool)) {
+  if (BKE_brush_has_cube_tip(cache->brush, PAINT_MODE_SCULPT)) {
     radius += (radius * M_SQRT2 - radius) * (1.0f - cache->brush->tip_roundness);
   }
 
@@ -3103,7 +3098,7 @@ static void update_brush_local_mat(Sculpt *sd, Object *ob)
   if (cache->mirror_symmetry_pass == 0 && cache->radial_symmetry_pass == 0) {
     const Brush *brush = BKE_paint_brush(&sd->paint);
     const MTex *mask_tex = BKE_brush_mask_texture_get(brush, OB_MODE_SCULPT);
-    calc_brush_local_mat(mask_tex, ob, cache->brush_local_mat, cache->brush_local_mat_inv);
+    calc_brush_local_mat(mask_tex->rot, ob, cache->brush_local_mat, cache->brush_local_mat_inv);
   }
 }
 
@@ -3614,7 +3609,7 @@ static void do_brush_action(Sculpt *sd,
     float radius_scale = 1.0f;
 
     /* Corners of square brushes can go outside the brush radius. */
-    if (sculpt_tool_has_cube_tip(brush->sculpt_tool)) {
+    if (BKE_brush_has_cube_tip(brush, PAINT_MODE_SCULPT)) {
       radius_scale = M_SQRT2;
     }
 
@@ -3686,10 +3681,7 @@ static void do_brush_action(Sculpt *sd,
     update_sculpt_normal(sd, ob, nodes, totnode);
   }
 
-  const MTex *mask_tex = BKE_brush_mask_texture_get(brush, static_cast<eObjectMode>(ob->mode));
-  if (mask_tex->brush_map_mode == MTEX_MAP_MODE_AREA) {
-    update_brush_local_mat(sd, ob);
-  }
+  update_brush_local_mat(sd, ob);
 
   if (brush->sculpt_tool == SCULPT_TOOL_POSE && SCULPT_stroke_is_first_brush_step(ss->cache)) {
     SCULPT_pose_brush_init(sd, ob, ss, brush);
@@ -6456,4 +6448,27 @@ void SCULPT_topology_islands_ensure(Object *ob)
 
   ss->islands_valid = true;
 }
+
+void SCULPT_cube_tip_init(Sculpt * /*sd*/, Object *ob, Brush *brush, float mat[4][4])
+{
+  SculptSession *ss = ob->sculpt;
+  float scale[4][4];
+  float tmat[4][4];
+  float unused[4][4];
+
+  zero_m4(mat);
+  calc_brush_local_mat(0.0, ob, unused, mat);
+
+  /* Note: we ignore the radius scaling done inside of calc_brush_local_mat to
+   * duplicate prior behavior.
+   *
+   * TODO: try disabling this and check that all edge cases work properly.
+   */
+  normalize_m4(mat);
+
+  scale_m4_fl(scale, ss->cache->radius);
+  mul_m4_m4m4(tmat, mat, scale);
+  mul_v3_fl(tmat[1], brush->tip_scale_x);
+  invert_m4_m4(mat, tmat);
+}
 /** \} */

source/blender/editors/sculpt_paint/sculpt_intern.hh

@@ -227,6 +227,7 @@ struct SculptUndoNode {
 struct SculptRakeData {
   float follow_dist;
   float follow_co[3];
+  float angle;
 };
 
 /**
@@ -1247,6 +1248,7 @@ SculptBrushTestFn SCULPT_brush_test_init_with_falloff_shape(SculptSession *ss,
                                                             char falloff_shape);
 const float *SCULPT_brush_frontface_normal_from_falloff_shape(SculptSession *ss,
                                                               char falloff_shape);
+void SCULPT_cube_tip_init(Sculpt *sd, Object *ob, Brush *brush, float mat[4][4]);
 
 /**
  * Return a multiplier for brush strength on a particular vertex.

source/blender/editors/sculpt_paint/sculpt_paint_color.cc

@@ -280,28 +280,12 @@ void SCULPT_do_paint_brush(PaintModeSettings *paint_mode_settings,
 
   float area_no[3];
   float mat[4][4];
-  float scale[4][4];
-  float tmat[4][4];
 
   /* If the brush is round the tip does not need to be aligned to the surface, so this saves a
    * whole iteration over the affected nodes. */
   if (brush->tip_roundness < 1.0f) {
-    SCULPT_calc_area_normal(sd, ob, nodes, totnode, area_no);
+    SCULPT_cube_tip_init(sd, ob, brush, mat);
 
-    cross_v3_v3v3(mat[0], area_no, ss->cache->grab_delta_symmetry);
-    mat[0][3] = 0;
-    cross_v3_v3v3(mat[1], area_no, mat[0]);
-    mat[1][3] = 0;
-    copy_v3_v3(mat[2], area_no);
-    mat[2][3] = 0;
-    copy_v3_v3(mat[3], ss->cache->location);
-    mat[3][3] = 1;
-    normalize_m4(mat);
-
-    scale_m4_fl(scale, ss->cache->radius);
-    mul_m4_m4m4(tmat, mat, scale);
-    mul_v3_fl(tmat[1], brush->tip_scale_x);
-    invert_m4_m4(mat, tmat);
     if (is_zero_m4(mat)) {
       return;
     }

source/creator/CMakeLists.txt

@@ -15,6 +15,7 @@ set(INC
   ../blender/makesrna
   ../blender/render
   ../blender/windowmanager
+  ../blender/bmesh
 )
 
 set(LIB