Fix #116418: Stroke direction wrong on curved surfaces

Even though the brush rotation is computed as a 2D angle (based on the mouse
movement), it currently gets applied by rotating the projected X direction
around the the normal in 3D.

This patch ensures that rotation gets applied first, and only then does the
motion direction get projected into the tangent plane. A potential issue with
the current approach is that the random perturbations will also be applied in
2D, but this seems to be fine from discussions with @JulienKaspar and @Sergey.

Also, there was an error where the location should probably be converted *to*
world coordinates.

All these changes seem to fix the issue described in #116418.

I also noticed some minor "inconsistencies" with how the rotation is applied:
For curve strokes, the direction of the curve corresponded to the upward
direction of the brush. For view plane, area plane mapping and anchored strokes,
the mouse motion indicated the downward direction of the brush.
For compatibility, I tried my best to enforce the latter conventions throughout,
but I'm not so confident about oversights.

Pull Request: https://projects.blender.org/blender/blender/pulls/116539

source/blender/blenkernel/intern/brush.cc

@@ -2116,7 +2116,7 @@ float BKE_brush_sample_tex_3d(const Scene *scene,
     if (mtex->brush_map_mode == MTEX_MAP_MODE_VIEW) {
       /* keep coordinates relative to mouse */
 
-      rotation += ups->brush_rotation;
+      rotation -= ups->brush_rotation;
 
       x = point_2d[0] - ups->tex_mouse[0];
       y = point_2d[1] - ups->tex_mouse[1];
@@ -2134,7 +2134,7 @@ float BKE_brush_sample_tex_3d(const Scene *scene,
       y = point_2d[1];
     }
     else if (mtex->brush_map_mode == MTEX_MAP_MODE_RANDOM) {
-      rotation += ups->brush_rotation;
+      rotation -= ups->brush_rotation;
       /* these contain a random coordinate */
       x = point_2d[0] - ups->tex_mouse[0];
       y = point_2d[1] - ups->tex_mouse[1];
@@ -2229,7 +2229,7 @@ float BKE_brush_sample_masktex(
     if (mtex->brush_map_mode == MTEX_MAP_MODE_VIEW) {
       /* keep coordinates relative to mouse */
 
-      rotation += ups->brush_rotation_sec;
+      rotation -= ups->brush_rotation_sec;
 
       x = point_2d[0] - ups->mask_tex_mouse[0];
       y = point_2d[1] - ups->mask_tex_mouse[1];
@@ -2247,7 +2247,7 @@ float BKE_brush_sample_masktex(
       y = point_2d[1];
     }
     else if (mtex->brush_map_mode == MTEX_MAP_MODE_RANDOM) {
-      rotation += ups->brush_rotation_sec;
+      rotation -= ups->brush_rotation_sec;
       /* these contain a random coordinate */
       x = point_2d[0] - ups->mask_tex_mouse[0];
       y = point_2d[1] - ups->mask_tex_mouse[1];

source/blender/blenkernel/intern/paint.cc

@@ -1352,11 +1352,11 @@ bool paint_calculate_rake_rotation(UnifiedPaintSettings *ups,
     }
 
     float dpos[2];
-    sub_v2_v2v2(dpos, ups->last_rake, mouse_pos);
+    sub_v2_v2v2(dpos, mouse_pos, ups->last_rake);
 
     /* Limit how often we update the angle to prevent jitter. */
     if (len_squared_v2(dpos) >= r * r) {
-      rotation = atan2f(dpos[0], dpos[1]);
+      rotation = atan2f(dpos[1], dpos[0]) + float(0.5f * M_PI);
 
       copy_v2_v2(ups->last_rake, mouse_pos);
 

source/blender/editors/sculpt_paint/paint_cursor.cc

@@ -584,7 +584,7 @@ static bool paint_draw_tex_overlay(UnifiedPaintSettings *ups,
 
       /* Brush rotation. */
       GPU_matrix_translate_2fv(center);
-      GPU_matrix_rotate_2d(-RAD2DEGF(primary ? ups->brush_rotation : ups->brush_rotation_sec));
+      GPU_matrix_rotate_2d(RAD2DEGF(primary ? ups->brush_rotation : ups->brush_rotation_sec));
       GPU_matrix_translate_2f(-center[0], -center[1]);
 
       /* Scale based on tablet pressure. */

source/blender/editors/sculpt_paint/paint_stroke.cc

@@ -400,7 +400,7 @@ static bool paint_brush_update(bContext *C,
 
     ups->anchored_size = ups->pixel_radius = sqrtf(dx * dx + dy * dy);
 
-    ups->brush_rotation = ups->brush_rotation_sec = atan2f(dx, dy) + float(M_PI);
+    ups->brush_rotation = ups->brush_rotation_sec = atan2f(dy, dx) + float(0.5f * M_PI);
 
     if (brush->flag & BRUSH_EDGE_TO_EDGE) {
       halfway[0] = dx * 0.5f + stroke->initial_mouse[0];
@@ -1373,7 +1373,7 @@ static bool paint_stroke_curve_end(bContext *C, wmOperator *op, PaintStroke *str
 
       for (j = 0; j < PAINT_CURVE_NUM_SEGMENTS; j++) {
         if (do_rake) {
-          float rotation = atan2f(tangents[2 * j], tangents[2 * j + 1]);
+          float rotation = atan2f(tangents[2 * j + 1], tangents[2 * j]) + float(0.5f * M_PI);
           paint_update_brush_rake_rotation(ups, br, rotation);
         }
 

source/blender/editors/sculpt_paint/sculpt.cc

@@ -2712,20 +2712,22 @@ static void update_sculpt_normal(Sculpt *sd, Object *ob, Span<PBVHNode *> nodes)
   }
 }
 
-static void calc_local_y(ViewContext *vc, const float center[3], float y[3])
+static void calc_local_from_screen(ViewContext *vc,
+                                   const float center[3],
+                                   const float screen_dir[2],
+                                   float r_local_dir[3])
 {
   Object *ob = vc->obact;
   float loc[3];
-  const float xy_delta[2] = {0.0f, 1.0f};
 
-  mul_v3_m4v3(loc, ob->world_to_object, center);
+  mul_v3_m4v3(loc, ob->object_to_world, center);
   const float zfac = ED_view3d_calc_zfac(vc->rv3d, loc);
 
-  ED_view3d_win_to_delta(vc->region, xy_delta, zfac, y);
-  normalize_v3(y);
+  ED_view3d_win_to_delta(vc->region, screen_dir, zfac, r_local_dir);
+  normalize_v3(r_local_dir);
 
-  add_v3_v3(y, ob->loc);
-  mul_m4_v3(ob->world_to_object, y);
+  add_v3_v3(r_local_dir, ob->loc);
+  mul_m4_v3(ob->world_to_object, r_local_dir);
 }
 
 static void calc_brush_local_mat(const float rotation,
@@ -2738,7 +2740,6 @@ static void calc_brush_local_mat(const float rotation,
   float mat[4][4];
   float scale[4][4];
   float angle, v[3];
-  float up[3];
 
   /* Ensure `ob->world_to_object` is up to date. */
   invert_m4_m4(ob->world_to_object, ob->object_to_world);
@@ -2749,17 +2750,33 @@ static void calc_brush_local_mat(const float rotation,
   mat[2][3] = 0.0f;
   mat[3][3] = 1.0f;
 
-  /* Get view's up vector in object-space. */
-  calc_local_y(cache->vc, cache->location, up);
+  /* Read rotation (user angle, rake, etc.) to find the view's movement direction (negative X of
+   * the brush). */
+  angle = rotation + cache->special_rotation;
+  /* By convention, motion direction points down the brush's Y axis, the angle represents the X
+   * axis, normal is a 90 deg ccw rotation of the motion direction. */
+  float motion_normal_screen[2];
+  motion_normal_screen[0] = cosf(angle);
+  motion_normal_screen[1] = sinf(angle);
+  /* Convert view's brush transverse direction to object-space,
+   * i.e. the normal of the plane described by the motion */
+  float motion_normal_local[3];
+  calc_local_from_screen(cache->vc, cache->location, motion_normal_screen, motion_normal_local);
 
-  /* Calculate the X axis of the local matrix. */
-  cross_v3_v3v3(v, up, cache->sculpt_normal);
-  /* Apply rotation (user angle, rake, etc.) to X axis. */
-  angle = rotation - cache->special_rotation;
-  rotate_v3_v3v3fl(mat[0], v, cache->sculpt_normal, angle);
+  /* Calculate the movement direction for the local matrix.
+   * Note that there is a deliberate prioritization here: Our calculations are
+   * designed such that the _motion vector_ gets projected into the tangent space;
+   * in most cases this will be more intuitive than projecting the transverse
+   * direction (which is orthogonal to the motion direction and therefore less
+   * apparent to the user).
+   * The Y-axis of the brush-local frame has to lie in the intersection of the tangent plane
+   * and the motion plane. */
+
+  cross_v3_v3v3(v, cache->sculpt_normal, motion_normal_local);
+  normalize_v3_v3(mat[1], v);
 
   /* Get other axes. */
-  cross_v3_v3v3(mat[1], cache->sculpt_normal, mat[0]);
+  cross_v3_v3v3(mat[0], mat[1], cache->sculpt_normal);
   copy_v3_v3(mat[2], cache->sculpt_normal);
 
   /* Set location. */

source/blender/makesdna/DNA_scene_types.h

@@ -1351,6 +1351,9 @@ typedef struct UnifiedPaintSettings {
   float average_stroke_accum[3];
   int average_stroke_counter;
 
+  /* How much brush should be rotated in the view plane, 0 means x points right, y points up.
+   * The convention is that the brush's _negative_ Y axis points in the tangent direction (of the
+   * mouse curve, Bezier curve, etc.) */
   float brush_rotation;
   float brush_rotation_sec;