Fix: Inserting keys no longer sets the FCurve channel color

When pressing `I` in the viewport, the colors of the FCurve channels were no longer set correctly.

Caused by #113504

Fix by removing the flag that determined that in the first place,
and read straight from the user preferences. Then move the code
that sets the mode on the FCurve into the function that actually creates it.

For 99% of cases the code went to the user preference flag `AUTOKEY_FLAG_XYZ2RGB`
and if that was set, the `INSERTKEY_XYZ2RGB` would be set. The only case where this
was not from the user preferences was from custom keying sets.
There was an override flag for FCurve colors on custom keying sets.
I removed that with this patch since the use case is not apparent
and custom keying sets are hardly used.

Pull Request: https://projects.blender.org/blender/blender/pulls/115297

scripts/startup/bl_ui/properties_scene.py

@@ -95,10 +95,6 @@ class SceneKeyingSetsPanel:
             "use_insertkey_override_visual", "use_insertkey_visual",
             userpref_fallback="use_visual_keying",
         )
-        SceneKeyingSetsPanel._draw_keyframing_setting(
-            context, layout, ks, ksp, iface_("XYZ to RGB"),
-            "use_insertkey_override_xyz_to_rgb", "use_insertkey_xyz_to_rgb",
-        )
 
     @staticmethod
     def _draw_keyframing_setting(context, layout, ks, ksp, label, toggle_prop, prop, userpref_fallback=None):

source/blender/animrig/intern/action.cc

@@ -13,6 +13,9 @@
 #include "BLI_string.h"
 #include "DEG_depsgraph_build.hh"
 #include "DNA_anim_types.h"
+
+#include "RNA_access.hh"
+#include "RNA_path.hh"
 #include "RNA_prototypes.h"
 
 namespace blender::animrig {
@@ -57,6 +60,22 @@ FCurve *action_fcurve_ensure(Main *bmain,
   fcu->rna_path = BLI_strdup(rna_path);
   fcu->array_index = array_index;
 
+  if (U.autokey_flag & AUTOKEY_FLAG_XYZ2RGB) {
+    /* For Loc/Rot/Scale and also Color F-Curves, the color of the F-Curve in the Graph Editor,
+     * is determined by the array index for the F-Curve.
+     */
+    PropertyRNA *prop;
+    PointerRNA r_ptr;
+    RNA_path_resolve_property(ptr, rna_path, &r_ptr, &prop);
+    PropertySubType prop_subtype = RNA_property_subtype(prop);
+    if (ELEM(prop_subtype, PROP_TRANSLATION, PROP_XYZ, PROP_EULER, PROP_COLOR, PROP_COORDS)) {
+      fcu->color_mode = FCURVE_COLOR_AUTO_RGB;
+    }
+    else if (ELEM(prop_subtype, PROP_QUATERNION)) {
+      fcu->color_mode = FCURVE_COLOR_AUTO_YRGB;
+    }
+  }
+
   if (group) {
     bActionGroup *agrp = BKE_action_group_find_name(act, group);
 

source/blender/animrig/intern/keyframing.cc

@@ -37,10 +37,8 @@
 #include "ED_keyframing.hh"
 #include "MEM_guardedalloc.h"
 #include "RNA_access.hh"
-#include "RNA_define.hh"
 #include "RNA_path.hh"
 #include "RNA_prototypes.h"
-#include "RNA_types.hh"
 
 #include "WM_api.hh"
 #include "WM_types.hh"
@@ -547,20 +545,6 @@ static bool insert_keyframe_fcurve_value(Main *bmain,
 
   const bool is_new_curve = (fcu->totvert == 0);
 
-  /* Set color mode if the F-Curve is new (i.e. without any keyframes). */
-  if (is_new_curve && (flag & INSERTKEY_XYZ2RGB)) {
-    /* For Loc/Rot/Scale and also Color F-Curves, the color of the F-Curve in the Graph Editor,
-     * is determined by the array index for the F-Curve
-     */
-    PropertySubType prop_subtype = RNA_property_subtype(prop);
-    if (ELEM(prop_subtype, PROP_TRANSLATION, PROP_XYZ, PROP_EULER, PROP_COLOR, PROP_COORDS)) {
-      fcu->color_mode = FCURVE_COLOR_AUTO_RGB;
-    }
-    else if (ELEM(prop_subtype, PROP_QUATERNION)) {
-      fcu->color_mode = FCURVE_COLOR_AUTO_YRGB;
-    }
-  }
-
   /* If the curve has only one key, make it cyclic if appropriate. */
   const bool is_cyclic_action = (flag & INSERTKEY_CYCLE_AWARE) && BKE_action_is_cyclic(act);
 

source/blender/editors/animation/keyframing.cc

@@ -91,11 +91,6 @@ eInsertKeyFlags ANIM_get_keyframing_flags(Scene *scene, const bool use_autokey_m
     if (is_autokey_flag(scene, AUTOKEY_FLAG_INSERTNEEDED)) {
       flag |= INSERTKEY_NEEDED;
     }
-
-    /* default F-Curve color mode - RGB from XYZ indices */
-    if (is_autokey_flag(scene, AUTOKEY_FLAG_XYZ2RGB)) {
-      flag |= INSERTKEY_XYZ2RGB;
-    }
   }
 
   /* only if including settings from the autokeying mode... */

source/blender/editors/animation/keyingsets.cc

@@ -294,10 +294,6 @@ static int add_keyingset_button_exec(bContext *C, wmOperator *op)
 
     keyingflag |= ANIM_get_keyframing_flags(scene, false);
 
-    if (blender::animrig::is_autokey_flag(scene, AUTOKEY_FLAG_XYZ2RGB)) {
-      keyingflag |= INSERTKEY_XYZ2RGB;
-    }
-
     /* call the API func, and set the active keyingset index */
     ks = BKE_keyingset_add(
         &scene->keyingsets, "ButtonKeyingSet", "Button Keying Set", flag, keyingflag);
@@ -1011,7 +1007,6 @@ static eInsertKeyFlags keyingset_apply_keying_flags(const eInsertKeyFlags base_f
    */
   APPLY_KEYINGFLAG_OVERRIDE(INSERTKEY_NEEDED)
   APPLY_KEYINGFLAG_OVERRIDE(INSERTKEY_MATRIX)
-  APPLY_KEYINGFLAG_OVERRIDE(INSERTKEY_XYZ2RGB)
 
 #undef APPLY_KEYINGFLAG_OVERRIDE
 

source/blender/makesdna/DNA_anim_types.h

@@ -1035,8 +1035,6 @@ typedef enum eInsertKeyFlags {
   /* INSERTKEY_FASTR = (1 << 3), */ /* UNUSED */
   /** only replace an existing keyframe (this overrides INSERTKEY_NEEDED) */
   INSERTKEY_REPLACE = (1 << 4),
-  /** transform F-Curves should have XYZ->RGB color mode */
-  INSERTKEY_XYZ2RGB = (1 << 5),
   /** ignore user-prefs (needed for predictable API use) */
   INSERTKEY_NO_USERPREF = (1 << 6),
   /**

source/blender/makesrna/intern/rna_animation.cc

@@ -51,12 +51,6 @@ const EnumPropertyItem rna_enum_keying_flag_items[] = {
      0,
      "Visual Keying",
      "Insert keyframes based on 'visual transforms'"},
-    {INSERTKEY_XYZ2RGB,
-     "INSERTKEY_XYZ_TO_RGB",
-     0,
-     "XYZ=RGB Colors",
-     "Color for newly added transformation F-Curves (Location, Rotation, Scale) "
-     "and also Color is based on the transform axis"},
     {0, nullptr, 0, nullptr, nullptr},
 };
 
@@ -72,12 +66,6 @@ const EnumPropertyItem rna_enum_keying_flag_api_items[] = {
      0,
      "Visual Keying",
      "Insert keyframes based on 'visual transforms'"},
-    {INSERTKEY_XYZ2RGB,
-     "INSERTKEY_XYZ_TO_RGB",
-     0,
-     "XYZ=RGB Colors",
-     "Color for newly added transformation F-Curves (Location, Rotation, Scale) "
-     "and also Color is based on the transform axis"},
     {INSERTKEY_REPLACE,
      "INSERTKEY_REPLACE",
      0,
@@ -877,17 +865,6 @@ static void rna_def_common_keying_flags(StructRNA *srna, short reg)
     RNA_def_property_flag(prop, PROP_REGISTER_OPTIONAL);
   }
 
-  prop = RNA_def_property(srna, "use_insertkey_override_xyz_to_rgb", PROP_BOOLEAN, PROP_NONE);
-  RNA_def_property_boolean_sdna(prop, nullptr, "keyingoverride", INSERTKEY_XYZ2RGB);
-  RNA_def_property_ui_text(
-      prop,
-      "Override F-Curve Colors - XYZ to RGB",
-      "Override default setting to set color for newly added transformation F-Curves "
-      "(Location, Rotation, Scale) to be based on the transform axis");
-  if (reg) {
-    RNA_def_property_flag(prop, PROP_REGISTER_OPTIONAL);
-  }
-
   /* value to override defaults with */
   prop = RNA_def_property(srna, "use_insertkey_needed", PROP_BOOLEAN, PROP_NONE);
   RNA_def_property_boolean_sdna(prop, nullptr, "keyingflag", INSERTKEY_NEEDED);
@@ -905,16 +882,6 @@ static void rna_def_common_keying_flags(StructRNA *srna, short reg)
   if (reg) {
     RNA_def_property_flag(prop, PROP_REGISTER_OPTIONAL);
   }
-
-  prop = RNA_def_property(srna, "use_insertkey_xyz_to_rgb", PROP_BOOLEAN, PROP_NONE);
-  RNA_def_property_boolean_sdna(prop, nullptr, "keyingflag", INSERTKEY_XYZ2RGB);
-  RNA_def_property_ui_text(prop,
-                           "F-Curve Colors - XYZ to RGB",
-                           "Color for newly added transformation F-Curves (Location, Rotation, "
-                           "Scale) is based on the transform axis");
-  if (reg) {
-    RNA_def_property_flag(prop, PROP_REGISTER_OPTIONAL);
-  }
 }
 
 /* --- */