Fix T68971: Copy As New Driver from Material node creates a bad reference.

NodeTree structures of materials and some other data blocks are
effectively node group datablock objects that are contained inside
the parent block. Thus, direct references to them are only valid
while blender is running, and are lost on save.

Fix Copy As New Driver to create a reference that goes through
the owner datablock, by adding a new ID flag to mark private
pseudo-datablocks.

Also fix functions that return full paths to structures and
properties, e.g. used in python tooltips. Functions for paths
from ID to struct or property can't be changed because of
Animation Data related code.

Reviewers: mont29

Differential Revision: https://developer.blender.org/D5559

source/blender/blenkernel/BKE_blender_version.h

@@ -27,7 +27,7 @@
  * \note Use #STRINGIFY() rather than defining with quotes.
  */
 #define BLENDER_VERSION 281
-#define BLENDER_SUBVERSION 3
+#define BLENDER_SUBVERSION 4
 /** Several breakages with 280, e.g. collections vs layers. */
 #define BLENDER_MINVERSION 280
 #define BLENDER_MINSUBVERSION 0

source/blender/blenkernel/BKE_node.h

@@ -384,6 +384,7 @@ void ntreeUserIncrefID(struct bNodeTree *ntree);
 void ntreeUserDecrefID(struct bNodeTree *ntree);
 
 struct bNodeTree *ntreeFromID(const struct ID *id);
+struct ID *BKE_node_tree_find_owner_ID(struct Main *bmain, struct bNodeTree *ntree);
 
 void ntreeMakeLocal(struct Main *bmain,
                     struct bNodeTree *ntree,

source/blender/blenkernel/intern/library.c

@@ -1417,6 +1417,9 @@ void BKE_libblock_copy_ex(Main *bmain, const ID *id, ID **r_newid, const int fla
   const bool use_nodetree_alloc_exception = ((GS(id->name) == ID_NT) && (bmain != NULL) &&
                                              (BLI_findindex(&bmain->nodetrees, id) < 0));
 
+  /* The id->flag bits to copy over. */
+  const int copy_flag_mask = LIB_PRIVATE_DATA;
+
   BLI_assert((flag & LIB_ID_CREATE_NO_MAIN) != 0 || bmain != NULL);
   BLI_assert((flag & LIB_ID_CREATE_NO_MAIN) != 0 || (flag & LIB_ID_CREATE_NO_ALLOCATE) == 0);
   BLI_assert((flag & LIB_ID_CREATE_NO_MAIN) == 0 || (flag & LIB_ID_CREATE_NO_USER_REFCOUNT) != 0);
@@ -1448,6 +1451,8 @@ void BKE_libblock_copy_ex(Main *bmain, const ID *id, ID **r_newid, const int fla
     memcpy(cpn + id_offset, cp + id_offset, id_len - id_offset);
   }
 
+  new_id->flag = (new_id->flag & ~copy_flag_mask) | (id->flag & copy_flag_mask);
+
   if (id->properties) {
     new_id->properties = IDP_CopyProperty_ex(id->properties, flag);
   }

source/blender/blenkernel/intern/node.c

@@ -1406,6 +1406,7 @@ bNodeTree *ntreeAddTree(Main *bmain, const char *name, const char *idname)
   }
   else {
     ntree = MEM_callocN(sizeof(bNodeTree), "new node tree");
+    ntree->id.flag |= LIB_PRIVATE_DATA;
     *((short *)ntree->id.name) = ID_NT;
     BLI_strncpy(ntree->id.name + 2, name, sizeof(ntree->id.name));
   }
@@ -2172,6 +2173,7 @@ void ntreeSetOutput(bNodeTree *ntree)
    * might be different for editor or for "real" use... */
 }
 
+/* Returns the private NodeTree object of the datablock, if it has one. */
 bNodeTree *ntreeFromID(const ID *id)
 {
   switch (GS(id->name)) {
@@ -2192,6 +2194,28 @@ bNodeTree *ntreeFromID(const ID *id)
   }
 }
 
+/* Finds and returns the datablock that privately owns the given tree, or NULL. */
+ID *BKE_node_tree_find_owner_ID(Main *bmain, struct bNodeTree *ntree)
+{
+  ListBase *lists[] = {&bmain->materials,
+                       &bmain->lights,
+                       &bmain->worlds,
+                       &bmain->textures,
+                       &bmain->scenes,
+                       &bmain->linestyles,
+                       NULL};
+
+  for (int i = 0; lists[i] != NULL; i++) {
+    LISTBASE_FOREACH (ID *, id, lists[i]) {
+      if (ntreeFromID(id) == ntree) {
+        return id;
+      }
+    }
+  }
+
+  return NULL;
+}
+
 void ntreeMakeLocal(Main *bmain, bNodeTree *ntree, bool id_in_mainlist, const bool lib_local)
 {
   BKE_id_make_local_generic(bmain, &ntree->id, id_in_mainlist, lib_local);

source/blender/blenloader/intern/versioning_280.c

@@ -3693,6 +3693,17 @@ void blo_do_versions_280(FileData *fd, Library *UNUSED(lib), Main *bmain)
     }
   }
 
+  if (!MAIN_VERSION_ATLEAST(bmain, 281, 4)) {
+    ID *id;
+    FOREACH_MAIN_ID_BEGIN (bmain, id) {
+      bNodeTree *ntree = ntreeFromID(id);
+      if (ntree) {
+        ntree->id.flag |= LIB_PRIVATE_DATA;
+      }
+    }
+    FOREACH_MAIN_ID_END;
+  }
+
   {
     /* Versioning code until next subversion bump goes here. */
   }

source/blender/editors/interface/interface_ops.c

@@ -100,10 +100,12 @@ static bool copy_data_path_button_poll(bContext *C)
 
 static int copy_data_path_button_exec(bContext *C, wmOperator *op)
 {
+  Main *bmain = CTX_data_main(C);
   PointerRNA ptr;
   PropertyRNA *prop;
   char *path;
   int index;
+  ID *id;
 
   const bool full_path = RNA_boolean_get(op->ptr, "full_path");
 
@@ -111,18 +113,20 @@ static int copy_data_path_button_exec(bContext *C, wmOperator *op)
   UI_context_active_but_prop_get(C, &ptr, &prop, &index);
 
   if (ptr.owner_id != NULL) {
-
     if (full_path) {
-
       if (prop) {
-        path = RNA_path_full_property_py_ex(&ptr, prop, index, true);
+        path = RNA_path_full_property_py_ex(bmain, &ptr, prop, index, true);
       }
       else {
-        path = RNA_path_full_struct_py(&ptr);
+        path = RNA_path_full_struct_py(bmain, &ptr);
       }
     }
     else {
-      path = RNA_path_from_ID_to_property(&ptr, prop);
+      path = RNA_path_from_real_ID_to_property_index(bmain, &ptr, prop, 0, -1, &id);
+
+      if (!path) {
+        path = RNA_path_from_ID_to_property(&ptr, prop);
+      }
     }
 
     if (path) {
@@ -185,8 +189,9 @@ static bool copy_as_driver_button_poll(bContext *C)
   return 0;
 }
 
-static int copy_as_driver_button_exec(bContext *C, wmOperator *UNUSED(op))
+static int copy_as_driver_button_exec(bContext *C, wmOperator *op)
 {
+  Main *bmain = CTX_data_main(C);
   PointerRNA ptr;
   PropertyRNA *prop;
   int index;
@@ -195,14 +200,19 @@ static int copy_as_driver_button_exec(bContext *C, wmOperator *UNUSED(op))
   UI_context_active_but_prop_get(C, &ptr, &prop, &index);
 
   if (ptr.owner_id && ptr.data && prop) {
+    ID *id;
     int dim = RNA_property_array_dimension(&ptr, prop, NULL);
-    char *path = RNA_path_from_ID_to_property_index(&ptr, prop, dim, index);
+    char *path = RNA_path_from_real_ID_to_property_index(bmain, &ptr, prop, dim, index, &id);
 
     if (path) {
-      ANIM_copy_as_driver(ptr.owner_id, path, RNA_property_identifier(prop));
+      ANIM_copy_as_driver(id, path, RNA_property_identifier(prop));
       MEM_freeN(path);
       return OPERATOR_FINISHED;
     }
+    else {
+      BKE_reportf(op->reports, RPT_ERROR, "Could not compute a valid data path");
+      return OPERATOR_CANCELLED;
+    }
   }
 
   return OPERATOR_CANCELLED;

source/blender/editors/interface/interface_region_tooltip.c

@@ -860,10 +860,10 @@ static uiTooltipData *ui_tooltip_data_from_button(bContext *C, uiBut *but)
       /* move ownership (no need for re-alloc) */
       if (but->rnaprop) {
         field->text = RNA_path_full_property_py_ex(
-            &but->rnapoin, but->rnaprop, but->rnaindex, true);
+            CTX_data_main(C), &but->rnapoin, but->rnaprop, but->rnaindex, true);
       }
       else {
-        field->text = RNA_path_full_struct_py(&but->rnapoin);
+        field->text = RNA_path_full_struct_py(CTX_data_main(C), &but->rnapoin);
       }
     }
   }

source/blender/editors/space_console/space_console.c

@@ -26,6 +26,7 @@
 #include "BLI_blenlib.h"
 #include "BLI_utildefines.h"
 
+#include "BKE_global.h"
 #include "BKE_context.h"
 #include "BKE_screen.h"
 
@@ -173,7 +174,7 @@ static void id_drop_copy(wmDrag *drag, wmDropBox *drop)
   ID *id = WM_drag_ID(drag, 0);
 
   /* copy drag path to properties */
-  char *text = RNA_path_full_ID_py(id);
+  char *text = RNA_path_full_ID_py(G_MAIN, id);
   RNA_string_set(drop->ptr, "text", text);
   MEM_freeN(text);
 }

source/blender/editors/space_text/space_text.c

@@ -29,6 +29,7 @@
 
 #include "BLI_blenlib.h"
 
+#include "BKE_global.h"
 #include "BKE_context.h"
 #include "BKE_library.h"
 #include "BKE_screen.h"
@@ -356,7 +357,7 @@ static void text_drop_paste(wmDrag *drag, wmDropBox *drop)
   ID *id = WM_drag_ID(drag, 0);
 
   /* copy drag path to properties */
-  text = RNA_path_full_ID_py(id);
+  text = RNA_path_full_ID_py(G_MAIN, id);
   RNA_string_set(drop->ptr, "text", text);
   MEM_freeN(text);
 }

source/blender/makesdna/DNA_ID.h

@@ -467,7 +467,11 @@ typedef enum ID_Type {
 
 /* id->flag (persitent). */
 enum {
+  /* Don't delete the datablock even if unused. */
   LIB_FAKEUSER = 1 << 9,
+  /* The datablock structure is a sub-object of a different one.
+   * Direct persistent references are not allowed. */
+  LIB_PRIVATE_DATA = 1 << 10,
 };
 
 /**

source/blender/makesrna/RNA_access.h

@@ -1151,24 +1151,37 @@ struct PropertyElemRNA {
 };
 bool RNA_path_resolve_elements(PointerRNA *ptr, const char *path, struct ListBase *r_elements);
 
+struct ID *RNA_find_real_ID_and_path(struct Main *bmain, struct ID *id, const char **r_path);
+
 char *RNA_path_from_ID_to_struct(PointerRNA *ptr);
+
+char *RNA_path_from_real_ID_to_struct(struct Main *bmain, PointerRNA *ptr, struct ID **r_real);
+
 char *RNA_path_from_ID_to_property(PointerRNA *ptr, PropertyRNA *prop);
 char *RNA_path_from_ID_to_property_index(PointerRNA *ptr,
                                          PropertyRNA *prop,
                                          int array_dim,
                                          int index);
 
+char *RNA_path_from_real_ID_to_property_index(struct Main *bmain,
+                                              PointerRNA *ptr,
+                                              PropertyRNA *prop,
+                                              int array_dim,
+                                              int index,
+                                              struct ID **r_real);
+
 char *RNA_path_resolve_from_type_to_property(struct PointerRNA *ptr,
                                              struct PropertyRNA *prop,
                                              const struct StructRNA *type);
 
-char *RNA_path_full_ID_py(struct ID *id);
-char *RNA_path_full_struct_py(struct PointerRNA *ptr);
-char *RNA_path_full_property_py_ex(PointerRNA *ptr,
-                                   PropertyRNA *prop,
-                                   int index,
-                                   bool use_fallback);
-char *RNA_path_full_property_py(struct PointerRNA *ptr, struct PropertyRNA *prop, int index);
+char *RNA_path_full_ID_py(struct Main *bmain, struct ID *id);
+char *RNA_path_full_struct_py(struct Main *bmain, struct PointerRNA *ptr);
+char *RNA_path_full_property_py_ex(
+    struct Main *bmain, PointerRNA *ptr, PropertyRNA *prop, int index, bool use_fallback);
+char *RNA_path_full_property_py(struct Main *bmain,
+                                struct PointerRNA *ptr,
+                                struct PropertyRNA *prop,
+                                int index);
 char *RNA_path_struct_property_py(struct PointerRNA *ptr, struct PropertyRNA *prop, int index);
 char *RNA_path_property_py(struct PointerRNA *ptr, struct PropertyRNA *prop, int index);
 

source/blender/makesrna/intern/rna_access.c

@@ -47,6 +47,7 @@
 #include "BKE_fcurve.h"
 #include "BKE_main.h"
 #include "BKE_report.h"
+#include "BKE_node.h"
 
 #include "DEG_depsgraph.h"
 
@@ -5757,6 +5758,61 @@ static char *rna_path_from_ID_to_idpgroup(PointerRNA *ptr)
   }
 }
 
+/**
+ * Find the actual ID pointer and path from it to the given ID.
+ *
+ * \param id: ID reference to search the global owner for.
+ * \param[out] r_path: Path from the real ID to the initial ID.
+ * \return The ID pointer, or NULL in case of failure.
+ */
+ID *RNA_find_real_ID_and_path(Main *bmain, ID *id, const char **r_path)
+{
+  if (r_path) {
+    *r_path = "";
+  }
+
+  if ((id != NULL) && (id->flag & LIB_PRIVATE_DATA)) {
+    switch (GS(id->name)) {
+      case ID_NT:
+        if (r_path) {
+          *r_path = "node_tree";
+        }
+        return BKE_node_tree_find_owner_ID(bmain, (bNodeTree *)id);
+
+      default:
+        return NULL;
+    }
+  }
+  else {
+    return id;
+  }
+}
+
+static char *rna_prepend_real_ID_path(Main *bmain, ID *id, char *path, ID **r_real)
+{
+  if (path) {
+    const char *prefix;
+    char *new_path = NULL;
+
+    *r_real = RNA_find_real_ID_and_path(bmain, id, &prefix);
+
+    if (*r_real) {
+      if (prefix[0]) {
+        new_path = BLI_sprintfN("%s%s%s", prefix, path[0] == '[' ? "" : ".", path);
+      }
+      else {
+        return path;
+      }
+    }
+
+    MEM_freeN(path);
+    return new_path;
+  }
+  else {
+    return NULL;
+  }
+}
+
 char *RNA_path_from_ID_to_struct(PointerRNA *ptr)
 {
   char *ptrpath = NULL;
@@ -5799,6 +5855,13 @@ char *RNA_path_from_ID_to_struct(PointerRNA *ptr)
   return ptrpath;
 }
 
+char *RNA_path_from_real_ID_to_struct(Main *bmain, PointerRNA *ptr, struct ID **r_real)
+{
+  char *path = RNA_path_from_ID_to_struct(ptr);
+
+  return rna_prepend_real_ID_path(bmain, ptr->owner_id, path, r_real);
+}
+
 static void rna_path_array_multi_from_flat_index(const int dimsize[RNA_MAX_ARRAY_LENGTH],
                                                  const int totdims,
                                                  const int index_dim,
@@ -5905,6 +5968,14 @@ char *RNA_path_from_ID_to_property(PointerRNA *ptr, PropertyRNA *prop)
   return RNA_path_from_ID_to_property_index(ptr, prop, 0, -1);
 }
 
+char *RNA_path_from_real_ID_to_property_index(
+    Main *bmain, PointerRNA *ptr, PropertyRNA *prop, int index_dim, int index, ID **r_real)
+{
+  char *path = RNA_path_from_ID_to_property_index(ptr, prop, index_dim, index);
+
+  return rna_prepend_real_ID_path(bmain, ptr->owner_id, path, r_real);
+}
+
 /**
  * \return the path to given ptr/prop from the closest ancestor of given type,
  * if any (else return NULL).
@@ -5951,20 +6022,34 @@ char *RNA_path_resolve_from_type_to_property(PointerRNA *ptr,
  * Get the ID as a python representation, eg:
  *   bpy.data.foo["bar"]
  */
-char *RNA_path_full_ID_py(ID *id)
+char *RNA_path_full_ID_py(Main *bmain, ID *id)
 {
+  const char *path;
+  ID *id_real = RNA_find_real_ID_and_path(bmain, id, &path);
+
+  if (id_real) {
+    id = id_real;
+  }
+  else {
+    path = "";
+  }
+
   char id_esc[(sizeof(id->name) - 2) * 2];
 
   BLI_strescape(id_esc, id->name + 2, sizeof(id_esc));
 
-  return BLI_sprintfN("bpy.data.%s[\"%s\"]", BKE_idcode_to_name_plural(GS(id->name)), id_esc);
+  return BLI_sprintfN("bpy.data.%s[\"%s\"]%s%s",
+                      BKE_idcode_to_name_plural(GS(id->name)),
+                      id_esc,
+                      path[0] ? "." : "",
+                      path);
 }
 
 /**
  * Get the ID.struct as a python representation, eg:
  *   bpy.data.foo["bar"].some_struct
  */
-char *RNA_path_full_struct_py(struct PointerRNA *ptr)
+char *RNA_path_full_struct_py(Main *bmain, struct PointerRNA *ptr)
 {
   char *id_path;
   char *data_path;
@@ -5976,7 +6061,7 @@ char *RNA_path_full_struct_py(struct PointerRNA *ptr)
   }
 
   /* never fails */
-  id_path = RNA_path_full_ID_py(ptr->owner_id);
+  id_path = RNA_path_full_ID_py(bmain, ptr->owner_id);
 
   data_path = RNA_path_from_ID_to_struct(ptr);
 
@@ -5996,10 +6081,8 @@ char *RNA_path_full_struct_py(struct PointerRNA *ptr)
  * Get the ID.struct.property as a python representation, eg:
  *   bpy.data.foo["bar"].some_struct.some_prop[10]
  */
-char *RNA_path_full_property_py_ex(PointerRNA *ptr,
-                                   PropertyRNA *prop,
-                                   int index,
-                                   bool use_fallback)
+char *RNA_path_full_property_py_ex(
+    Main *bmain, PointerRNA *ptr, PropertyRNA *prop, int index, bool use_fallback)
 {
   char *id_path;
   const char *data_delim;
@@ -6013,7 +6096,7 @@ char *RNA_path_full_property_py_ex(PointerRNA *ptr,
   }
 
   /* never fails */
-  id_path = RNA_path_full_ID_py(ptr->owner_id);
+  id_path = RNA_path_full_ID_py(bmain, ptr->owner_id);
 
   data_path = RNA_path_from_ID_to_property(ptr, prop);
   if (data_path) {
@@ -6046,9 +6129,9 @@ char *RNA_path_full_property_py_ex(PointerRNA *ptr,
   return ret;
 }
 
-char *RNA_path_full_property_py(PointerRNA *ptr, PropertyRNA *prop, int index)
+char *RNA_path_full_property_py(Main *bmain, PointerRNA *ptr, PropertyRNA *prop, int index)
 {
-  return RNA_path_full_property_py_ex(ptr, prop, index, false);
+  return RNA_path_full_property_py_ex(bmain, ptr, prop, index, false);
 }
 
 /**
@@ -6646,16 +6729,16 @@ char *RNA_pointer_as_string_id(bContext *C, PointerRNA *ptr)
   return cstring;
 }
 
-static char *rna_pointer_as_string__bldata(PointerRNA *ptr)
+static char *rna_pointer_as_string__bldata(Main *bmain, PointerRNA *ptr)
 {
   if (ptr->type == NULL || ptr->owner_id == NULL) {
     return BLI_strdup("None");
   }
   else if (RNA_struct_is_ID(ptr->type)) {
-    return RNA_path_full_ID_py(ptr->owner_id);
+    return RNA_path_full_ID_py(bmain, ptr->owner_id);
   }
   else {
-    return RNA_path_full_struct_py(ptr);
+    return RNA_path_full_struct_py(bmain, ptr);
   }
 }
 
@@ -6672,7 +6755,7 @@ char *RNA_pointer_as_string(bContext *C,
     return RNA_pointer_as_string_id(C, ptr_prop);
   }
   else {
-    return rna_pointer_as_string__bldata(ptr_prop);
+    return rna_pointer_as_string__bldata(CTX_data_main(C), ptr_prop);
   }
 }
 

source/blender/windowmanager/intern/wm_operators.c

@@ -539,7 +539,7 @@ char *WM_prop_pystring_assign(bContext *C, PointerRNA *ptr, PropertyRNA *prop, i
 
   if (lhs == NULL) {
     /* fallback to bpy.data.foo[id] if we dont find in the context */
-    lhs = RNA_path_full_property_py(ptr, prop, index);
+    lhs = RNA_path_full_property_py(CTX_data_main(C), ptr, prop, index);
   }
 
   if (!lhs) {