LibOverrides: fix handling of hierarchy root in complex cases.

This affects essentially the Outliner 'create hierarchy' tool currenlty.
Previously code did not handle properly hierarchy root in case overrides
where created from a non-root ID (e.g. an object inside of a linked
collection), and in case additional partial overrides were added to an
existing partially overrided hierarchy.

Also did some renaming on the go to avoid using 'reference' in override
context for anything else but the reference linked IDs.

source/blender/blenkernel/BKE_lib_override.h

@@ -92,7 +92,13 @@ struct ID *BKE_lib_override_library_create_from_id(struct Main *bmain,
  *
  * \param id_hierarchy_root: the override ID that is the root of the hierarchy. May be NULL, in
  * which case it is assumed that the given `id_root_reference` is tagged for override, and its
- * newly created override will be used as hierarchy root.
+ * newly created override will be used as hierarchy root. Must be NULL if
+ * `id_hierarchy_root_reference` is not NULL.
+ *
+ * \param id_hierarchy_root_reference: the linked ID that is the root of the hierarchy. Must be
+ * tagged for override. May be NULL, in which case it is assumed that the given `id_root_reference`
+ * is tagged for override, and its newly created override will be used as hierarchy root. Must be
+ * NULL if `id_hierarchy_root` is not NULL.
  *
  * \param do_no_main: Create the new override data outside of Main database.
  * Used for resyncing of linked overrides.
@@ -103,6 +109,7 @@ bool BKE_lib_override_library_create_from_tag(struct Main *bmain,
                                               struct Library *owner_library,
                                               const struct ID *id_root_reference,
                                               struct ID *id_hierarchy_root,
+                                              const struct ID *id_hierarchy_root_reference,
                                               bool do_no_main);
 /**
  * Advanced 'smart' function to create fully functional overrides.
@@ -119,11 +126,18 @@ bool BKE_lib_override_library_create_from_tag(struct Main *bmain,
  * \param owner_library: the library in which the overrides should be created. Besides versioning
  * and resync code path, this should always be NULL (i.e. the local .blend file).
  *
- * \param id_root: The root ID to create an override from.
+ * \param id_root_reference: The linked root ID to create an override from. May be a sub-root of
+ * the overall hierarchy, in which case calling code is expected to have already tagged required
+ * 'path' of IDs leading from the given `id_hierarchy_root` to the given `id_root`.
  *
- * \param id_reference: Some reference ID used to do some post-processing after overrides have been
+ * \param id_hierarchy_root_reference: The ID to be used a hierarchy root of the overrides to be
- * created, may be NULL. Typically, the Empty object instantiating the linked collection we
+ * created. Can be either the linked root ID of the whole override hierarchy, (typically the same
- * override, currently.
+ * as `id_root`, unless a sub-part only of the hierarchy is overridden), or the already existing
+ * override hierarchy root if part of the hierarchy is already overridden.
+ *
+ * \param id_instance_hint: Some ID used as hint/reference to do some post-processing after
+ * overrides have been created, may be NULL. Typically, the Empty object instantiating the linked
+ * collection we override, currently.
  *
  * \param r_id_root_override: if not NULL, the override generated for the given \a id_root.
  *
@@ -133,8 +147,9 @@ bool BKE_lib_override_library_create(struct Main *bmain,
                                      struct Scene *scene,
                                      struct ViewLayer *view_layer,
                                      struct Library *owner_library,
-                                     struct ID *id_root,
+                                     struct ID *id_root_reference,
-                                     struct ID *id_reference,
+                                     struct ID *id_hierarchy_root_reference,
+                                     struct ID *id_instance_hint,
                                      struct ID **r_id_root_override);
 /**
  * Create a library override template.

source/blender/blenkernel/intern/lib_override.c

@@ -321,17 +321,36 @@ ID *BKE_lib_override_library_create_from_id(Main *bmain,
   return local_id;
 }
 
+/* TODO: Make this static local function instead? API is becoming complex, and it's not used
+ * outside of this file anyway. */
 bool BKE_lib_override_library_create_from_tag(Main *bmain,
                                               Library *owner_library,
                                               const ID *id_root_reference,
                                               ID *id_hierarchy_root,
+                                              const ID *id_hierarchy_root_reference,
                                               const bool do_no_main)
 {
-  BLI_assert(id_root_reference != NULL);
+  BLI_assert(id_root_reference != NULL && ID_IS_LINKED(id_root_reference));
-  BLI_assert(id_hierarchy_root != NULL || (id_root_reference->tag & LIB_TAG_DOIT) != 0);
+  /* If we do not have any hierarchy root given, then the root reference must be tagged for
-  BLI_assert(id_hierarchy_root == NULL ||
+   * override. */
-             (ID_IS_OVERRIDE_LIBRARY_REAL(id_hierarchy_root) &&
+  BLI_assert(id_hierarchy_root != NULL || id_hierarchy_root_reference != NULL ||
-              id_hierarchy_root->override_library->reference == id_root_reference));
+             (id_root_reference->tag & LIB_TAG_DOIT) != 0);
+  /* At least one of the hierarchy root pointers must be NULL, passing both is useless and can
+   * create confusion. */
+  BLI_assert(ELEM(NULL, id_hierarchy_root, id_hierarchy_root_reference));
+
+  if (id_hierarchy_root != NULL) {
+    /* If the hierarchy root is given, it must be a valid existing override (used during partial
+     * resync process mainly). */
+    BLI_assert((ID_IS_OVERRIDE_LIBRARY_REAL(id_hierarchy_root) &&
+                id_hierarchy_root->override_library->reference->lib == id_root_reference->lib));
+  }
+  if (!ELEM(id_hierarchy_root_reference, NULL, id_root_reference)) {
+    /* If the reference hierarchy root is given, it must be from the same library as the reference
+     * root, and also tagged for override. */
+    BLI_assert((id_hierarchy_root_reference->lib == id_root_reference->lib &&
+                (id_hierarchy_root_reference->tag & LIB_TAG_DOIT) != 0));
+  }
 
   const Library *reference_library = id_root_reference->lib;
 
@@ -387,7 +406,12 @@ bool BKE_lib_override_library_create_from_tag(Main *bmain,
   /* Only remap new local ID's pointers, we don't want to force our new overrides onto our whole
    * existing linked IDs usages. */
   if (success) {
-    if (id_root_reference->newid != NULL) {
+    if (id_hierarchy_root_reference != NULL) {
+      id_hierarchy_root = id_hierarchy_root_reference->newid;
+    }
+    else if (id_root_reference->newid != NULL &&
+             (id_hierarchy_root == NULL ||
+              id_hierarchy_root->override_library->reference == id_root_reference)) {
       id_hierarchy_root = id_root_reference->newid;
     }
     BLI_assert(id_hierarchy_root != NULL);
@@ -887,12 +911,13 @@ static void lib_override_overrides_group_tag(LibOverrideGroupTagData *data)
 static bool lib_override_library_create_do(Main *bmain,
                                            Scene *scene,
                                            Library *owner_library,
-                                           ID *id_root)
+                                           ID *id_root_reference,
+                                           ID *id_hierarchy_root_reference)
 {
   BKE_main_relations_create(bmain, 0);
   LibOverrideGroupTagData data = {.bmain = bmain,
                                   .scene = scene,
-                                  .id_root = id_root,
+                                  .id_root = id_root_reference,
                                   .tag = LIB_TAG_DOIT,
                                   .missing_tag = LIB_TAG_MISSING,
                                   .is_override = false,
@@ -906,8 +931,18 @@ static bool lib_override_library_create_do(Main *bmain,
   BKE_main_relations_free(bmain);
   lib_override_group_tag_data_clear(&data);
 
-  const bool success = BKE_lib_override_library_create_from_tag(
+  bool success = false;
-      bmain, owner_library, id_root, NULL, false);
+  if (id_hierarchy_root_reference->lib != id_root_reference->lib) {
+    BLI_assert(ID_IS_OVERRIDE_LIBRARY_REAL(id_hierarchy_root_reference));
+    BLI_assert(id_hierarchy_root_reference->override_library->reference->lib ==
+               id_root_reference->lib);
+    success = BKE_lib_override_library_create_from_tag(
+        bmain, owner_library, id_root_reference, id_hierarchy_root_reference, NULL, false);
+  }
+  else {
+    success = BKE_lib_override_library_create_from_tag(
+        bmain, owner_library, id_root_reference, NULL, id_hierarchy_root_reference, false);
+  }
 
   return success;
 }
@@ -917,7 +952,7 @@ static void lib_override_library_create_post_process(Main *bmain,
                                                      ViewLayer *view_layer,
                                                      const Library *owner_library,
                                                      ID *id_root,
-                                                     ID *id_reference,
+                                                     ID *id_instance_hint,
                                                      Collection *residual_storage,
                                                      const bool is_resync)
 {
@@ -941,8 +976,8 @@ static void lib_override_library_create_post_process(Main *bmain,
       (!ID_IS_LINKED(id_root->newid) || id_root->newid->lib == owner_library)) {
     switch (GS(id_root->name)) {
       case ID_GR: {
-        Object *ob_reference = id_reference != NULL && GS(id_reference->name) == ID_OB ?
+        Object *ob_reference = id_instance_hint != NULL && GS(id_instance_hint->name) == ID_OB ?
-                                   (Object *)id_reference :
+                                   (Object *)id_instance_hint :
                                    NULL;
         Collection *collection_new = ((Collection *)id_root->newid);
         if (is_resync && BKE_collection_is_in_scene(collection_new)) {
@@ -951,10 +986,10 @@ static void lib_override_library_create_post_process(Main *bmain,
         if (ob_reference != NULL) {
           BKE_collection_add_from_object(bmain, scene, ob_reference, collection_new);
         }
-        else if (id_reference != NULL) {
+        else if (id_instance_hint != NULL) {
-          BLI_assert(GS(id_reference->name) == ID_GR);
+          BLI_assert(GS(id_instance_hint->name) == ID_GR);
           BKE_collection_add_from_collection(
-              bmain, scene, ((Collection *)id_reference), collection_new);
+              bmain, scene, ((Collection *)id_instance_hint), collection_new);
         }
         else {
           BKE_collection_add_from_collection(
@@ -1047,26 +1082,32 @@ bool BKE_lib_override_library_create(Main *bmain,
                                      Scene *scene,
                                      ViewLayer *view_layer,
                                      Library *owner_library,
-                                     ID *id_root,
+                                     ID *id_root_reference,
-                                     ID *id_reference,
+                                     ID *id_hierarchy_root_reference,
+                                     ID *id_instance_hint,
                                      ID **r_id_root_override)
 {
   if (r_id_root_override != NULL) {
     *r_id_root_override = NULL;
   }
 
-  const bool success = lib_override_library_create_do(bmain, scene, owner_library, id_root);
+  if (id_hierarchy_root_reference == NULL) {
+    id_hierarchy_root_reference = id_root_reference;
+  }
+
+  const bool success = lib_override_library_create_do(
+      bmain, scene, owner_library, id_root_reference, id_hierarchy_root_reference);
 
   if (!success) {
     return success;
   }
 
   if (r_id_root_override != NULL) {
-    *r_id_root_override = id_root->newid;
+    *r_id_root_override = id_root_reference->newid;
   }
 
   lib_override_library_create_post_process(
-      bmain, scene, view_layer, owner_library, id_root, id_reference, NULL, false);
+      bmain, scene, view_layer, owner_library, id_root_reference, id_instance_hint, NULL, false);
 
   /* Cleanup. */
   BKE_main_id_newptr_and_tag_clear(bmain);
@@ -1527,7 +1568,7 @@ static bool lib_override_library_resync(Main *bmain,
    * override IDs (including within the old overrides themselves, since those are tagged too
    * above). */
   const bool success = BKE_lib_override_library_create_from_tag(
-      bmain, NULL, id_root_reference, id_root->override_library->hierarchy_root, true);
+      bmain, NULL, id_root_reference, id_root->override_library->hierarchy_root, NULL, true);
 
   if (!success) {
     BLI_ghash_free(linkedref_to_old_override, NULL, NULL);

source/blender/blenkernel/intern/lib_override_proxy_conversion.c

@@ -42,7 +42,7 @@ bool BKE_lib_override_library_proxy_convert(Main *bmain,
   const bool is_override_instancing_object = ob_proxy_group != NULL;
   ID *id_root = is_override_instancing_object ? &ob_proxy_group->instance_collection->id :
                                                 &ob_proxy->proxy->id;
-  ID *id_reference = is_override_instancing_object ? &ob_proxy_group->id : &ob_proxy->id;
+  ID *id_instance_hint = is_override_instancing_object ? &ob_proxy_group->id : &ob_proxy->id;
 
   /* In some cases the instance collection of a proxy object may be local (see e.g. T83875). Not
    * sure this is a valid state, but for now just abort the overriding process. */
@@ -81,7 +81,7 @@ bool BKE_lib_override_library_proxy_convert(Main *bmain,
   FOREACH_MAIN_ID_END;
 
   return BKE_lib_override_library_create(
-      bmain, scene, view_layer, ob_proxy->id.lib, id_root, id_reference, NULL);
+      bmain, scene, view_layer, ob_proxy->id.lib, id_root, id_root, id_instance_hint, NULL);
 }
 
 static void lib_override_library_proxy_convert_do(Main *bmain,

source/blender/editors/object/object_relations.c

@@ -2318,7 +2318,7 @@ static int make_override_library_exec(bContext *C, wmOperator *op)
   BKE_main_id_tag_all(bmain, LIB_TAG_DOIT, false);
 
   const bool success = BKE_lib_override_library_create(
-      bmain, scene, view_layer, NULL, id_root, &obact->id, NULL);
+      bmain, scene, view_layer, NULL, id_root, id_root, &obact->id, NULL);
 
   /* Remove the instance empty from this scene, the items now have an overridden collection
    * instead. */

source/blender/editors/space_outliner/outliner_tools.cc

@@ -766,29 +766,30 @@ static void id_override_library_create_fn(bContext *C,
                                           void *user_data)
 {
   BLI_assert(TSE_IS_REAL_ID(tselem));
-  ID *id_root = tselem->id;
+  ID *id_root_reference = tselem->id;
   OutlinerLibOverrideData *data = reinterpret_cast<OutlinerLibOverrideData *>(user_data);
   const bool do_hierarchy = data->do_hierarchy;
   bool success = false;
 
-  ID *id_reference = nullptr;
+  ID *id_instance_hint = nullptr;
   bool is_override_instancing_object = false;
   if (tsep != nullptr && tsep->type == TSE_SOME_ID && tsep->id != nullptr &&
       GS(tsep->id->name) == ID_OB && !ID_IS_OVERRIDE_LIBRARY(tsep->id)) {
     Object *ob = (Object *)tsep->id;
-    if (ob->type == OB_EMPTY && &ob->instance_collection->id == id_root) {
+    if (ob->type == OB_EMPTY && &ob->instance_collection->id == id_root_reference) {
-      BLI_assert(GS(id_root->name) == ID_GR);
+      BLI_assert(GS(id_root_reference->name) == ID_GR);
       /* Empty instantiating the collection we override, we need to pass it to BKE overriding code
        * for proper handling. */
-      id_reference = tsep->id;
+      id_instance_hint = tsep->id;
       is_override_instancing_object = true;
     }
   }
 
-  if (ID_IS_OVERRIDABLE_LIBRARY(id_root) || (ID_IS_LINKED(id_root) && do_hierarchy)) {
+  if (ID_IS_OVERRIDABLE_LIBRARY(id_root_reference) ||
+      (ID_IS_LINKED(id_root_reference) && do_hierarchy)) {
     Main *bmain = CTX_data_main(C);
 
-    id_root->tag |= LIB_TAG_DOIT;
+    id_root_reference->tag |= LIB_TAG_DOIT;
 
     /* For now, remap all local usages of linked ID to local override one here. */
     ID *id_iter;
@@ -804,38 +805,100 @@ static void id_override_library_create_fn(bContext *C,
 
     if (do_hierarchy) {
       /* Tag all linked parents in tree hierarchy to be also overridden. */
+      ID *id_hierarchy_root_reference = id_root_reference;
       while ((te = te->parent) != nullptr) {
         if (!TSE_IS_REAL_ID(te->store_elem)) {
           continue;
         }
-        if (!ID_IS_LINKED(te->store_elem->id)) {
+
+        /* Tentative hierarchy root. */
+        ID *id_current_hierarchy_root = te->store_elem->id;
+
+        /* If the parent ID is from a different library than the reference root one, we are done
+         * with upwards tree processing in any case. */
+        if (id_current_hierarchy_root->lib != id_root_reference->lib) {
+          if (ID_IS_OVERRIDE_LIBRARY_VIRTUAL(id_current_hierarchy_root)) {
+            /* Virtual overrides (i.e. embedded IDs), we can simply keep processing their parent to
+             * get an actual real override. */
+            continue;
+          }
+
+          /* If the parent ID is already an override, and is valid (i.e. local override), we can
+           * access its hierarchy root directly. */
+          if (!ID_IS_LINKED(id_current_hierarchy_root) &&
+              ID_IS_OVERRIDE_LIBRARY_REAL(id_current_hierarchy_root) &&
+              id_current_hierarchy_root->override_library->reference->lib ==
+                  id_root_reference->lib) {
+            id_hierarchy_root_reference =
+                id_current_hierarchy_root->override_library->hierarchy_root;
+            BLI_assert(ID_IS_OVERRIDE_LIBRARY_REAL(id_hierarchy_root_reference));
+            break;
+          }
+
+          if (ID_IS_LINKED(id_current_hierarchy_root)) {
+            /* No local 'anchor' was found for the hierarchy to override, do not proceed, as this
+             * would most likely generate invisible/confusing/hard to use and manage overrides. */
+            BKE_main_id_tag_all(bmain, LIB_TAG_DOIT, false);
+            BKE_reportf(reports,
+                        RPT_WARNING,
+                        "Invalid anchor ('%s') found, needed to create library override from "
+                        "data-block '%s'",
+                        id_current_hierarchy_root->name,
+                        id_root_reference->name);
+            return;
+          }
+
+          /* In all other cases, `id_current_hierarchy_root` cannot be a valid hierarchy root, so
+           * current `id_hierarchy_root_reference` is our best candidate. */
+
           break;
         }
+
         /* If some element in the tree needs to be overridden, but its ID is not overridable,
          * abort. */
-        if (!ID_IS_OVERRIDABLE_LIBRARY_HIERARCHY(te->store_elem->id)) {
+        if (!ID_IS_OVERRIDABLE_LIBRARY_HIERARCHY(id_current_hierarchy_root)) {
           BKE_main_id_tag_all(bmain, LIB_TAG_DOIT, false);
           BKE_reportf(reports,
                       RPT_WARNING,
                       "Could not create library override from data-block '%s', one of its parents "
                       "is not overridable ('%s')",
-                      id_root->name,
+                      id_root_reference->name,
-                      te->store_elem->id->name);
+                      id_current_hierarchy_root->name);
           return;
         }
-        te->store_elem->id->tag |= LIB_TAG_DOIT;
+        id_current_hierarchy_root->tag |= LIB_TAG_DOIT;
+        id_hierarchy_root_reference = id_current_hierarchy_root;
+      }
+
+      /* That case can happen when linked data is a complex mix involving several libraries and/or
+       * linked overrides. E.g. a mix of overrides from one library, and indirectly linked data
+       * from another library. Do not try to support such cases for now. */
+      if (!((id_hierarchy_root_reference->lib == id_root_reference->lib) ||
+            (!ID_IS_LINKED(id_hierarchy_root_reference) &&
+             ID_IS_OVERRIDE_LIBRARY_REAL(id_hierarchy_root_reference) &&
+             id_hierarchy_root_reference->override_library->reference->lib ==
+                 id_root_reference->lib))) {
+        BKE_main_id_tag_all(bmain, LIB_TAG_DOIT, false);
+        BKE_reportf(reports,
+                    RPT_WARNING,
+                    "Invalid hierarchy root ('%s') found, needed to create library override from "
+                    "data-block '%s'",
+                    id_hierarchy_root_reference->name,
+                    id_root_reference->name);
+        return;
       }
 
       success = BKE_lib_override_library_create(bmain,
                                                 CTX_data_scene(C),
                                                 CTX_data_view_layer(C),
                                                 nullptr,
-                                                id_root,
+                                                id_root_reference,
-                                                id_reference,
+                                                id_hierarchy_root_reference,
+                                                id_instance_hint,
                                                 nullptr);
     }
-    else if (ID_IS_OVERRIDABLE_LIBRARY(id_root)) {
+    else if (ID_IS_OVERRIDABLE_LIBRARY(id_root_reference)) {
-      success = BKE_lib_override_library_create_from_id(bmain, id_root, true) != nullptr;
+      success = BKE_lib_override_library_create_from_id(bmain, id_root_reference, true) != nullptr;
 
       /* Cleanup. */
       BKE_main_id_newptr_and_tag_clear(bmain);
@@ -845,14 +908,14 @@ static void id_override_library_create_fn(bContext *C,
     /* Remove the instance empty from this scene, the items now have an overridden collection
      * instead. */
     if (success && is_override_instancing_object) {
-      ED_object_base_free_and_unlink(bmain, scene, (Object *)id_reference);
+      ED_object_base_free_and_unlink(bmain, scene, (Object *)id_instance_hint);
     }
   }
   if (!success) {
     BKE_reportf(reports,
                 RPT_WARNING,
                 "Could not create library override from data-block '%s'",
-                id_root->name);
+                id_root_reference->name);
   }
 }
 

source/blender/makesrna/intern/rna_ID.c

@@ -696,7 +696,7 @@ static ID *rna_ID_override_create(ID *id, Main *bmain, bool remap_local_usages)
 }
 
 static ID *rna_ID_override_hierarchy_create(
-    ID *id, Main *bmain, Scene *scene, ViewLayer *view_layer, ID *id_reference)
+    ID *id, Main *bmain, Scene *scene, ViewLayer *view_layer, ID *id_instance_hint)
 {
   if (!ID_IS_OVERRIDABLE_LIBRARY(id)) {
     return NULL;
@@ -706,7 +706,7 @@ static ID *rna_ID_override_hierarchy_create(
 
   ID *id_root_override = NULL;
   BKE_lib_override_library_create(
-      bmain, scene, view_layer, NULL, id, id_reference, &id_root_override);
+      bmain, scene, view_layer, NULL, id, id, id_instance_hint, &id_root_override);
 
   WM_main_add_notifier(NC_ID | NA_ADDED, NULL);
   WM_main_add_notifier(NC_WM | ND_LIB_OVERRIDE_CHANGED, NULL);
@@ -2057,7 +2057,7 @@ static void rna_def_ID(BlenderRNA *brna)
                   "reference",
                   "ID",
                   "",
-                  "Another ID (usually an Object or Collection) used to decide where to "
+                  "Another ID (usually an Object or Collection) used as a hint to decide where to "
                   "instantiate the new overrides");
 
   func = RNA_def_function(srna, "override_template_create", "rna_ID_override_template_create");