Fix T69809, T69810: sculpt gone or crashing after render

Make a distinction between flush sculpt changes for rendering, and forcing
sculpt data structures to be rebuilt after mesh changes. Also don't use PBVH
for renders.

source/blender/blenkernel/BKE_multires.h

@@ -53,8 +53,8 @@ void multires_mark_as_modified(struct Depsgraph *depsgraph,
                                struct Object *object,
                                enum MultiresModifiedFlags flags);
 
-void multires_force_update(struct Object *ob);
-void multires_force_render_update(struct Object *ob);
+void multires_flush_sculpt_updates(struct Object *ob);
+void multires_force_sculpt_rebuild(struct Object *ob);
 void multires_force_external_reload(struct Object *ob);
 
 /* internal, only called in subsurf_ccg.c */

source/blender/blenkernel/intern/library_remap.c

@@ -389,7 +389,7 @@ static void libblock_remap_data_postprocess_obdata_relink(Main *bmain, Object *o
   if (ob->data == new_id) {
     switch (GS(new_id->name)) {
       case ID_ME:
-        multires_force_update(ob);
+        multires_force_sculpt_rebuild(ob);
         break;
       case ID_CU:
         BKE_curve_type_test(ob);

source/blender/blenkernel/intern/mesh.c

@@ -1080,12 +1080,12 @@ void BKE_mesh_assign_object(Main *bmain, Object *ob, Mesh *me)
 {
   Mesh *old = NULL;
 
-  multires_force_update(ob);
-
   if (ob == NULL) {
     return;
   }
 
+  multires_force_sculpt_rebuild(ob);
+
   if (ob->type == OB_MESH) {
     old = ob->data;
     if (old) {

source/blender/blenkernel/intern/multires.c

@@ -406,24 +406,24 @@ void multires_mark_as_modified(Depsgraph *depsgraph, Object *object, MultiresMod
   multires_ccg_mark_as_modified(subdiv_ccg, flags);
 }
 
-void multires_force_update(Object *ob)
+void multires_flush_sculpt_updates(Object *ob)
 {
-  if (ob == NULL) {
-    return;
-  }
-  SculptSession *sculpt_session = ob->sculpt;
-  if (sculpt_session != NULL && sculpt_session->pbvh != NULL) {
-    PBVH *pbvh = sculpt_session->pbvh;
-    if (BKE_pbvh_type(pbvh) == PBVH_GRIDS) {
+  if (ob && ob->sculpt && ob->sculpt->pbvh != NULL) {
+    SculptSession *sculpt_session = ob->sculpt;
+    if (BKE_pbvh_type(sculpt_session->pbvh) == PBVH_GRIDS) {
       Mesh *mesh = ob->data;
       multiresModifier_reshapeFromCCG(
           sculpt_session->multires->totlvl, mesh, sculpt_session->subdiv_ccg);
     }
-    else {
-      /* NOTE: Disabled for until OpenSubdiv is enabled by default. */
-      // BLI_assert(!"multires_force_update is used on non-grids PBVH");
-    }
-    BKE_pbvh_free(pbvh);
+  }
+}
+
+void multires_force_sculpt_rebuild(Object *ob)
+{
+  multires_flush_sculpt_updates(ob);
+
+  if (ob && ob->sculpt && ob->sculpt->pbvh != NULL) {
+    BKE_pbvh_free(ob->sculpt->pbvh);
     ob->sculpt->pbvh = NULL;
   }
 }
@@ -433,14 +433,7 @@ void multires_force_external_reload(Object *ob)
   Mesh *me = BKE_mesh_from_object(ob);
 
   CustomData_external_reload(&me->ldata, &me->id, CD_MASK_MDISPS, me->totloop);
-  multires_force_update(ob);
-}
-
-void multires_force_render_update(Object *ob)
-{
-  if (ob && (ob->mode & OB_MODE_SCULPT) && modifiers_findByType(ob, eModifierType_Multires)) {
-    multires_force_update(ob);
-  }
+  multires_force_sculpt_rebuild(ob);
 }
 
 /* reset the multires levels to match the number of mdisps */
@@ -624,7 +617,7 @@ static void multires_del_higher(MultiresModifierData *mmd, Object *ob, int lvl)
   mdisps = CustomData_get_layer(&me->ldata, CD_MDISPS);
   gpm = CustomData_get_layer(&me->ldata, CD_GRID_PAINT_MASK);
 
-  multires_force_update(ob);
+  multires_force_sculpt_rebuild(ob);
 
   if (mdisps && levels > 0) {
     if (lvl > 0) {
@@ -689,7 +682,7 @@ void multiresModifier_del_levels(MultiresModifierData *mmd,
   CustomData_external_read(&me->ldata, &me->id, CD_MASK_MDISPS, me->totloop);
   mdisps = CustomData_get_layer(&me->ldata, CD_MDISPS);
 
-  multires_force_update(ob);
+  multires_force_sculpt_rebuild(ob);
 
   if (mdisps && levels > 0 && direction == 1) {
     multires_del_higher(mmd, ob, lvl);
@@ -781,7 +774,7 @@ void multiresModifier_base_apply(MultiresModifierData *mmd, Scene *scene, Object
   float(*origco)[3];
   int i, j, k, offset, totlvl;
 
-  multires_force_update(ob);
+  multires_force_sculpt_rebuild(ob);
 
   me = BKE_mesh_from_object(ob);
   totlvl = mmd->totlvl;
@@ -928,7 +921,7 @@ static void multires_subdivide(
 
   BLI_assert(totlvl > lvl);
 
-  multires_force_update(ob);
+  multires_force_sculpt_rebuild(ob);
 
   mdisps = CustomData_get_layer(&me->ldata, CD_MDISPS);
   if (!mdisps) {

source/blender/draw/engines/basic/basic_engine.c

@@ -162,7 +162,8 @@ static void basic_cache_populate(void *vedata, Object *ob)
     }
   }
 
-  const bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d);
+  const bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d) &&
+                               !DRW_state_is_image_render();
   const bool do_cull = (draw_ctx->v3d &&
                         (draw_ctx->v3d->shading.flag & V3D_SHADING_BACKFACE_CULLING));
   DRWShadingGroup *shgrp = (do_cull) ? stl->g_data->depth_shgrp_cull : stl->g_data->depth_shgrp;

source/blender/draw/engines/eevee/eevee_materials.c

@@ -1454,7 +1454,8 @@ void EEVEE_materials_cache_populate(EEVEE_Data *vedata,
   Scene *scene = draw_ctx->scene;
   GHash *material_hash = stl->g_data->material_hash;
 
-  bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d);
+  bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d) &&
+                         !DRW_state_is_image_render();
 
   /* First get materials for this mesh. */
   if (ELEM(ob->type, OB_MESH, OB_CURVE, OB_SURF, OB_FONT, OB_MBALL)) {

source/blender/draw/engines/workbench/workbench_deferred.c

@@ -945,7 +945,8 @@ static void workbench_cache_populate_texture_paint_mode(WORKBENCH_Data *vedata,
   const DRWContextState *draw_ctx = DRW_context_state_get();
 
   Scene *scene = draw_ctx->scene;
-  const bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d);
+  const bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d) &&
+                               !DRW_state_is_image_render();
   WORKBENCH_MaterialData *material;
 
   /* Force workbench to render active object textured when in texture paint mode */

source/blender/draw/engines/workbench/workbench_forward.c

@@ -560,7 +560,8 @@ static void workbench_forward_cache_populate_texture_paint_mode(WORKBENCH_Data *
   const DRWContextState *draw_ctx = DRW_context_state_get();
 
   Scene *scene = draw_ctx->scene;
-  const bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d);
+  const bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d) &&
+                               !DRW_state_is_image_render();
   WORKBENCH_MaterialData *material;
 
   /* Force workbench to render active object textured when in texture paint mode */
@@ -635,7 +636,8 @@ void workbench_forward_cache_populate(WORKBENCH_Data *vedata, Object *ob)
 
   WORKBENCH_MaterialData *material;
   if (ELEM(ob->type, OB_MESH, OB_CURVE, OB_SURF, OB_FONT, OB_MBALL)) {
-    const bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d);
+    const bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d) &&
+                                 !DRW_state_is_image_render();
     const int materials_len = MAX2(1, ob->totcol);
     const Mesh *me = (ob->type == OB_MESH) ? ob->data : NULL;
     const bool use_texture_paint_drawing = !(DRW_state_is_image_render() &&

source/blender/draw/modes/overlay_mode.c

@@ -394,7 +394,8 @@ static void overlay_cache_populate(void *vedata, Object *ob)
     if ((!pd->show_overlays) ||
         (((ob != draw_ctx->object_edit) && !is_edit_mode) || has_edit_mesh_cage) ||
         ob->type != OB_MESH) {
-      const bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d);
+      const bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d) &&
+                                   !DRW_state_is_image_render();
       const bool all_wires = (ob->dtx & OB_DRAW_ALL_EDGES);
       const bool is_wire = (ob->dt < OB_SOLID);
       const bool is_xray = (ob->dtx & OB_DRAWXRAY);

source/blender/editors/object/object_bake.c

@@ -371,7 +371,7 @@ static int multiresbake_image_exec_locked(bContext *C, wmOperator *op)
 
     ob = base->object;
 
-    multires_force_update(ob);
+    multires_flush_sculpt_updates(ob);
 
     /* copy data stored in job descriptor */
     bkr.scene = scene;
@@ -435,7 +435,7 @@ static void init_multiresbake_job(bContext *C, MultiresBakeJob *bkj)
 
     ob = base->object;
 
-    multires_force_update(ob);
+    multires_flush_sculpt_updates(ob);
 
     data = MEM_callocN(sizeof(MultiresBakerJobData), "multiresBaker derivedMesh_data");
 

source/blender/editors/object/object_modifier.c

@@ -677,7 +677,7 @@ static int modifier_apply_obdata(
 
     /* Multires: ensure that recent sculpting is applied */
     if (md_eval->type == eModifierType_Multires) {
-      multires_force_update(ob);
+      multires_force_sculpt_rebuild(ob);
     }
 
     if (mmd && mmd->totlvl && mti->type == eModifierTypeType_OnlyDeform) {

source/blender/editors/sculpt_paint/sculpt.c

@@ -1099,7 +1099,7 @@ static bool sculpt_brush_test_cyl(SculptBrushTest *test,
 static bool sculpt_automasking_enabled(SculptSession *ss, const Brush *br)
 {
   // REMOVE WITH PBVH_GRIDS
-  if (BKE_pbvh_type(ss->pbvh) == PBVH_GRIDS) {
+  if (ss->pbvh && BKE_pbvh_type(ss->pbvh) == PBVH_GRIDS) {
     return false;
   }
 
@@ -7529,13 +7529,11 @@ void ED_object_sculptmode_exit_ex(Main *bmain, Depsgraph *depsgraph, Scene *scen
   const int mode_flag = OB_MODE_SCULPT;
   Mesh *me = BKE_mesh_from_object(ob);
 
-  MultiresModifierData *mmd = BKE_sculpt_multires_active(scene, ob);
-  if (mmd) {
-    multires_force_update(ob);
-  }
+  multires_flush_sculpt_updates(ob);
 
   /* Not needed for now. */
 #if 0
+  MultiresModifierData *mmd = BKE_sculpt_multires_active(scene, ob);
   const int flush_recalc = ed_object_sculptmode_flush_recalc_flag(scene, ob, mmd);
 #endif
 

source/blender/editors/util/ed_util.c

@@ -245,7 +245,7 @@ bool ED_editors_flush_edits(Main *bmain, bool for_render)
        * may cause a flush on saving: T53986. */
       if ((ob->sculpt && ob->sculpt->cache) == 0) {
         /* flush multires changes (for sculpt) */
-        multires_force_update(ob);
+        multires_flush_sculpt_updates(ob);
         has_edited = true;
 
         if (for_render) {

source/blender/makesrna/intern/rna_modifier.c

@@ -915,7 +915,7 @@ static void rna_MultiresModifier_type_set(PointerRNA *ptr, int value)
   Object *ob = (Object *)ptr->owner_id;
   MultiresModifierData *mmd = (MultiresModifierData *)ptr->data;
 
-  multires_force_update(ob);
+  multires_force_sculpt_rebuild(ob);
   mmd->simple = value;
 }