Object Mode: Rework outline drawing.

This changes quite a few things.
- Outline is now per object.
- No more outline at object intersection (fix hairs problem).
- Simplify the code quite a bit.

We use a R16UI buffer to save one id per object outline. We convert this id
to color when detecting the outline.

Added textureGatherOffsets to the code but could not test on current hardware
so leaving it commented for now.

source/blender/draw/CMakeLists.txt

@@ -229,6 +229,7 @@ data_to_c_simple(modes/shaders/object_empty_image_vert.glsl SRC)
 data_to_c_simple(modes/shaders/object_outline_resolve_frag.glsl SRC)
 data_to_c_simple(modes/shaders/object_outline_expand_frag.glsl SRC)
 data_to_c_simple(modes/shaders/object_outline_detect_frag.glsl SRC)
+data_to_c_simple(modes/shaders/object_outline_prepass_frag.glsl SRC)
 data_to_c_simple(modes/shaders/object_grid_frag.glsl SRC)
 data_to_c_simple(modes/shaders/object_grid_vert.glsl SRC)
 data_to_c_simple(modes/shaders/object_lightprobe_grid_vert.glsl SRC)

source/blender/draw/modes/object_mode.c

@@ -40,6 +40,8 @@
 
 #include "BIF_gl.h"
 
+#include "BLI_string_utils.h"
+
 #include "BKE_anim.h"
 #include "BKE_camera.h"
 #include "BKE_curve.h"
@@ -69,6 +71,7 @@ extern struct GPUUniformBuffer *globals_ubo; /* draw_common.c */
 extern struct GPUTexture *globals_ramp; /* draw_common.c */
 extern GlobalsUboStorage ts;
 
+extern char datatoc_object_outline_prepass_frag_glsl[];
 extern char datatoc_object_outline_resolve_frag_glsl[];
 extern char datatoc_object_outline_detect_frag_glsl[];
 extern char datatoc_object_outline_expand_frag_glsl[];
@@ -223,6 +226,13 @@ typedef struct OBJECT_PrivateData {
 	DRWShadingGroup *points_select;
 	DRWShadingGroup *points_select_group;
 	DRWShadingGroup *points_transform;
+
+	/* Outlines id offset */
+	int id_ofs_active;
+	int id_ofs_active_group;
+	int id_ofs_select;
+	int id_ofs_select_group;
+	int id_ofs_transform;
 } OBJECT_PrivateData; /* Transient data */
 
 static struct {
@@ -232,6 +242,7 @@ static struct {
 	struct Gwn_VertFormat *empty_image_wire_format;
 
 	/* fullscreen shaders */
+	GPUShader *outline_prepass_sh;
 	GPUShader *outline_resolve_sh;
 	GPUShader *outline_resolve_aa_sh;
 	GPUShader *outline_detect_sh;
@@ -259,6 +270,7 @@ static struct {
 	bool draw_grid;
 	/* Temp buffer textures */
 	struct GPUTexture *outlines_depth_tx;
+	struct GPUTexture *outlines_id_tx;
 	struct GPUTexture *outlines_color_tx;
 	struct GPUTexture *outlines_blur_tx;
 } e_data = {NULL}; /* Engine data */
@@ -288,14 +300,20 @@ static void OBJECT_engine_init(void *vedata)
 	if (DRW_state_is_fbo()) {
 		e_data.outlines_depth_tx = DRW_texture_pool_query_2D(size[0], size[1], DRW_TEX_DEPTH_24,
 		                                                     &draw_engine_object_type);
-		e_data.outlines_color_tx = DRW_texture_pool_query_2D(size[0], size[1], DRW_TEX_RGBA_8,
+		/* XXX TODO DRW_TEX_R_16U can overflow, it would cause no harm
-		                                                     &draw_engine_object_type);
+		 * (only bad colored or missing outlines) but we should
+		 * use 32bits only if the scene have that many objects */
+		e_data.outlines_id_tx = DRW_texture_pool_query_2D(size[0], size[1], DRW_TEX_R_16U,
+		                                                  &draw_engine_object_type);
 
 		GPU_framebuffer_ensure_config(&fbl->outlines_fb, {
 			GPU_ATTACHMENT_TEXTURE(e_data.outlines_depth_tx),
-			GPU_ATTACHMENT_TEXTURE(e_data.outlines_color_tx)
+			GPU_ATTACHMENT_TEXTURE(e_data.outlines_id_tx)
 		});
 
+		e_data.outlines_color_tx = DRW_texture_pool_query_2D(size[0], size[1], DRW_TEX_RGBA_8,
+		                                                     &draw_engine_object_type);
+
 		GPU_framebuffer_ensure_config(&fbl->expand_fb, {
 			GPU_ATTACHMENT_NONE,
 			GPU_ATTACHMENT_TEXTURE(e_data.outlines_color_tx)
@@ -310,66 +328,58 @@ static void OBJECT_engine_init(void *vedata)
 		});
 	}
 
+	/* Shaders */
 	if (!e_data.outline_resolve_sh) {
-		e_data.outline_resolve_sh = DRW_shader_create_fullscreen(datatoc_object_outline_resolve_frag_glsl, NULL);
+		/* Outline */
-	}
+		e_data.outline_prepass_sh = DRW_shader_create_3D(datatoc_object_outline_prepass_frag_glsl, NULL);
+
+		e_data.outline_resolve_sh = DRW_shader_create_fullscreen(datatoc_object_outline_resolve_frag_glsl, NULL);
 
-	if (!e_data.outline_resolve_aa_sh) {
 		e_data.outline_resolve_aa_sh = DRW_shader_create_with_lib(
 		            datatoc_common_fullscreen_vert_glsl, NULL,
 		            datatoc_object_outline_resolve_frag_glsl,
 		            datatoc_common_fxaa_lib_glsl,
 		            "#define FXAA_ALPHA\n"
 		            "#define USE_FXAA\n");
-	}
 
-	if (!e_data.outline_detect_sh) {
+		e_data.outline_detect_sh = DRW_shader_create_with_lib(
-		e_data.outline_detect_sh = DRW_shader_create_fullscreen(datatoc_object_outline_detect_frag_glsl, NULL);
+		            datatoc_common_fullscreen_vert_glsl, NULL,
-	}
+		            datatoc_object_outline_detect_frag_glsl,
+		            datatoc_common_globals_lib_glsl,
+		            NULL);
 
-	if (!e_data.outline_fade_sh) {
 		e_data.outline_fade_sh = DRW_shader_create_fullscreen(datatoc_object_outline_expand_frag_glsl, NULL);
-	}
 
-	if (!e_data.object_empty_image_sh) {
+		/* Empty images */
 		e_data.object_empty_image_sh = DRW_shader_create_with_lib(
 		           datatoc_object_empty_image_vert_glsl, NULL,
 		           datatoc_object_empty_image_frag_glsl,
 		           datatoc_common_globals_lib_glsl, NULL);
-	}
 
-	if (!e_data.object_empty_image_wire_sh) {
 		e_data.object_empty_image_wire_sh = DRW_shader_create_with_lib(
 		           datatoc_object_empty_image_vert_glsl, NULL,
 		           datatoc_object_empty_image_frag_glsl,
 		           datatoc_common_globals_lib_glsl,
 		           "#define USE_WIRE\n");
-	}
 
-	if (!e_data.grid_sh) {
+		/* Grid */
 		e_data.grid_sh = DRW_shader_create_with_lib(
 		        datatoc_object_grid_vert_glsl, NULL,
 		        datatoc_object_grid_frag_glsl,
 		        datatoc_common_globals_lib_glsl, NULL);
-	}
 
-	if (!e_data.part_prim_sh) {
+		/* Particles */
 		e_data.part_prim_sh = DRW_shader_create(
 		        datatoc_object_particle_prim_vert_glsl, NULL, datatoc_gpu_shader_flat_color_frag_glsl, NULL);
-	}
 
-	if (!e_data.part_axis_sh) {
 		e_data.part_axis_sh = DRW_shader_create(
 		        datatoc_object_particle_prim_vert_glsl, NULL, datatoc_gpu_shader_flat_color_frag_glsl,
 		        "#define USE_AXIS\n");
-	}
 
-	if (!e_data.part_dot_sh) {
 		e_data.part_dot_sh = DRW_shader_create(
 		        datatoc_object_particle_dot_vert_glsl, NULL, datatoc_object_particle_dot_frag_glsl, NULL);
-	}
 
-	if (!e_data.lightprobe_grid_sh) {
+		/* Lightprobes */
 		e_data.lightprobe_grid_sh = DRW_shader_create(
 		        datatoc_object_lightprobe_grid_vert_glsl, NULL, datatoc_gpu_shader_uniform_color_frag_glsl, NULL);
 	}
@@ -558,6 +568,7 @@ static void OBJECT_engine_free(void)
 	MEM_SAFE_FREE(e_data.particle_format);
 	MEM_SAFE_FREE(e_data.empty_image_format);
 	MEM_SAFE_FREE(e_data.empty_image_wire_format);
+	DRW_SHADER_FREE_SAFE(e_data.outline_prepass_sh);
 	DRW_SHADER_FREE_SAFE(e_data.outline_resolve_sh);
 	DRW_SHADER_FREE_SAFE(e_data.outline_resolve_aa_sh);
 	DRW_SHADER_FREE_SAFE(e_data.outline_detect_sh);
@@ -571,10 +582,10 @@ static void OBJECT_engine_free(void)
 	DRW_SHADER_FREE_SAFE(e_data.lightprobe_grid_sh);
 }
 
-static DRWShadingGroup *shgroup_outline(DRWPass *pass, const float col[4], GPUShader *sh)
+static DRWShadingGroup *shgroup_outline(DRWPass *pass, const int *ofs, GPUShader *sh)
 {
 	DRWShadingGroup *grp = DRW_shgroup_create(sh, pass);
-	DRW_shgroup_uniform_vec4(grp, "color", col, 1);
+	DRW_shgroup_uniform_int(grp, "baseId", ofs, 1);
 
 	return grp;
 }
@@ -602,12 +613,19 @@ static DRWShadingGroup *shgroup_theme_id_to_outline_or(
 {
 	switch (theme_id) {
 		case TH_ACTIVE:
+			stl->g_data->id_ofs_active++;
 			return stl->g_data->outlines_active;
 		case TH_SELECT:
+			stl->g_data->id_ofs_select++;
 			return stl->g_data->outlines_select;
-		case TH_GROUP_ACTIVE:
+		case TH_GROUP:
+			stl->g_data->id_ofs_select_group++;
 			return stl->g_data->outlines_select_group;
+		case TH_GROUP_ACTIVE:
+			stl->g_data->id_ofs_active_group++;
+			return stl->g_data->outlines_active_group;
 		case TH_TRANSFORM:
+			stl->g_data->id_ofs_transform++;
 			return stl->g_data->outlines_transform;
 		default:
 			return fallback;
@@ -622,8 +640,10 @@ static DRWShadingGroup *shgroup_theme_id_to_wire_or(
 			return stl->g_data->wire_active;
 		case TH_SELECT:
 			return stl->g_data->wire_select;
-		case TH_GROUP_ACTIVE:
+		case TH_GROUP:
 			return stl->g_data->wire_select_group;
+		case TH_GROUP_ACTIVE:
+			return stl->g_data->wire_active_group;
 		case TH_TRANSFORM:
 			return stl->g_data->wire_transform;
 		default:
@@ -639,8 +659,10 @@ static DRWShadingGroup *shgroup_theme_id_to_point_or(
 			return stl->g_data->points_active;
 		case TH_SELECT:
 			return stl->g_data->points_select;
-		case TH_GROUP_ACTIVE:
+		case TH_GROUP:
 			return stl->g_data->points_select_group;
+		case TH_GROUP_ACTIVE:
+			return stl->g_data->points_active_group;
 		case TH_TRANSFORM:
 			return stl->g_data->points_transform;
 		default:
@@ -786,18 +808,24 @@ static void OBJECT_cache_init(void *vedata)
 		DRWState state = DRW_STATE_WRITE_COLOR | DRW_STATE_WRITE_DEPTH | DRW_STATE_DEPTH_LESS | DRW_STATE_WIRE;
 		psl->outlines = DRW_pass_create("Outlines Depth Pass", state);
 
-		GPUShader *sh = GPU_shader_get_builtin_shader(GPU_SHADER_3D_UNIFORM_COLOR);
+		GPUShader *sh = e_data.outline_prepass_sh;
 
 		/* Select */
-		stl->g_data->outlines_select = shgroup_outline(psl->outlines, ts.colorSelect, sh);
+		stl->g_data->outlines_select = shgroup_outline(psl->outlines, &stl->g_data->id_ofs_select, sh);
-		stl->g_data->outlines_select_group = shgroup_outline(psl->outlines, ts.colorGroupActive, sh);
+		stl->g_data->outlines_select_group = shgroup_outline(psl->outlines, &stl->g_data->id_ofs_select_group, sh);
 
 		/* Transform */
-		stl->g_data->outlines_transform = shgroup_outline(psl->outlines, ts.colorTransform, sh);
+		stl->g_data->outlines_transform = shgroup_outline(psl->outlines, &stl->g_data->id_ofs_transform, sh);
 
 		/* Active */
-		stl->g_data->outlines_active = shgroup_outline(psl->outlines, ts.colorActive, sh);
+		stl->g_data->outlines_active = shgroup_outline(psl->outlines, &stl->g_data->id_ofs_active, sh);
-		stl->g_data->outlines_active_group = shgroup_outline(psl->outlines, ts.colorGroupActive, sh);
+		stl->g_data->outlines_active_group = shgroup_outline(psl->outlines, &stl->g_data->id_ofs_active_group, sh);
+
+		stl->g_data->id_ofs_select = 0;
+		stl->g_data->id_ofs_select_group = 0;
+		stl->g_data->id_ofs_active = 0;
+		stl->g_data->id_ofs_active_group = 0;
+		stl->g_data->id_ofs_transform = 0;
 	}
 
 	{
@@ -822,10 +850,12 @@ static void OBJECT_cache_init(void *vedata)
 		psl->outlines_search = DRW_pass_create("Outlines Detect Pass", state);
 
 		DRWShadingGroup *grp = DRW_shgroup_create(e_data.outline_detect_sh, psl->outlines_search);
-		DRW_shgroup_uniform_texture_ref(grp, "outlineColor", &e_data.outlines_color_tx);
+		DRW_shgroup_uniform_texture_ref(grp, "outlineId", &e_data.outlines_id_tx);
 		DRW_shgroup_uniform_texture_ref(grp, "outlineDepth", &e_data.outlines_depth_tx);
 		DRW_shgroup_uniform_texture_ref(grp, "sceneDepth", &dtxl->depth);
+		DRW_shgroup_uniform_block(grp, "globalsBlock", globals_ubo);
 		DRW_shgroup_uniform_float(grp, "alphaOcclu", &alphaOcclu, 1);
+		DRW_shgroup_uniform_int(grp, "idOffsets", &stl->g_data->id_ofs_active, 5);
 		DRW_shgroup_call_add(grp, quad, NULL);
 
 		psl->outlines_expand = DRW_pass_create("Outlines Expand Pass", state);
@@ -2038,6 +2068,7 @@ static void OBJECT_draw_scene(void *vedata)
 	OBJECT_PassList *psl = ((OBJECT_Data *)vedata)->psl;
 	OBJECT_StorageList *stl = ((OBJECT_Data *)vedata)->stl;
 	OBJECT_FramebufferList *fbl = ((OBJECT_Data *)vedata)->fbl;
+	OBJECT_PrivateData *g_data = stl->g_data;
 	DefaultFramebufferList *dfbl = DRW_viewport_framebuffer_list_get();
 	DefaultTextureList *dtxl = DRW_viewport_texture_list_get();
 
@@ -2045,7 +2076,18 @@ static void OBJECT_draw_scene(void *vedata)
 
 	if (DRW_state_is_fbo()) {
 		DRW_stats_group_start("Outlines");
-		
+
+		int id_ct_select = g_data->id_ofs_select;
+		int id_ct_select_group = g_data->id_ofs_select_group;
+		int id_ct_active = g_data->id_ofs_active;
+		int id_ct_active_group = g_data->id_ofs_active_group;
+
+		g_data->id_ofs_active = 1;
+		g_data->id_ofs_active_group = g_data->id_ofs_active + id_ct_active + 1;
+		g_data->id_ofs_select = g_data->id_ofs_active_group + id_ct_active_group + 1;
+		g_data->id_ofs_select_group = g_data->id_ofs_select + id_ct_select + 1;
+		g_data->id_ofs_transform = g_data->id_ofs_select_group + id_ct_select_group + 1;
+
 		/* Render filled polygon on a separate framebuffer */
 		GPU_framebuffer_bind(fbl->outlines_fb);
 		GPU_framebuffer_clear_color_depth(fbl->outlines_fb, clearcol, 1.0f);

source/blender/draw/modes/shaders/object_outline_detect_frag.glsl

@@ -3,77 +3,67 @@ in vec4 uvcoordsvar;
 
 out vec4 FragColor;
 
-uniform sampler2D outlineColor;
+uniform usampler2D outlineId;
 uniform sampler2D outlineDepth;
 uniform sampler2D sceneDepth;
 
+uniform int idOffsets[5];
+
 uniform float alphaOcclu;
 uniform vec2 viewportSize;
 
-void search_outline(ivec2 uv, vec4 ref_col, inout bool ref_occlu, inout bool outline)
+vec4 convert_id_to_color(int id)
 {
-	if (!outline) {
+	if (id == 0) {
-		vec4 color = texelFetch(outlineColor, uv, 0).rgba;
+		return vec4(0.0);
-		if (color != ref_col) {
+	}
-			outline = true;
+	if (id < idOffsets[1]) {
-		}
+		return colorActive;
-		else {
+	}
-			float depth = texelFetch(outlineDepth, uv, 0).r;
+	else if (id < idOffsets[2]) {
-			float scene_depth = texelFetch(sceneDepth, uv, 0).r;
+		return colorGroupActive;
-			bool occlu = (depth > scene_depth);
+	}
-
+	else if (id < idOffsets[3]) {
-			if (occlu != ref_occlu && !ref_occlu) {
+		return colorSelect;
-				outline = true;
+	}
-			}
+	else if (id < idOffsets[4]) {
-		}
+		return colorGroup;
+	}
+	else {
+		return colorTransform;
 	}
 }
 
+const ivec2 ofs[4] = ivec2[4](
+    ivec2( 1,  0), ivec2( 0,  1),
+    ivec2(-1,  0), ivec2( 0, -1)
+);
+
 void main()
 {
-	ivec2 uv = ivec2(gl_FragCoord.xy);
+	ivec2 texel = ivec2(gl_FragCoord.xy);
+	vec2 uv = gl_FragCoord.xy / vec2(textureSize(outlineId, 0).xy);
 
-	vec4 color[4];
+	uvec4 id;
-	/* Idea : Use a 16bit ID to identify the color
+	uint ref_id = texelFetch(outlineId, texel, 0).r;
-	 * and store the colors in a UBO. And fetch all ids
+#if 0 /* commented out until being tested */
-	 * for discontinuity check with one textureGather \o/ */
+	id = textureGatherOffsets(outlineId, uv, ofs);
-	vec4 ref_col = texelFetch(outlineColor, uv, 0).rgba;
-	color[0] = texelFetchOffset(outlineColor, uv, 0, ivec2( 1,  0)).rgba;
-	color[1] = texelFetchOffset(outlineColor, uv, 0, ivec2( 0,  1)).rgba;
-	color[2] = texelFetchOffset(outlineColor, uv, 0, ivec2(-1,  0)).rgba;
-	color[3] = texelFetchOffset(outlineColor, uv, 0, ivec2( 0, -1)).rgba;
-
-	/* TODO GATHER */
-	vec4 depths;
-	float depth = texelFetch(outlineDepth, uv, 0).r;
-	depths.x = texelFetchOffset(outlineDepth, uv, 0, ivec2( 1,  0)).r;
-	depths.y = texelFetchOffset(outlineDepth, uv, 0, ivec2( 0,  1)).r;
-	depths.z = texelFetchOffset(outlineDepth, uv, 0, ivec2(-1,  0)).r;
-	depths.w = texelFetchOffset(outlineDepth, uv, 0, ivec2( 0, -1)).r;
-
-	vec4 scene_depths;
-	float scene_depth = texelFetch(sceneDepth, uv, 0).r;
-	scene_depths.x = texelFetchOffset(sceneDepth, uv, 0, ivec2( 1,  0)).r;
-	scene_depths.y = texelFetchOffset(sceneDepth, uv, 0, ivec2( 0,  1)).r;
-	scene_depths.z = texelFetchOffset(sceneDepth, uv, 0, ivec2(-1,  0)).r;
-	scene_depths.w = texelFetchOffset(sceneDepth, uv, 0, ivec2( 0, -1)).r;
-
-	bool ref_occlu = (depth > scene_depth);
-	bool outline = false;
-
-#if 1
-	bvec4 occlu = (!ref_occlu) ? notEqual(greaterThan(depths, scene_depths), bvec4(ref_occlu)) : bvec4(false);
-	outline = (!outline) ? (color[0] != ref_col) || occlu.x : true;
-	outline = (!outline) ? (color[1] != ref_col) || occlu.y : true;
-	outline = (!outline) ? (color[2] != ref_col) || occlu.z : true;
-	outline = (!outline) ? (color[3] != ref_col) || occlu.w : true;
 #else
-	search_outline(uv + ivec2( 1,  0), ref_col, ref_occlu, outline);
+	id.x = texelFetchOffset(outlineId, texel, 0, ofs[0]).r;
-	search_outline(uv + ivec2( 0,  1), ref_col, ref_occlu, outline);
+	id.y = texelFetchOffset(outlineId, texel, 0, ofs[1]).r;
-	search_outline(uv + ivec2(-1,  0), ref_col, ref_occlu, outline);
+	id.z = texelFetchOffset(outlineId, texel, 0, ofs[2]).r;
-	search_outline(uv + ivec2( 0, -1), ref_col, ref_occlu, outline);
+	id.w = texelFetchOffset(outlineId, texel, 0, ofs[3]).r;
 #endif
 
-	FragColor = ref_col;
+	float ref_depth = texelFetch(outlineDepth, texel, 0).r;
-	FragColor.a *= (outline) ? (ref_occlu) ? alphaOcclu : 1.0 : 0.0;
+	float scene_depth = texelFetch(sceneDepth, texel, 0).r;
+
+	/* Avoid bad cases of zfighting for occlusion only. */
+	const float epsilon = 3.0 / 8388608.0;
+	bool occluded = (ref_depth > scene_depth + epsilon);
+	bool outline = any(notEqual(id, uvec4(ref_id)));
+
+	FragColor = convert_id_to_color(int(ref_id));
+	FragColor.a *= (occluded) ? alphaOcclu : 1.0;
+	FragColor.a = (outline) ? FragColor.a : 0.0;
 }

source/blender/draw/modes/shaders/object_outline_prepass_frag.glsl

@@ -0,0 +1,10 @@
+uniform int callId;
+uniform int baseId;
+
+/* using uint because 16bit uint can contain more ids than int. */
+out uint outId;
+
+void main()
+{
+	outId = uint(baseId + callId);
+}