tmaq
/
tornavis
1
0
Fork 0
Code Issues 2 Pull Requests 29 Packages Projects 24 Releases Wiki Activity

Compositor: Use RBF Interpolation in Keying Screen node 

This patch changes the interpolation algorithm utilized by the Keying
Screen node to a Gaussian Radial Basis Function Interpolation. This is
proposed because the current Voronoi triangulation based interpolation
has the following properties:

- Not temporally stable since the triangulation can abruptly change as
  tracking markers change position.
- Not smooth in the mathematical sense, which is also readily visible in
  the artists sense.
- Computationally expensive due to the triangulation and naive
  rasterization algorithm.

On the other hand, the RBF interpolation method is temporally stable and
continuous, smooth and infinitely differentiable, and relatively simple
to compute assuming low number of markers, which is typically the case
for keying screen objects.

This breaks backward compatibility, but the keying screen is only used
as a secondary input for keying in typical compositor setups, so one
should expect minimal difference in outputs.

Pull Request: https://projects.blender.org/blender/blender/pulls/112480
This commit is contained in:
Omar Emara 2023-10-04 07:07:04 +02:00 committed by Omar Emara
parent 0b302d2139
commit 75c947a467
6 changed files with 95 additions and 246 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
source/blender/compositor/nodes/COM_KeyingScreenNode.cc
View File

@ -25,6 +25,7 @@ void KeyingScreenNode::convert_to_operations(NodeConverter &converter,
KeyingScreenOperation *operation = new KeyingScreenOperation();
operation->set_movie_clip(clip);
operation->set_tracking_object(keyingscreen_data->tracking_object);
operation->set_smoothness(keyingscreen_data->smoothness);
operation->set_framenumber(context.get_framenumber());
converter.add_operation(operation);

296
source/blender/compositor/operations/COM_KeyingScreenOperation.cc
View File

@ -6,8 +6,11 @@
#include "DNA_defaults.h"
#include "BLI_array.hh"
#include "BLI_math_color.h"
#include "BLI_math_geom.h"
#include "BLI_math_vector.hh"
#include "BLI_math_vector_types.hh"
#include "BKE_movieclip.h"
#include "BKE_tracking.h"
@ -24,58 +27,32 @@ KeyingScreenOperation::KeyingScreenOperation()
framenumber_ = 0;
tracking_object_[0] = 0;
flags_.complex = true;
cached_triangulation_ = nullptr;
cached_marker_points_ = nullptr;
}
void KeyingScreenOperation::init_execution()
{
init_mutex();
if (execution_model_ == eExecutionModel::FullFrame) {
BLI_assert(cached_triangulation_ == nullptr);
BLI_assert(cached_marker_points_ == nullptr);
if (movie_clip_) {
cached_triangulation_ = build_voronoi_triangulation();
cached_marker_points_ = compute_marker_points();
}
}
else {
cached_triangulation_ = nullptr;
cached_marker_points_ = nullptr;
}
}
void KeyingScreenOperation::deinit_execution()
{
if (cached_triangulation_) {
TriangulationData *triangulation = cached_triangulation_;
if (triangulation->triangulated_points) {
MEM_freeN(triangulation->triangulated_points);
}
if (triangulation->triangles) {
MEM_freeN(triangulation->triangles);
}
if (triangulation->triangles_AABB) {
MEM_freeN(triangulation->triangles_AABB);
}
MEM_freeN(cached_triangulation_);
cached_triangulation_ = nullptr;
}
delete cached_marker_points_;
cached_marker_points_ = nullptr;
}
KeyingScreenOperation::TriangulationData *KeyingScreenOperation::build_voronoi_triangulation()
Array<KeyingScreenOperation::MarkerPoint> *KeyingScreenOperation::compute_marker_points()
{
MovieClipUser user = *DNA_struct_default_get(MovieClipUser);
TriangulationData *triangulation;
MovieTracking *tracking = &movie_clip_->tracking;
ImBuf *ibuf;
ListBase edges = {nullptr, nullptr};
int sites_total;
int i;
int width = this->get_width();
int height = this->get_height();
int clip_frame = BKE_movieclip_remap_scene_to_clip_frame(movie_clip_, framenumber_);
const MovieTrackingObject *tracking_object = nullptr;
if (tracking_object_[0]) {
@ -87,10 +64,12 @@ KeyingScreenOperation::TriangulationData *KeyingScreenOperation::build_voronoi_t
else {
tracking_object = BKE_tracking_object_get_active(tracking);
}
BLI_assert(tracking_object != 0);
BLI_assert(tracking_object != nullptr);
int clip_frame = BKE_movieclip_remap_scene_to_clip_frame(movie_clip_, framenumber_);
/* count sites */
sites_total = 0;
int sites_total = 0;
LISTBASE_FOREACH (MovieTrackingTrack *, track, &tracking_object->tracks) {
const MovieTrackingMarker *marker = BKE_tracking_marker_get(track, clip_frame);
@ -112,17 +91,16 @@ KeyingScreenOperation::TriangulationData *KeyingScreenOperation::build_voronoi_t
return nullptr;
}
MovieClipUser user = *DNA_struct_default_get(MovieClipUser);
BKE_movieclip_user_set_frame(&user, clip_frame);
ibuf = BKE_movieclip_get_ibuf(movie_clip_, &user);
ImBuf *ibuf = BKE_movieclip_get_ibuf(movie_clip_, &user);
if (!ibuf) {
return nullptr;
}
triangulation = MEM_cnew<TriangulationData>("keying screen triangulation data");
Array<MarkerPoint> *marker_points = new Array<MarkerPoint>(sites_total);
VoronoiSite *sites = (VoronoiSite *)MEM_callocN(sizeof(VoronoiSite) * sites_total,
"keyingscreen voronoi sites");
int track_index = 0;
LISTBASE_FOREACH_INDEX (MovieTrackingTrack *, track, &tracking_object->tracks, track_index) {
const MovieTrackingMarker *marker = BKE_tracking_marker_get(track, clip_frame);
@ -130,155 +108,57 @@ KeyingScreenOperation::TriangulationData *KeyingScreenOperation::build_voronoi_t
continue;
}
float pos[2];
add_v2_v2v2(pos, marker->pos, track->offset);
const float2 position = float2(marker->pos) + float2(track->offset);
if (!IN_RANGE_INCL(pos[0], 0.0f, 1.0f) || !IN_RANGE_INCL(pos[1], 0.0f, 1.0f)) {
if (!IN_RANGE_INCL(position.x, 0.0f, 1.0f) || !IN_RANGE_INCL(position.y, 0.0f, 1.0f)) {
continue;
}
ImBuf *pattern_ibuf = BKE_tracking_get_pattern_imbuf(ibuf, track, marker, true, false);
VoronoiSite *site = &sites[track_index];
zero_v3(site->color);
MarkerPoint &marker_point = (*marker_points)[track_index];
marker_point.position = position;
marker_point.color = float4(0.0f);
if (pattern_ibuf) {
for (int j = 0; j < pattern_ibuf->x * pattern_ibuf->y; j++) {
if (pattern_ibuf->float_buffer.data) {
add_v3_v3(site->color, &pattern_ibuf->float_buffer.data[4 * j]);
marker_point.color += float4(&pattern_ibuf->float_buffer.data[4 * j]);
}
else {
uchar *rrgb = pattern_ibuf->byte_buffer.data;
site->color[0] += srgb_to_linearrgb(float(rrgb[4 * j + 0]) / 255.0f);
site->color[1] += srgb_to_linearrgb(float(rrgb[4 * j + 1]) / 255.0f);
site->color[2] += srgb_to_linearrgb(float(rrgb[4 * j + 2]) / 255.0f);
marker_point.color += float4(srgb_to_linearrgb(float(rrgb[4 * j + 0]) / 255.0f),
srgb_to_linearrgb(float(rrgb[4 * j + 1]) / 255.0f),
srgb_to_linearrgb(float(rrgb[4 * j + 2]) / 255.0f),
srgb_to_linearrgb(float(rrgb[4 * j + 3]) / 255.0f));
}
}
mul_v3_fl(site->color, 1.0f / (pattern_ibuf->x * pattern_ibuf->y));
marker_point.color /= pattern_ibuf->x * pattern_ibuf->y;
IMB_freeImBuf(pattern_ibuf);
}
site->co[0] = pos[0] * width;
site->co[1] = pos[1] * height;
site++;
}
IMB_freeImBuf(ibuf);
BLI_voronoi_compute(sites, sites_total, width, height, &edges);
BLI_voronoi_triangulate(sites,
sites_total,
&edges,
width,
height,
&triangulation->triangulated_points,
&triangulation->triangulated_points_total,
&triangulation->triangles,
&triangulation->triangles_total);
MEM_freeN(sites);
BLI_freelistN(&edges);
if (triangulation->triangles_total) {
rcti *rect;
rect = triangulation->triangles_AABB = (rcti *)MEM_callocN(
sizeof(rcti) * triangulation->triangles_total, "voronoi triangulation AABB");
for (i = 0; i < triangulation->triangles_total; i++, rect++) {
int *triangle = triangulation->triangles[i];
VoronoiTriangulationPoint *a = &triangulation->triangulated_points[triangle[0]],
*b = &triangulation->triangulated_points[triangle[1]],
*c = &triangulation->triangulated_points[triangle[2]];
float min[2], max[2];
INIT_MINMAX2(min, max);
minmax_v2v2_v2(min, max, a->co);
minmax_v2v2_v2(min, max, b->co);
minmax_v2v2_v2(min, max, c->co);
rect->xmin = int(min[0]);
rect->ymin = int(min[1]);
rect->xmax = int(max[0]) + 1;
rect->ymax = int(max[1]) + 1;
}
}
return triangulation;
return marker_points;
}
KeyingScreenOperation::TileData *KeyingScreenOperation::triangulate(const rcti *rect)
{
TileData *tile_data;
TriangulationData *triangulation;
int triangles_allocated = 0;
int chunk_size = 20;
int i;
triangulation = cached_triangulation_;
if (!triangulation) {
return nullptr;
}
tile_data = MEM_cnew<TileData>("keying screen tile data");
for (i = 0; i < triangulation->triangles_total; i++) {
if (BLI_rcti_isect(rect, &triangulation->triangles_AABB[i], nullptr)) {
tile_data->triangles_total++;
if (tile_data->triangles_total > triangles_allocated) {
if (!tile_data->triangles) {
tile_data->triangles = (int *)MEM_mallocN(sizeof(int) * chunk_size,
"keying screen tile triangles chunk");
}
else {
tile_data->triangles = (int *)MEM_reallocN(
tile_data->triangles, sizeof(int) * (triangles_allocated + chunk_size));
}
triangles_allocated += chunk_size;
}
tile_data->triangles[tile_data->triangles_total - 1] = i;
}
}
return tile_data;
}
void *KeyingScreenOperation::initialize_tile_data(rcti *rect)
void *KeyingScreenOperation::initialize_tile_data(rcti * /* rect*/)
{
if (movie_clip_ == nullptr) {
return nullptr;
}
if (!cached_triangulation_) {
if (!cached_marker_points_) {
lock_mutex();
if (cached_triangulation_ == nullptr) {
cached_triangulation_ = build_voronoi_triangulation();
if (cached_marker_points_ == nullptr) {
cached_marker_points_ = compute_marker_points();
}
unlock_mutex();
}
return triangulate(rect);
}
void KeyingScreenOperation::deinitialize_tile_data(rcti * /*rect*/, void *data)
{
TileData *tile_data = (TileData *)data;
if (tile_data->triangles) {
MEM_freeN(tile_data->triangles);
}
MEM_freeN(tile_data);
return nullptr;
}
void KeyingScreenOperation::determine_canvas(const rcti &preferred_area, rcti &r_area)
@ -298,95 +178,59 @@ void KeyingScreenOperation::determine_canvas(const rcti &preferred_area, rcti &r
}
}
void KeyingScreenOperation::execute_pixel(float output[4], int x, int y, void *data)
void KeyingScreenOperation::execute_pixel(float output[4], int x, int y, void * /* data */)
{
output[0] = 0.0f;
output[1] = 0.0f;
output[2] = 0.0f;
output[3] = 1.0f;
if (movie_clip_ && data) {
TriangulationData *triangulation = cached_triangulation_;
TileData *tile_data = (TileData *)data;
int i;
float co[2] = {float(x), float(y)};
for (i = 0; i < tile_data->triangles_total; i++) {
int triangle_idx = tile_data->triangles[i];
rcti *rect = &triangulation->triangles_AABB[triangle_idx];
if (IN_RANGE_INCL(x, rect->xmin, rect->xmax) && IN_RANGE_INCL(y, rect->ymin, rect->ymax)) {
int *triangle = triangulation->triangles[triangle_idx];
VoronoiTriangulationPoint *a = &triangulation->triangulated_points[triangle[0]],
*b = &triangulation->triangulated_points[triangle[1]],
*c = &triangulation->triangulated_points[triangle[2]];
float w[3];
if (barycentric_coords_v2(a->co, b->co, c->co, co, w)) {
if (barycentric_inside_triangle_v2(w)) {
output[0] = a->color[0] * w[0] + b->color[0] * w[1] + c->color[0] * w[2];
output[1] = a->color[1] * w[0] + b->color[1] * w[1] + c->color[1] * w[2];
output[2] = a->color[2] * w[0] + b->color[2] * w[1] + c->color[2] * w[2];
break;
}
}
}
}
if (!cached_marker_points_) {
copy_v4_fl(output, 0.0f);
return;
}
const int2 size = int2(this->get_width(), this->get_height());
const float2 normalized_pixel_location = float2(x, y) / float2(size);
const float squared_shape_parameter = math::square(1.0f / smoothness_);
float4 weighted_sum = float4(0.0f);
float sum_of_weights = 0.0f;
for (const MarkerPoint &marker_point : *cached_marker_points_) {
const float2 difference = normalized_pixel_location - marker_point.position;
const float squared_distance = math::dot(difference, difference);
const float gaussian = math::exp(-squared_distance * squared_shape_parameter);
weighted_sum += marker_point.color * gaussian;
sum_of_weights += gaussian;
}
weighted_sum /= sum_of_weights;
copy_v4_v4(output, weighted_sum);
}
void KeyingScreenOperation::update_memory_buffer_partial(MemoryBuffer *output,
const rcti &area,
Span<MemoryBuffer *> inputs)
{
if (movie_clip_ == nullptr) {
output->fill(area, COM_COLOR_BLACK);
if (!cached_marker_points_) {
output->fill(area, COM_COLOR_TRANSPARENT);
return;
}
TileData *tri_area = this->triangulate(&area);
BLI_assert(tri_area != nullptr);
const int2 size = int2(this->get_width(), this->get_height());
const float squared_shape_parameter = math::square(1.0f / smoothness_);
const int *triangles = tri_area->triangles;
const int num_triangles = tri_area->triangles_total;
const TriangulationData *triangulation = cached_triangulation_;
for (BuffersIterator<float> it = output->iterate_with(inputs, area); !it.is_end(); ++it) {
copy_v4_v4(it.out, COM_COLOR_BLACK);
const float2 normalized_pixel_location = float2(it.x, it.y) / float2(size);
const float co[2] = {float(it.x), float(it.y)};
for (int i = 0; i < num_triangles; i++) {
const int triangle_idx = triangles[i];
const rcti *rect = &triangulation->triangles_AABB[triangle_idx];
if (!BLI_rcti_isect_pt(rect, it.x, it.y)) {
continue;
}
const int *triangle = triangulation->triangles[triangle_idx];
const VoronoiTriangulationPoint &a = triangulation->triangulated_points[triangle[0]];
const VoronoiTriangulationPoint &b = triangulation->triangulated_points[triangle[1]];
const VoronoiTriangulationPoint &c = triangulation->triangulated_points[triangle[2]];
float w[3];
if (!barycentric_coords_v2(a.co, b.co, c.co, co, w)) {
continue;
}
if (barycentric_inside_triangle_v2(w)) {
it.out[0] = a.color[0] * w[0] + b.color[0] * w[1] + c.color[0] * w[2];
it.out[1] = a.color[1] * w[0] + b.color[1] * w[1] + c.color[1] * w[2];
it.out[2] = a.color[2] * w[0] + b.color[2] * w[1] + c.color[2] * w[2];
break;
}
float4 weighted_sum = float4(0.0f);
float sum_of_weights = 0.0f;
for (const MarkerPoint &marker_point : *cached_marker_points_) {
const float2 difference = normalized_pixel_location - marker_point.position;
const float squared_distance = math::dot(difference, difference);
const float gaussian = math::exp(-squared_distance * squared_shape_parameter);
weighted_sum += marker_point.color * gaussian;
sum_of_weights += gaussian;
}
}
weighted_sum /= sum_of_weights;
if (tri_area->triangles) {
MEM_freeN(tri_area->triangles);
copy_v4_v4(it.out, weighted_sum);
}
MEM_freeN(tri_area);
}
} // namespace blender::compositor

34
source/blender/compositor/operations/COM_KeyingScreenOperation.h
View File

@ -10,11 +10,12 @@
#include "DNA_movieclip_types.h"
#include "BLI_array.hh"
#include "BLI_listbase.h"
#include "BLI_math_base.hh"
#include "BLI_math_vector_types.hh"
#include "BLI_string.h"
#include "BLI_voronoi_2d.h"
namespace blender::compositor {
/**
@ -22,22 +23,15 @@ namespace blender::compositor {
*/
class KeyingScreenOperation : public MultiThreadedOperation {
protected:
typedef struct TriangulationData {
VoronoiTriangulationPoint *triangulated_points;
int (*triangles)[3];
int triangulated_points_total, triangles_total;
rcti *triangles_AABB;
} TriangulationData;
/* TODO(manzanilla): rename to #TrianguledArea on removing tiled implementation. */
typedef struct TileData {
int *triangles;
int triangles_total;
} TileData;
struct MarkerPoint {
float2 position;
float4 color;
};
MovieClip *movie_clip_;
float smoothness_;
int framenumber_;
TriangulationData *cached_triangulation_;
Array<MarkerPoint> *cached_marker_points_;
char tracking_object_[64];
/**
@ -45,7 +39,7 @@ class KeyingScreenOperation : public MultiThreadedOperation {
*/
void determine_canvas(const rcti &preferred_area, rcti &r_area) override;
TriangulationData *build_voronoi_triangulation();
Array<MarkerPoint> *compute_marker_points();
public:
KeyingScreenOperation();
@ -54,7 +48,6 @@ class KeyingScreenOperation : public MultiThreadedOperation {
void deinit_execution() override;
void *initialize_tile_data(rcti *rect) override;
void deinitialize_tile_data(rcti *rect, void *data) override;
void set_movie_clip(MovieClip *clip)
{
@ -64,6 +57,10 @@ class KeyingScreenOperation : public MultiThreadedOperation {
{
BLI_strncpy(tracking_object_, object, sizeof(tracking_object_));
}
void set_smoothness(float smoothness)
{
smoothness_ = math::interpolate(0.15f, 1.0f, smoothness);
}
void set_framenumber(int framenumber)
{
framenumber_ = framenumber;
@ -74,9 +71,6 @@ class KeyingScreenOperation : public MultiThreadedOperation {
void update_memory_buffer_partial(MemoryBuffer *output,
const rcti &area,
Span<MemoryBuffer *> inputs) override;
private:
TileData *triangulate(const rcti *rect);
};
} // namespace blender::compositor

1
source/blender/makesdna/DNA_node_types.h
View File

@ -1345,6 +1345,7 @@ typedef struct TexNodeOutput {
typedef struct NodeKeyingScreenData {
char tracking_object[64];
float smoothness;
} NodeKeyingScreenData;
typedef struct NodeKeyingData {

6
source/blender/makesrna/intern/rna_nodetree.cc
View File

@ -8146,6 +8146,12 @@ static void def_cmp_keyingscreen(StructRNA *srna)
RNA_def_property_string_sdna(prop, nullptr, "tracking_object");
RNA_def_property_ui_text(prop, "Tracking Object", "");
RNA_def_property_update(prop, NC_NODE | NA_EDITED, "rna_Node_update");
prop = RNA_def_property(srna, "smoothness", PROP_FLOAT, PROP_FACTOR);
RNA_def_property_float_sdna(prop, nullptr, "smoothness");
RNA_def_property_range(prop, 0.0f, 1.0f);
RNA_def_property_ui_text(prop, "Smoothness", "");
RNA_def_property_update(prop, NC_NODE | NA_EDITED, "rna_Node_update");
}
static void def_cmp_keying(StructRNA *srna)

3
source/blender/nodes/composite/nodes/node_composite_keyingscreen.cc
View File

@ -41,6 +41,7 @@ static void node_composit_init_keyingscreen(const bContext *C, PointerRNA *ptr)
bNode *node = (bNode *)ptr->data;
NodeKeyingScreenData *data = MEM_cnew<NodeKeyingScreenData>(__func__);
data->smoothness = 0.0f;
node->storage = data;
const Scene *scene = CTX_data_scene(C);
@ -78,6 +79,8 @@ static void node_composit_buts_keyingscreen(uiLayout *layout, bContext *C, Point
col = uiLayoutColumn(layout, true);
uiItemPointerR(col, ptr, "tracking_object", &tracking_ptr, "objects", "", ICON_OBJECT_DATA);
}
uiItemR(layout, ptr, "smoothness", UI_ITEM_NONE, nullptr, ICON_NONE);
}
using namespace blender::realtime_compositor;