tmaq
/
tornavis
1
0
Fork 0
Code Issues 2 Pull Requests 29 Packages Projects 24 Releases Wiki Activity

VSE: Add sound strip retiming support 

This patch contains changes needed for retiming sound strips.

`BKE_sound_set_scene_sound_pitch()` is replaced by
`BKE_sound_set_scene_sound_pitch_constant_range()` which uses new
Audaspace interface to set pitch in bulk.
This is done in `SEQ_retiming_sound_animation_data_set()` where retimed
sections are created for each strip. When strip is inside of meta
strip(s), the retimed sections of meta and actual strip are split where
they intersect and pitch is multiplied where they overlap. Each section
will have pitch value that is provided to audaspace.

Waveform overlay now represents retimed audio accurately.

Ref: #100337

Pull Request: https://projects.blender.org/blender/blender/pulls/105072
This commit is contained in:
Richard Antalik 2023-04-21 16:53:27 +02:00 committed by Richard Antalik
parent b44dace9d8
commit b21695a507
14 changed files with 177 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
source/blender/blenkernel/BKE_sound.h
View File

@ -154,6 +154,11 @@ void BKE_sound_set_scene_sound_volume(void *handle, float volume, char animated)
void BKE_sound_set_scene_sound_pitch(void *handle, float pitch, char animated);
void BKE_sound_set_scene_sound_pitch_constant_range(void *handle,
int frame_start,
int frame_end,
float pitch);
void BKE_sound_set_scene_sound_pan(void *handle, float pan, char animated);
void BKE_sound_update_sequencer(struct Main *main, struct bSound *sound);

9
source/blender/blenkernel/intern/sound.c
View File

@ -819,6 +819,15 @@ void BKE_sound_set_scene_sound_pitch(void *handle, float pitch, char animated)
AUD_SequenceEntry_setAnimationData(handle, AUD_AP_PITCH, sound_cfra, &pitch, animated);
}
void BKE_sound_set_scene_sound_pitch_constant_range(void *handle,
int frame_start,
int frame_end,
float pitch)
{
AUD_SequenceEntry_setConstantRangeAnimationData(
handle, AUD_AP_PITCH, frame_start, frame_end, &pitch);
}
void BKE_sound_set_scene_sound_pan(void *handle, float pan, char animated)
{
AUD_SequenceEntry_setAnimationData(handle, AUD_AP_PANNING, sound_cfra, &pan, animated);

17
source/blender/editors/space_sequencer/sequencer_draw.c
View File

@ -413,7 +413,6 @@ static void draw_seq_waveform_overlay(
const float frames_per_pixel = BLI_rctf_size_x(&region->v2d.cur) / region->winx;
const float samples_per_frame = SOUND_WAVE_SAMPLES_PER_SECOND / FPS;
float samples_per_pixel = samples_per_frame * frames_per_pixel;
/* Align strip start with nearest pixel to prevent waveform flickering. */
const float x1_aligned = align_frame_with_pixel(x1, frames_per_pixel);
@ -439,15 +438,17 @@ static void draw_seq_waveform_overlay(
size_t wave_data_len = 0;
/* Offset must be also aligned, otherwise waveform flickers when moving left handle. */
const float strip_offset = align_frame_with_pixel(seq->startofs + seq->anim_startofs,
frames_per_pixel);
float start_sample = strip_offset * samples_per_frame;
start_sample += seq->sound->offset_time * SOUND_WAVE_SAMPLES_PER_SECOND;
float start_frame = SEQ_time_left_handle_frame_get(scene, seq);
/* Add off-screen part of strip to offset. */
start_sample += (frame_start - x1_aligned) * samples_per_frame;
start_frame += (frame_start - x1_aligned);
start_frame += seq->sound->offset_time / FPS;
for (int i = 0; i < pixels_to_draw; i++) {
float sample = start_sample + i * samples_per_pixel;
float timeline_frame = start_frame + i * frames_per_pixel;
/* TODO: Use linear interpolation between frames to avoid bad drawing quality. */
float frame_index = SEQ_give_frame_index(scene, seq, timeline_frame);
float sample = frame_index * samples_per_frame;
int sample_index = round_fl_to_int(sample);
if (sample_index < 0) {
@ -468,6 +469,8 @@ static void draw_seq_waveform_overlay(
value_min = (1.0f - f) * value_min + f * waveform->data[sample_index * 3 + 3];
value_max = (1.0f - f) * value_max + f * waveform->data[sample_index * 3 + 4];
rms = (1.0f - f) * rms + f * waveform->data[sample_index * 3 + 5];
float samples_per_pixel = samples_per_frame * frames_per_pixel;
if (samples_per_pixel > 1.0f) {
/* We need to sum up the values we skip over until the next step. */
float next_pos = sample + samples_per_pixel;

1
source/blender/sequencer/SEQ_retiming.h
View File

@ -39,6 +39,7 @@ float SEQ_retiming_handle_speed_get(const struct Sequence *seq,
const struct SeqRetimingHandle *handle);
int SEQ_retiming_handle_index_get(const struct Sequence *seq,
const struct SeqRetimingHandle *handle);
void SEQ_retiming_sound_animation_data_set(const struct Scene *scene, const struct Sequence *seq);
float SEQ_retiming_handle_timeline_frame_get(const struct Scene *scene,
const struct Sequence *seq,
const struct SeqRetimingHandle *handle);

2
source/blender/sequencer/SEQ_time.h
View File

@ -73,6 +73,8 @@ int SEQ_time_find_next_prev_edit(struct Scene *scene,
bool SEQ_time_strip_intersects_frame(const struct Scene *scene,
const struct Sequence *seq,
int timeline_frame);
/* Convert timeline frame so strip frame index. */
float SEQ_give_frame_index(const struct Scene *scene, struct Sequence *seq, float timeline_frame);
/**
* Returns true if strip has frames without content to render.
*/

2
source/blender/sequencer/intern/effects.c
View File

@ -2614,7 +2614,7 @@ float seq_speed_effect_target_frame_get(Scene *scene,
}
SEQ_effect_handle_get(seq_speed); /* Ensure, that data are initialized. */
int frame_index = seq_give_frame_index(scene, seq_speed, timeline_frame);
int frame_index = SEQ_give_frame_index(scene, seq_speed, timeline_frame);
SpeedControlVars *s = (SpeedControlVars *)seq_speed->effectdata;
const Sequence *source = seq_speed->seq1;

2
source/blender/sequencer/intern/image_cache.c
View File

@ -142,7 +142,7 @@ static float seq_cache_timeline_frame_to_frame_index(Scene *scene,
* images or extended frame range of movies will only generate one cache entry. No special
* treatment in converting frame index to timeline_frame is needed. */
if (ELEM(type, SEQ_CACHE_STORE_RAW, SEQ_CACHE_STORE_THUMBNAIL)) {
return seq_give_frame_index(scene, seq, timeline_frame);
return SEQ_give_frame_index(scene, seq, timeline_frame);
}
return timeline_frame - SEQ_time_start_frame_get(seq);

2
source/blender/sequencer/intern/proxy.c
View File

@ -209,7 +209,7 @@ ImBuf *seq_proxy_fetch(const SeqRenderData *context, Sequence *seq, int timeline
}
if (proxy->storage & SEQ_STORAGE_PROXY_CUSTOM_FILE) {
int frameno = (int)seq_give_frame_index(context->scene, seq, timeline_frame) +
int frameno = (int)SEQ_give_frame_index(context->scene, seq, timeline_frame) +
seq->anim_startofs;
if (proxy->anim == NULL) {
if (seq_proxy_get_fname(

6
source/blender/sequencer/intern/render.c
View File

@ -238,7 +238,7 @@ StripElem *SEQ_render_give_stripelem(const Scene *scene, Sequence *seq, int time
* all other strips don't use this...
*/
int frame_index = (int)seq_give_frame_index(scene, seq, timeline_frame);
int frame_index = (int)SEQ_give_frame_index(scene, seq, timeline_frame);
if (frame_index == -1 || se == NULL) {
return NULL;
@ -1042,7 +1042,7 @@ static ImBuf *seq_render_movie_strip_custom_file_proxy(const SeqRenderData *cont
}
}
int frameno = (int)seq_give_frame_index(context->scene, seq, timeline_frame) +
int frameno = (int)SEQ_give_frame_index(context->scene, seq, timeline_frame) +
seq->anim_startofs;
return IMB_anim_absolute(proxy->anim, frameno, IMB_TC_NONE, IMB_PROXY_NONE);
}
@ -1658,7 +1658,7 @@ static ImBuf *do_render_strip_uncached(const SeqRenderData *context,
bool *r_is_proxy_image)
{
ImBuf *ibuf = NULL;
float frame_index = seq_give_frame_index(context->scene, seq, timeline_frame);
float frame_index = SEQ_give_frame_index(context->scene, seq, timeline_frame);
int type = (seq->type & SEQ_TYPE_EFFECT) ? SEQ_TYPE_EFFECT : seq->type;
switch (type) {
case SEQ_TYPE_META: {

4
source/blender/sequencer/intern/sequencer.c
View File

@ -990,9 +990,7 @@ static bool seq_update_seq_cb(Sequence *seq, void *user_data)
}
BKE_sound_set_scene_sound_volume(
seq->scene_sound, seq->volume, (seq->flag & SEQ_AUDIO_VOLUME_ANIMATED) != 0);
BKE_sound_set_scene_sound_pitch(seq->scene_sound,
SEQ_sound_pitch_get(scene, seq),
(seq->flag & SEQ_AUDIO_PITCH_ANIMATED) != 0);
SEQ_retiming_sound_animation_data_set(scene, seq);
BKE_sound_set_scene_sound_pan(
seq->scene_sound, seq->pan, (seq->flag & SEQ_AUDIO_PAN_ANIMATED) != 0);
}

1
source/blender/sequencer/intern/strip_add.c
View File

@ -438,6 +438,7 @@ Sequence *SEQ_add_movie_strip(Main *bmain, Scene *scene, ListBase *seqbase, SeqL
if (load_data->flags & SEQ_LOAD_MOVIE_SYNC_FPS) {
scene->r.frs_sec = fps_denom;
scene->r.frs_sec_base = fps_num;
DEG_id_tag_update(&scene->id, ID_RECALC_AUDIO_FPS | ID_RECALC_SEQUENCER_STRIPS);
}
load_data->r_video_stream_start = IMD_anim_get_offset(anim_arr[0]);

133
source/blender/sequencer/intern/strip_retiming.cc
View File

@ -13,6 +13,7 @@
#include "BLI_listbase.h"
#include "BLI_math.h"
#include "BLI_span.hh"
#include "BLI_vector.hh"
#include "BKE_fcurve.h"
#include "BKE_movieclip.h"
@ -118,6 +119,7 @@ bool SEQ_retiming_is_active(const Sequence *seq)
bool SEQ_retiming_is_allowed(const Sequence *seq)
{
return ELEM(seq->type,
SEQ_TYPE_SOUND_RAM,
SEQ_TYPE_IMAGE,
SEQ_TYPE_META,
SEQ_TYPE_SCENE,
@ -126,7 +128,7 @@ bool SEQ_retiming_is_allowed(const Sequence *seq)
SEQ_TYPE_MASK);
}
float seq_retiming_evaluate(const Sequence *seq, const int frame_index)
float seq_retiming_evaluate(const Sequence *seq, const float frame_index)
{
const SeqRetimingHandle *previous_handle = retiming_find_segment_start_handle(seq, frame_index);
const SeqRetimingHandle *next_handle = previous_handle + 1;
@ -140,9 +142,8 @@ float seq_retiming_evaluate(const Sequence *seq, const int frame_index)
}
const int segment_length = next_handle->strip_frame_index - previous_handle->strip_frame_index;
const int segment_frame_index = frame_index - previous_handle->strip_frame_index;
const float segment_fac = segment_frame_index / float(segment_length);
const float segment_frame_index = frame_index - previous_handle->strip_frame_index;
const float segment_fac = segment_frame_index / (float)segment_length;
const float target_diff = next_handle->retiming_factor - previous_handle->retiming_factor;
return previous_handle->retiming_factor + (target_diff * segment_fac);
}
@ -245,6 +246,130 @@ float SEQ_retiming_handle_speed_get(const Sequence *seq, const SeqRetimingHandle
return speed;
}
class RetimingRange {
public:
int start, end;
float speed;
enum eIntersectType {
FULL,
PARTIAL_START,
PARTIAL_END,
INSIDE,
NONE,
};
RetimingRange(int start_frame, int end_frame, float speed)
: start(start_frame), end(end_frame), speed(speed)
{
}
const eIntersectType intersect_type(const RetimingRange &other) const
{
if (other.start <= start && other.end >= end) {
return FULL;
}
if (other.start > start && other.start < end && other.end > start && other.end < end) {
return INSIDE;
}
if (other.start > start && other.start < end) {
return PARTIAL_END;
}
if (other.end > start && other.end < end) {
return PARTIAL_START;
}
return NONE;
}
};
class RetimingRangeData {
public:
blender::Vector<RetimingRange> ranges;
RetimingRangeData(const Sequence *seq)
{
MutableSpan handles = SEQ_retiming_handles_get(seq);
for (const SeqRetimingHandle &handle : handles) {
if (handle.strip_frame_index == 0) {
continue;
}
const SeqRetimingHandle *handle_prev = &handle - 1;
float speed = SEQ_retiming_handle_speed_get(seq, &handle);
int frame_start = SEQ_time_start_frame_get(seq) + handle_prev->strip_frame_index;
int frame_end = SEQ_time_start_frame_get(seq) + handle.strip_frame_index;
RetimingRange range = RetimingRange(frame_start, frame_end, speed);
ranges.append(range);
}
}
RetimingRangeData &operator*=(const RetimingRangeData rhs)
{
if (ranges.is_empty()) {
for (const RetimingRange &rhs_range : rhs.ranges) {
RetimingRange range = RetimingRange(rhs_range.start, rhs_range.end, rhs_range.speed);
ranges.append(range);
}
return *this;
}
for (int i = 0; i < ranges.size(); i++) {
RetimingRange &range = ranges[i];
for (const RetimingRange &rhs_range : rhs.ranges) {
if (range.intersect_type(rhs_range) == range.NONE) {
continue;
}
else if (range.intersect_type(rhs_range) == range.FULL) {
range.speed *= rhs_range.speed;
}
else if (range.intersect_type(rhs_range) == range.PARTIAL_START) {
RetimingRange range_left = RetimingRange(
range.start, rhs_range.end, range.speed * rhs_range.speed);
range.start = rhs_range.end + 1;
ranges.insert(i, range_left);
}
else if (range.intersect_type(rhs_range) == range.PARTIAL_END) {
RetimingRange range_left = RetimingRange(range.start, rhs_range.start - 1, range.speed);
range.start = rhs_range.start;
ranges.insert(i, range_left);
}
else if (range.intersect_type(rhs_range) == range.INSIDE) {
RetimingRange range_left = RetimingRange(range.start, rhs_range.start - 1, range.speed);
RetimingRange range_mid = RetimingRange(
rhs_range.start, rhs_range.start, rhs_range.speed * range.speed);
range.start = rhs_range.end + 1;
ranges.insert(i, range_left);
ranges.insert(i, range_mid);
break;
}
}
}
return *this;
}
};
static RetimingRangeData seq_retiming_range_data_get(const Scene *scene, const Sequence *seq)
{
RetimingRangeData strip_retiming_data = RetimingRangeData(seq);
const Sequence *meta_parent = seq_sequence_lookup_meta_by_seq(scene, seq);
if (meta_parent == nullptr) {
return strip_retiming_data;
}
RetimingRangeData meta_retiming_data = RetimingRangeData(meta_parent);
strip_retiming_data *= meta_retiming_data;
return strip_retiming_data;
}
void SEQ_retiming_sound_animation_data_set(const Scene *scene, const Sequence *seq)
{
RetimingRangeData retiming_data = seq_retiming_range_data_get(scene, seq);
for (const RetimingRange &range : retiming_data.ranges) {
BKE_sound_set_scene_sound_pitch_constant_range(
seq->scene_sound, range.start, range.end, range.speed);
}
}
float SEQ_retiming_handle_timeline_frame_get(const Scene *scene,
const Sequence *seq,
const SeqRetimingHandle *handle)

15
source/blender/sequencer/intern/strip_time.c
View File

@ -53,7 +53,16 @@ float seq_time_media_playback_rate_factor_get(const Scene *scene, const Sequence
return seq->media_playback_rate / scene_playback_rate;
}
float seq_give_frame_index(const Scene *scene, Sequence *seq, float timeline_frame)
int seq_time_strip_original_content_length_get(const Scene *scene, const Sequence *seq)
{
if (seq->type == SEQ_TYPE_SOUND_RAM) {
return seq->len;
}
return seq->len / seq_time_media_playback_rate_factor_get(scene, seq);
}
float SEQ_give_frame_index(const Scene *scene, Sequence *seq, float timeline_frame)
{
float frame_index;
float sta = SEQ_time_start_frame_get(seq);
@ -494,10 +503,6 @@ bool SEQ_time_has_still_frames(const Scene *scene, const Sequence *seq)
int SEQ_time_strip_length_get(const Scene *scene, const Sequence *seq)
{
if (seq->type == SEQ_TYPE_SOUND_RAM) {
return seq->len;
}
if (SEQ_retiming_is_active(seq)) {
SeqRetimingHandle *handle_start = seq->retiming_handles;
SeqRetimingHandle *handle_end = seq->retiming_handles + (SEQ_retiming_handles_count(seq) - 1);

5
source/blender/sequencer/intern/strip_time.h
View File

@ -16,7 +16,6 @@ struct Scene;
struct Sequence;
struct SeqCollection;
float seq_give_frame_index(const struct Scene *scene, struct Sequence *seq, float timeline_frame);
void seq_update_sound_bounds_recursive(const struct Scene *scene, struct Sequence *metaseq);
/* Describes gap between strips in timeline. */
@ -46,7 +45,9 @@ void seq_time_update_effects_strip_range(const struct Scene *scene, struct SeqCo
void seq_time_translate_handles(const struct Scene *scene, struct Sequence *seq, const int offset);
float seq_time_media_playback_rate_factor_get(const struct Scene *scene,
const struct Sequence *seq);
float seq_retiming_evaluate(const struct Sequence *seq, const int frame_index);
int seq_time_strip_original_content_length_get(const struct Scene *scene,
const struct Sequence *seq);
float seq_retiming_evaluate(const struct Sequence *seq, const float frame_index);
#ifdef __cplusplus
}