ffmpeg: cache swscale contexts instead of re-creating them

ffmpeg's libswscale is used to do RGB<->YUV conversions on movie reading
and writing. The "context" for the scale operation was being created
and destroyed for each movie clip / animation. Now, maintain a cache
of the scale contexts instead.

E.g. in Gold edit, it only ever needs two contexts (one for reading
source movie clips since they are all exactly the same resolution
and format; and one for rendering the resulting movie).

During playback, on some of the "slow" frames (camera cuts) this
saves 10-20ms (Windows, Ryzen 5950X). Rendering whole movie goes
from 390sec to 376sec.

Pull Request: https://projects.blender.org/blender/blender/pulls/118130

source/blender/blenkernel/BKE_writeffmpeg.hh

@@ -71,8 +71,18 @@ bool BKE_ffmpeg_alpha_channel_is_supported(const RenderData *rd);
 void *BKE_ffmpeg_context_create(void);
 void BKE_ffmpeg_context_free(void *context_v);
 
+void BKE_ffmpeg_exit();
+
+/**
+ * Gets a libswscale context for given size and format parameters.
+ * After you're done using the context, call #BKE_ffmpeg_sws_release_context
+ * to release it. Internally the contexts are coming from the context
+ * pool/cache.
+ */
 SwsContext *BKE_ffmpeg_sws_get_context(
     int width, int height, int av_src_format, int av_dst_format, int sws_flags);
+void BKE_ffmpeg_sws_release_context(SwsContext *ctx);
+
 void BKE_ffmpeg_sws_scale_frame(SwsContext *ctx, AVFrame *dst, const AVFrame *src);
 
 #endif

source/blender/blenkernel/intern/blender.cc

@@ -35,6 +35,7 @@
 #include "BKE_report.hh"
 #include "BKE_screen.hh"
 #include "BKE_studiolight.h"
+#include "BKE_writeffmpeg.hh"
 
 #include "DEG_depsgraph.hh"
 
@@ -74,6 +75,9 @@ void BKE_blender_free()
   BKE_callback_global_finalize();
 
   IMB_moviecache_destruct();
+#ifdef WITH_FFMPEG
+  BKE_ffmpeg_exit();
+#endif
 
   BKE_node_system_exit();
 }

source/blender/blenkernel/intern/writeffmpeg.cc

@@ -28,6 +28,7 @@
 #  include "BLI_math_base.h"
 #  include "BLI_threads.h"
 #  include "BLI_utildefines.h"
+#  include "BLI_vector.hh"
 
 #  include "BKE_global.hh"
 #  include "BKE_image.h"
@@ -56,6 +57,25 @@ extern "C" {
 
 struct StampData;
 
+/* libswscale context creation and destruction is expensive.
+ * Maintain a cache of already created contexts. */
+
+constexpr int64_t swscale_cache_max_entries = 32;
+
+struct SwscaleContext {
+  int width = 0, height = 0;
+  AVPixelFormat src_format = AV_PIX_FMT_NONE, dst_format = AV_PIX_FMT_NONE;
+  int flags = 0;
+
+  SwsContext *context = nullptr;
+  int64_t last_use_timestamp = 0;
+  bool is_used = false;
+};
+
+static ThreadMutex swscale_cache_lock = PTHREAD_MUTEX_INITIALIZER;
+static int64_t swscale_cache_timestamp = 0;
+static blender::Vector<SwscaleContext> *swscale_cache = nullptr;
+
 struct FFMpegContext {
   int ffmpeg_type;
   AVCodecID ffmpeg_codec;
@@ -665,7 +685,7 @@ static const AVCodec *get_av1_encoder(
   return codec;
 }
 
-SwsContext *BKE_ffmpeg_sws_get_context(
+static SwsContext *sws_create_context(
     int width, int height, int av_src_format, int av_dst_format, int sws_flags)
 {
 #  if defined(FFMPEG_SWSCALE_THREADING)
@@ -703,6 +723,127 @@ SwsContext *BKE_ffmpeg_sws_get_context(
 
   return c;
 }
+
+static void init_swscale_cache_if_needed()
+{
+  if (swscale_cache == nullptr) {
+    swscale_cache = new blender::Vector<SwscaleContext>();
+    swscale_cache_timestamp = 0;
+  }
+}
+
+static bool remove_oldest_swscale_context()
+{
+  int64_t oldest_index = -1;
+  int64_t oldest_time = 0;
+  for (int64_t index = 0; index < swscale_cache->size(); index++) {
+    SwscaleContext &ctx = (*swscale_cache)[index];
+    if (ctx.is_used) {
+      continue;
+    }
+    int64_t time = swscale_cache_timestamp - ctx.last_use_timestamp;
+    if (time > oldest_time) {
+      oldest_time = time;
+      oldest_index = index;
+    }
+  }
+
+  if (oldest_index >= 0) {
+    SwscaleContext &ctx = (*swscale_cache)[oldest_index];
+    sws_freeContext(ctx.context);
+    swscale_cache->remove_and_reorder(oldest_index);
+    return true;
+  }
+  return false;
+}
+
+static void maintain_swscale_cache_size()
+{
+  while (swscale_cache->size() > swscale_cache_max_entries) {
+    if (!remove_oldest_swscale_context()) {
+      /* Could not remove anything (all contexts are actively used),
+       * stop trying. */
+      break;
+    }
+  }
+}
+
+SwsContext *BKE_ffmpeg_sws_get_context(
+    int width, int height, int av_src_format, int av_dst_format, int sws_flags)
+{
+  BLI_mutex_lock(&swscale_cache_lock);
+
+  init_swscale_cache_if_needed();
+
+  swscale_cache_timestamp++;
+
+  /* Search for unused context that has suitable parameters. */
+  SwsContext *ctx = nullptr;
+  for (SwscaleContext &c : *swscale_cache) {
+    if (!c.is_used && c.width == width && c.height == height && c.src_format == av_src_format &&
+        c.dst_format == av_dst_format && c.flags == sws_flags)
+    {
+      ctx = c.context;
+      /* Mark as used. */
+      c.is_used = true;
+      c.last_use_timestamp = swscale_cache_timestamp;
+      break;
+    }
+  }
+  if (ctx == nullptr) {
+    /* No free matching context in cache: create a new one. */
+    ctx = sws_create_context(width, height, av_src_format, av_dst_format, sws_flags);
+    SwscaleContext c;
+    c.width = width;
+    c.height = height;
+    c.src_format = AVPixelFormat(av_src_format);
+    c.dst_format = AVPixelFormat(av_dst_format);
+    c.flags = sws_flags;
+    c.context = ctx;
+    c.is_used = true;
+    c.last_use_timestamp = swscale_cache_timestamp;
+    swscale_cache->append(c);
+
+    maintain_swscale_cache_size();
+  }
+
+  BLI_mutex_unlock(&swscale_cache_lock);
+  return ctx;
+}
+
+void BKE_ffmpeg_sws_release_context(SwsContext *ctx)
+{
+  BLI_mutex_lock(&swscale_cache_lock);
+  init_swscale_cache_if_needed();
+
+  bool found = false;
+  for (SwscaleContext &c : *swscale_cache) {
+    if (c.context == ctx) {
+      BLI_assert_msg(c.is_used, "Releasing ffmpeg swscale context that is not in use");
+      c.is_used = false;
+      found = true;
+      break;
+    }
+  }
+  BLI_assert_msg(found, "Releasing ffmpeg swscale context that is not in cache");
+  maintain_swscale_cache_size();
+
+  BLI_mutex_unlock(&swscale_cache_lock);
+}
+
+void BKE_ffmpeg_exit()
+{
+  BLI_mutex_lock(&swscale_cache_lock);
+  if (swscale_cache != nullptr) {
+    for (SwscaleContext &c : *swscale_cache) {
+      sws_freeContext(c.context);
+    }
+    delete swscale_cache;
+    swscale_cache = nullptr;
+  }
+  BLI_mutex_unlock(&swscale_cache_lock);
+}
+
 void BKE_ffmpeg_sws_scale_frame(SwsContext *ctx, AVFrame *dst, const AVFrame *src)
 {
 #  if defined(FFMPEG_SWSCALE_THREADING)
@@ -1677,7 +1818,7 @@ static void end_ffmpeg_impl(FFMpegContext *context, int is_autosplit)
   }
 
   if (context->img_convert_ctx != nullptr) {
-    sws_freeContext(context->img_convert_ctx);
+    BKE_ffmpeg_sws_release_context(context->img_convert_ctx);
     context->img_convert_ctx = nullptr;
   }
 }

source/blender/imbuf/intern/anim_movie.cc

@@ -1469,8 +1469,7 @@ static void free_anim_ffmpeg(ImBufAnim *anim)
     av_frame_free(&anim->pFrame_backup);
     av_frame_free(&anim->pFrameRGB);
     av_frame_free(&anim->pFrameDeinterlaced);
-
-    sws_freeContext(anim->img_convert_ctx);
+    BKE_ffmpeg_sws_release_context(anim->img_convert_ctx);
   }
   anim->duration_in_frames = 0;
 }