Reuse buffers for video frames

At least on OS X, allocating the buffers is one of the more expensive
parts of video playback, and on an arbitrary 720p H.264 file with simple
subtitles this cuts CPU usage while playing by about 30%.

src/async_video_provider.cpp

@@ -20,6 +20,7 @@
 #include "ass_file.h"
 #include "export_fixstyle.h"
 #include "include/aegisub/subtitles_provider.h"
+#include "video_frame.h"
 #include "video_provider_manager.h"
 
 #include <libaegisub/dispatch.h>
@@ -30,10 +31,22 @@ enum {
 };
 
 std::shared_ptr<VideoFrame> AsyncVideoProvider::ProcFrame(int frame_number, double time, bool raw) {
+	// Find an unused buffer to use or allocate a new one if needed
 	std::shared_ptr<VideoFrame> frame;
+	for (auto& buffer : buffers) {
+		if (buffer.use_count() == 1) {
+			frame = buffer;
+			break;
+		}
+	}
+
+	if (!frame) {
+		frame = std::make_shared<VideoFrame>();
+		buffers.push_back(frame);
+	}
 
 	try {
-		frame = source_provider->GetFrame(frame_number);
+		source_provider->GetFrame(frame_number, *frame);
 	}
 	catch (VideoProviderError const& err) { throw VideoProviderErrorEvent(err); }
 

src/async_video_provider.h

@@ -76,6 +76,8 @@ class AsyncVideoProvider {
 	/// they can be rendered
 	std::atomic<uint_fast32_t> version{ 0 };
 
+	std::vector<std::shared_ptr<VideoFrame>> buffers;
+
 public:
 	/// @brief Load the passed subtitle file
 	/// @param subs File to load

src/include/aegisub/video_provider.h

@@ -37,7 +37,6 @@
 #include <libaegisub/exception.h>
 #include <libaegisub/vfr.h>
 
-#include <memory>
 #include <string>
 
 struct VideoFrame;
@@ -47,7 +46,7 @@ public:
 	virtual ~VideoProvider() = default;
 
 	/// Override this method to actually get frames
-	virtual std::shared_ptr<VideoFrame> GetFrame(int n)=0;
+	virtual void GetFrame(int n, VideoFrame &frame)=0;
 
 	/// Set the YCbCr matrix to the specified one
 	///

src/subs_preview.cpp

@@ -101,18 +101,19 @@ void SubtitlesPreview::SetColour(agi::Color col) {
 void SubtitlesPreview::UpdateBitmap() {
 	if (!vid) return;
 
-	auto frame = vid->GetFrame(0);
+	VideoFrame frame;
+	vid->GetFrame(0, frame);
 
 	if (provider) {
 		try {
 			provider->LoadSubtitles(sub_file.get());
-			provider->DrawSubtitles(*frame, 0.1);
+			provider->DrawSubtitles(frame, 0.1);
 		}
 		catch (...) { }
 	}
 
 	// Convert frame to bitmap
-	*bmp = static_cast<wxBitmap>(GetImage(*frame));
+	*bmp = static_cast<wxBitmap>(GetImage(frame));
 	Refresh();
 }
 

src/video_frame.cpp

@@ -19,24 +19,6 @@
 #include <boost/gil/gil_all.hpp>
 #include <wx/image.h>
 
-VideoFrame::VideoFrame(const unsigned char *data, size_t width, size_t height, size_t pitch, bool flipped)
-: data(data, data + width * height * 4)
-, width(width)
-, height(height)
-, pitch(pitch)
-, flipped(flipped)
-{
-}
-
-VideoFrame::VideoFrame(std::vector<unsigned char>&& data, size_t width, size_t height, size_t pitch, bool flipped)
-: data(std::move(data))
-, width(width)
-, height(height)
-, pitch(pitch)
-, flipped(flipped)
-{
-}
-
 namespace {
 	// We actually have bgr_, not bgra, so we need a custom converter which ignores the alpha channel
 	struct color_converter {

src/video_frame.h

@@ -24,9 +24,6 @@ struct VideoFrame {
 	size_t height;
 	size_t pitch;
 	bool flipped;
-
-	VideoFrame(const unsigned char *data, size_t width, size_t height, size_t pitch, bool fipped);
-	VideoFrame(std::vector<unsigned char>&& data, size_t width, size_t height, size_t pitch, bool fipped);
 };
 
 wxImage GetImage(VideoFrame const& frame);

src/video_provider_avs.cpp

@@ -73,7 +73,7 @@ class AvisynthVideoProvider: public VideoProvider {
 public:
 	AvisynthVideoProvider(agi::fs::path const& filename, std::string const& colormatrix);
 
-	std::shared_ptr<VideoFrame> GetFrame(int n);
+	void GetFrame(int n, VideoFrame &frame) override;
 
 	void SetColorSpace(std::string const& matrix) override {
 		// Can't really do anything if this fails
@@ -309,11 +309,16 @@ AVSValue AvisynthVideoProvider::Open(agi::fs::path const& filename) {
 	throw VideoNotSupported("No function suitable for opening the video found");
 }
 
-std::shared_ptr<VideoFrame> AvisynthVideoProvider::GetFrame(int n) {
+void AvisynthVideoProvider::GetFrame(int n, VideoFrame &out) {
 	std::lock_guard<std::mutex> lock(avs.GetMutex());
 
 	auto frame = RGB32Video->GetFrame(n, avs.GetEnv());
-	return std::make_shared<VideoFrame>(frame->GetReadPtr(), frame->GetRowSize() / 4, frame->GetHeight(), frame->GetPitch(), true);
+	auto ptr = frame->GetReadPtr();
+	out.data.assign(ptr, ptr + frame->GetPitch() * frame->GetHeight());
+	out.flipped = true;
+	out.height = frame->GetHeight();
+	out.width = frame->GetRowSize() / 4;
+	out.pitch = frame->GetPitch();
 }
 }
 

src/video_provider_cache.cpp

@@ -25,14 +25,14 @@
 
 namespace {
 /// A video frame and its frame number
-struct CachedFrame final : public VideoFrame {
+struct CachedFrame {
+	VideoFrame frame;
 	int frame_number;
 
-	CachedFrame(int frame_number, VideoFrame const& frame)
-	: VideoFrame(frame.data.data(), frame.width, frame.height, frame.pitch, frame.flipped)
-	, frame_number(frame_number)
-	{
-	}
+	CachedFrame(VideoFrame const& frame, int frame_number)
+	: frame(frame), frame_number(frame_number) { }
+
+	CachedFrame(CachedFrame const&) = delete;
 };
 
 /// @class VideoProviderCache
@@ -45,19 +45,15 @@ class VideoProviderCache final : public VideoProvider {
 	///
 	/// Note that this is a soft limit. The cache stops allocating new frames
 	/// once it has exceeded the limit, but it never tries to shrink
-	const size_t max_cache_size;
+	const size_t max_cache_size = OPT_GET("Provider/Video/Cache/Size")->GetInt() << 20; // convert MB to bytes
 
 	/// Cache of video frames with the most recently used ones at the front
 	std::list<CachedFrame> cache;
 
 public:
-	VideoProviderCache(std::unique_ptr<VideoProvider> master)
-	: master(std::move(master))
-	, max_cache_size(OPT_GET("Provider/Video/Cache/Size")->GetInt() << 20) // convert MB to bytes
-	{
-	}
+	VideoProviderCache(std::unique_ptr<VideoProvider> master) : master(std::move(master)) { }
 
-	std::shared_ptr<VideoFrame> GetFrame(int n) override;
+	void GetFrame(int n, VideoFrame &frame) override;
 
 	void SetColorSpace(std::string const& m) override {
 		cache.clear();
@@ -78,25 +74,28 @@ public:
 	bool HasAudio() const override                 { return master->HasAudio(); }
 };
 
-std::shared_ptr<VideoFrame> VideoProviderCache::GetFrame(int n) {
+void VideoProviderCache::GetFrame(int n, VideoFrame &out) {
 	size_t total_size = 0;
 
 	for (auto cur = cache.begin(); cur != cache.end(); ++cur) {
 		if (cur->frame_number == n) {
 			cache.splice(cache.begin(), cache, cur); // Move to front
-			return std::make_shared<VideoFrame>(cache.front());
+			out = cache.front().frame;
+			return;
 		}
 
-		total_size += cur->data.size();
+		total_size += cur->frame.data.size();
 	}
 
-	auto frame = master->GetFrame(n);
+	master->GetFrame(n, out);
 
-	if (total_size >= max_cache_size)
-		cache.pop_back();
-	cache.emplace_front(n, *frame);
-
-	return frame;
+	if (total_size >= max_cache_size) {
+		cache.splice(cache.begin(), cache, --cache.end()); // Move last to front
+		cache.front().frame_number = n;
+		cache.front().frame = out;
+	}
+	else
+		cache.emplace_front(out, n);
 }
 }
 

src/video_provider_dummy.cpp

@@ -92,8 +92,12 @@ std::string DummyVideoProvider::MakeFilename(double fps, int frames, int width,
 	return agi::format("?dummy:%f:%d:%d:%d:%d:%d:%d:%s", fps, frames, width, height, (int)colour.r, (int)colour.g, (int)colour.b, (pattern ? "c" : ""));
 }
 
-std::shared_ptr<VideoFrame> DummyVideoProvider::GetFrame(int) {
-	return std::make_shared<VideoFrame>(data.data(), width, height, width * 4, false);
+void DummyVideoProvider::GetFrame(int, VideoFrame &frame) {
+	frame.data    = data;
+	frame.width   = width;
+	frame.height  = height;
+	frame.pitch   = width * 4;
+	frame.flipped = false;
 }
 
 namespace agi { class BackgroundRunner; }

src/video_provider_dummy.h

@@ -64,7 +64,7 @@ public:
 	/// string will result in a video with the given parameters
 	static std::string MakeFilename(double fps, int frames, int width, int height, agi::Color colour, bool pattern);
 
-	std::shared_ptr<VideoFrame> GetFrame(int n) override;
+	void GetFrame(int n, VideoFrame &frame) override;
 	void SetColorSpace(std::string const&) override { }
 
 	int GetFrameCount()             const override { return framecount; }

src/video_provider_ffmpegsource.cpp

@@ -70,7 +70,7 @@ class FFmpegSourceVideoProvider final : public VideoProvider, FFmpegSourceProvid
 public:
 	FFmpegSourceVideoProvider(agi::fs::path const& filename, std::string const& colormatrix, agi::BackgroundRunner *br);
 
-	std::shared_ptr<VideoFrame> GetFrame(int n) override;
+	void GetFrame(int n, VideoFrame &out) override;
 
 	void SetColorSpace(std::string const& matrix) override {
 #if FFMS_VERSION >= ((2 << 24) | (17 << 16) | (1 << 8) | 0)
@@ -285,14 +285,18 @@ void FFmpegSourceVideoProvider::LoadVideo(agi::fs::path const& filename, std::st
 		Timecodes = agi::vfr::Framerate(TimecodesVector);
 }
 
-std::shared_ptr<VideoFrame> FFmpegSourceVideoProvider::GetFrame(int n) {
+void FFmpegSourceVideoProvider::GetFrame(int n, VideoFrame &out) {
 	n = mid(0, n, GetFrameCount() - 1);
 
 	auto frame = FFMS_GetFrame(VideoSource, n, &ErrInfo);
 	if (!frame)
 		throw VideoDecodeError(std::string("Failed to retrieve frame: ") +  ErrInfo.Buffer);
 
-	return std::make_shared<VideoFrame>(frame->Data[0], Width, Height, frame->Linesize[0], false);
+	out.data.assign(frame->Data[0], frame->Data[0] + frame->Linesize[0] * Height);
+	out.flipped = false;
+	out.width = Width;
+	out.height = Height;
+	out.pitch = frame->Linesize[0];
 }
 }
 

src/video_provider_yuv4mpeg.cpp

@@ -143,7 +143,7 @@ class YUV4MPEGVideoProvider final : public VideoProvider {
 public:
 	YUV4MPEGVideoProvider(agi::fs::path const& filename);
 
-	std::shared_ptr<VideoFrame> GetFrame(int n) override;
+	void GetFrame(int n, VideoFrame &frame) override;
 	void SetColorSpace(std::string const&) override { }
 
 	int GetFrameCount() const override             { return num_frames; }
@@ -391,7 +391,7 @@ int YUV4MPEGVideoProvider::IndexFile(uint64_t pos) {
 	return framecount;
 }
 
-std::shared_ptr<VideoFrame> YUV4MPEGVideoProvider::GetFrame(int n) {
+void YUV4MPEGVideoProvider::GetFrame(int n, VideoFrame &frame) {
 	n = mid(0, n, num_frames - 1);
 
 	int uv_width = w / 2;
@@ -408,9 +408,8 @@ std::shared_ptr<VideoFrame> YUV4MPEGVideoProvider::GetFrame(int n) {
 	auto src_y = reinterpret_cast<const unsigned char *>(file.read(seek_table[n], luma_sz + chroma_sz * 2));
 	auto src_u = src_y + luma_sz;
 	auto src_v = src_u + chroma_sz;
-	std::vector<unsigned char> data;
-	data.resize(w * h * 4);
-	unsigned char *dst = &data[0];
+	frame.data.resize(w * h * 4);
+	unsigned char *dst = &frame.data[0];
 
 	for (int py = 0; py < h; ++py) {
 		for (int px = 0; px < w / 2; ++px) {
@@ -433,7 +432,10 @@ std::shared_ptr<VideoFrame> YUV4MPEGVideoProvider::GetFrame(int n) {
 		}
 	}
 
-	return std::make_shared<VideoFrame>(std::move(data), w, h, w * 4, false);
+	frame.flipped = false;
+	frame.width = w;
+	frame.height = h;
+	frame.pitch = w * 4;
 }
 }