From 869e5c7594c7a78945612194f1714cc5e7cb8ea4 Mon Sep 17 00:00:00 2001 From: Karl Blomster Date: Sun, 3 May 2009 18:05:30 +0000 Subject: [PATCH] Implemented an automated cache cleaner that keeps the FFMS2 index cache at a manageable size. Cleaning is started when video is opened with the FFMS2 video provider and runs in its own thread. Removal is based on last access time, and the FFMS2 providers now explicitly touch the index files on reading to make sure access time is updated. By default the cache is kept below 42MB and 20 files; to change the defaults change "FFmpegSource max cache size" and "FFmpegSource max cache files" in config.dat. Either or both may be set to 0, but the cleaner will not delete the last index file in the directory, regardless of cache settings and of the file's size. Originally committed to SVN as r2893. --- aegisub/src/audio_provider_ffmpegsource.cpp | 3 + aegisub/src/ffmpegsource_common.cpp | 129 ++++++++++++++++++++ aegisub/src/ffmpegsource_common.h | 20 +++ aegisub/src/options.cpp | 2 + aegisub/src/video_provider_ffmpegsource.cpp | 9 ++ 5 files changed, 163 insertions(+) diff --git a/aegisub/src/audio_provider_ffmpegsource.cpp b/aegisub/src/audio_provider_ffmpegsource.cpp index f243d1535..8c37476a9 100644 --- a/aegisub/src/audio_provider_ffmpegsource.cpp +++ b/aegisub/src/audio_provider_ffmpegsource.cpp @@ -124,6 +124,9 @@ void FFmpegSourceAudioProvider::LoadAudio(Aegisub::String filename) { } } + // update access time of index file so it won't get cleaned away + wxFileName(CacheName).Touch(); + // FIXME: provide a way to choose which audio track to load? int TrackNumber = FFMS_GetFirstTrackOfType(Index, FFMS_TYPE_AUDIO, FFMSErrMsg, MsgSize); if (TrackNumber < 0) { diff --git a/aegisub/src/ffmpegsource_common.cpp b/aegisub/src/ffmpegsource_common.cpp index 4c5fdfe27..e9659e8fc 100644 --- a/aegisub/src/ffmpegsource_common.cpp +++ b/aegisub/src/ffmpegsource_common.cpp @@ -39,11 +39,20 @@ /////////// // Headers +#include +#include #include "ffmpegsource_common.h" #include "md5.h" #include "standard_paths.h" #include "main.h" #include "frame_main.h" +#include "options.h" +#include + + +// lookit dis here static shit +wxMutex FFmpegSourceProvider::CleaningInProgress; + /////////////// // Update indexing progress @@ -134,4 +143,124 @@ wxString FFmpegSourceProvider::GetCacheFilename(const wxString& filename) return dirfn.GetShortPath() + _T("/") + fn.GetFullName(); } +///////////////////// +// fire and forget cleaning thread (well, almost) +bool FFmpegSourceProvider::CleanCache() { + wxLogDebug(_T("FFmpegSourceCacheCleaner: attempting to start thread")); + + FFmpegSourceCacheCleaner *CleaningThread = new FFmpegSourceCacheCleaner(this); + + if (CleaningThread->Create() != wxTHREAD_NO_ERROR) { + wxLogDebug(_T("FFmpegSourceCacheCleaner: thread creation failed")); + delete CleaningThread; + CleaningThread = NULL; + return false; + } + if (CleaningThread->Run() != wxTHREAD_NO_ERROR) { + wxLogDebug(_T("FFmpegSourceCacheCleaner: failed to start thread")); + delete CleaningThread; + CleaningThread = NULL; + return false; + } + + wxLogDebug(_T("FFmpegSourceCacheCleaner: thread started successfully")); + return true; +} + + +///////////////////// +// constructor +FFmpegSourceCacheCleaner::FFmpegSourceCacheCleaner(FFmpegSourceProvider *par) : wxThread(wxTHREAD_DETACHED) { + parent = par; +} + +///////////////////// +// all actual work happens here because I was too lazy to write more functions +wxThread::ExitCode FFmpegSourceCacheCleaner::Entry() { + wxMutexLocker lock(FFmpegSourceProvider::CleaningInProgress); + if (!lock.IsOk()) { + wxLogDebug(_T("FFmpegSourceCacheCleaner: cleaning already in progress, thread exiting")); + return (wxThread::ExitCode)1; + } + + wxString cachedirname = StandardPaths::DecodePath(_T("?user/ffms2cache/")); + wxDir cachedir; + if (!cachedir.Open(cachedirname)) { + wxLogDebug(_T("FFmpegSourceCacheCleaner: couldn't open cache directory %s"), cachedirname.c_str()); + return (wxThread::ExitCode)1; + } + + // sleep for a bit so we (hopefully) don't thrash the disk too much while indexing is in progress + wxThread::This()->Sleep(2000); + + // the option is in megabytes, we need bytes + // shift left by 20 is CLEARLY more efficient than multiplying by 1048576 + int64_t maxsize = Options.AsInt(_T("FFmpegSource max cache size")) << 20; + int64_t cursize = wxDir::GetTotalSize(cachedirname).GetValue(); + int maxfiles = Options.AsInt(_T("FFmpegSource max cache files")); + + if (!cachedir.HasFiles(_T("*.ffindex"))) { + wxLogDebug(_T("FFmpegSourceCacheCleaner: no index files in cache folder, exiting")); + return (wxThread::ExitCode)0; + } + + int deleted = 0; + int numfiles = 0; + std::multimap cachefiles; + wxString curfn_str; + wxFileName curfn; + wxDateTime curatime; + + // unusually paranoid sanity check + // (someone might have deleted the file(s) after we did HasFiles() above; does wxDir.Open() lock the dir?) + if (!cachedir.GetFirst(&curfn_str, _T("*.ffindex"), wxDIR_FILES)) { + wxLogDebug(_T("FFmpegSourceCacheCleaner: insanity/race condition/index dir fuckery detected, exiting")); + return (wxThread::ExitCode)1; + } + + numfiles++; + curfn = wxFileName(cachedirname, curfn_str); + curfn.GetTimes(&curatime, NULL, NULL); + // FIXME: will break when the time_t's wrap around!!1! + cachefiles.insert(std::pair(curatime.GetTicks(),curfn)); + + while (cachedir.GetNext(&curfn_str)) { + curfn = wxFileName(cachedirname, curfn_str); + curfn.GetTimes(&curatime, NULL, NULL); + cachefiles.insert(std::pair(curatime.GetTicks(),curfn)); + numfiles++; + + wxThread::This()->Sleep(250); + } + + if (numfiles <= maxfiles && cursize <= maxsize) { + wxLogDebug(_T("FFmpegSourceCacheCleaner: cache does not need cleaning (maxsize=%d, cursize=%d; maxfiles=%d, numfiles=%d), exiting"), + maxsize, cursize, maxfiles, numfiles); + return (wxThread::ExitCode)0; + } + + for (std::multimap::iterator i = cachefiles.begin(); i != cachefiles.end(); i++) { + // stop cleaning? + if ((cursize <= maxsize && numfiles <= maxfiles) || numfiles <= 1) + break; + + int64_t fsize = i->second.GetSize().GetValue(); + if (!wxRemoveFile(i->second.GetFullPath())) { + wxLogDebug(_T("FFmpegSourceCacheCleaner: failed to remove file %s"),i->second.GetFullPath().c_str()); + continue; + } + cursize -= fsize; + numfiles--; + deleted++; + + wxThread::This()->Sleep(250); + } + + wxLogDebug(_T("FFmpegSourceCacheCleaner: deleted %d files"), deleted); + wxLogDebug(_T("FFmpegSourceCacheCleaner: done, exiting")); + + return (wxThread::ExitCode)0; +} + + #endif WITH_FFMPEGSOURCE diff --git a/aegisub/src/ffmpegsource_common.h b/aegisub/src/ffmpegsource_common.h index 72d950a47..533a5ed87 100644 --- a/aegisub/src/ffmpegsource_common.h +++ b/aegisub/src/ffmpegsource_common.h @@ -38,12 +38,15 @@ /////////// // Headers +#include +#include #include "include/aegisub/aegisub.h" #include "../FFmpegSource2/ffms.h" #include "dialog_progress.h" class FFmpegSourceProvider { + friend class FFmpegSourceCacheCleaner; public: static const int FFMSTrackMaskAll = -1; static const int FFMSTrackMaskNone = 0; @@ -53,9 +56,26 @@ public: DialogProgress *ProgressDialog; }; + static wxMutex CleaningInProgress; + bool CleanCache(); + static int FFMS_CC UpdateIndexingProgress(int State, int64_t Current, int64_t Total, void *Private); FrameIndex *DoIndexing(FrameIndex *Index, wxString Filename, wxString Cachename, int Trackmask, bool IgnoreDecodeErrors); wxString GetCacheFilename(const wxString& filename); + + virtual FFmpegSourceProvider::~FFmpegSourceProvider() {} }; + +class FFmpegSourceCacheCleaner : public wxThread { +private: + FFmpegSourceProvider *parent; + +public: + FFmpegSourceCacheCleaner(FFmpegSourceProvider *par); + ~FFmpegSourceCacheCleaner() {}; + wxThread::ExitCode Entry(); +}; + + #endif /* WITH_FFMPEGSOURCE */ \ No newline at end of file diff --git a/aegisub/src/options.cpp b/aegisub/src/options.cpp index 5049b71af..8c0814af8 100644 --- a/aegisub/src/options.cpp +++ b/aegisub/src/options.cpp @@ -180,6 +180,8 @@ void OptionsManager::LoadDefaults(bool onlyDefaults,bool doOverride) { SetText(_T("Subtitles Provider"),_T(DEFAULT_PROVIDER_SUBTITLE)); #endif SetBool(_T("Video Use Pixel Shaders"),false,1700); + SetInt(_T("FFmpegSource max cache size"),42); + SetInt(_T("FFmpegSource max cache files"),20); // Audio Options SetModificationType(MOD_AUTOMATIC); diff --git a/aegisub/src/video_provider_ffmpegsource.cpp b/aegisub/src/video_provider_ffmpegsource.cpp index 8bd0373d2..9f765f490 100644 --- a/aegisub/src/video_provider_ffmpegsource.cpp +++ b/aegisub/src/video_provider_ffmpegsource.cpp @@ -39,6 +39,7 @@ /////////// // Headers +#include #include "include/aegisub/aegisub.h" #include "video_provider_ffmpegsource.h" #include "video_context.h" @@ -107,6 +108,14 @@ void FFmpegSourceVideoProvider::LoadVideo(Aegisub::String filename, double fps) } } + // update access time of index file so it won't get cleaned away + wxFileName(CacheName).Touch(); + + // we have now read the index and may proceed with cleaning the index cache + if (!CleanCache()) { + //do something? + } + // set thread count int Threads = Options.AsInt(_T("FFmpegSource decoding threads")); if (Threads < 1)