Implemented an automated cache cleaner that keeps the FFMS2 index cache at a manageable size. Cleaning is started when video is opened with the FFMS2 video provider and runs in its own thread. Removal is based on last access time, and the FFMS2 providers now explicitly touch the index files on reading to make sure access time is updated.

By default the cache is kept below 42MB and 20 files; to change the defaults change "FFmpegSource max cache size" and "FFmpegSource max cache files" in config.dat. Either or both may be set to 0, but the cleaner will not delete the last index file in the directory, regardless of cache settings and of the file's size.

Originally committed to SVN as r2893.
This commit is contained in:
Karl Blomster 2009-05-03 18:05:30 +00:00
parent e3ce74248d
commit 869e5c7594
5 changed files with 163 additions and 0 deletions

View File

@ -124,6 +124,9 @@ void FFmpegSourceAudioProvider::LoadAudio(Aegisub::String filename) {
}
}
// update access time of index file so it won't get cleaned away
wxFileName(CacheName).Touch();
// FIXME: provide a way to choose which audio track to load?
int TrackNumber = FFMS_GetFirstTrackOfType(Index, FFMS_TYPE_AUDIO, FFMSErrMsg, MsgSize);
if (TrackNumber < 0) {

View File

@ -39,11 +39,20 @@
///////////
// Headers
#include <wx/wxprec.h>
#include <wx/dir.h>
#include "ffmpegsource_common.h"
#include "md5.h"
#include "standard_paths.h"
#include "main.h"
#include "frame_main.h"
#include "options.h"
#include <map>
// lookit dis here static shit
wxMutex FFmpegSourceProvider::CleaningInProgress;
///////////////
// Update indexing progress
@ -134,4 +143,124 @@ wxString FFmpegSourceProvider::GetCacheFilename(const wxString& filename)
return dirfn.GetShortPath() + _T("/") + fn.GetFullName();
}
/////////////////////
// fire and forget cleaning thread (well, almost)
bool FFmpegSourceProvider::CleanCache() {
wxLogDebug(_T("FFmpegSourceCacheCleaner: attempting to start thread"));
FFmpegSourceCacheCleaner *CleaningThread = new FFmpegSourceCacheCleaner(this);
if (CleaningThread->Create() != wxTHREAD_NO_ERROR) {
wxLogDebug(_T("FFmpegSourceCacheCleaner: thread creation failed"));
delete CleaningThread;
CleaningThread = NULL;
return false;
}
if (CleaningThread->Run() != wxTHREAD_NO_ERROR) {
wxLogDebug(_T("FFmpegSourceCacheCleaner: failed to start thread"));
delete CleaningThread;
CleaningThread = NULL;
return false;
}
wxLogDebug(_T("FFmpegSourceCacheCleaner: thread started successfully"));
return true;
}
/////////////////////
// constructor
FFmpegSourceCacheCleaner::FFmpegSourceCacheCleaner(FFmpegSourceProvider *par) : wxThread(wxTHREAD_DETACHED) {
parent = par;
}
/////////////////////
// all actual work happens here because I was too lazy to write more functions
wxThread::ExitCode FFmpegSourceCacheCleaner::Entry() {
wxMutexLocker lock(FFmpegSourceProvider::CleaningInProgress);
if (!lock.IsOk()) {
wxLogDebug(_T("FFmpegSourceCacheCleaner: cleaning already in progress, thread exiting"));
return (wxThread::ExitCode)1;
}
wxString cachedirname = StandardPaths::DecodePath(_T("?user/ffms2cache/"));
wxDir cachedir;
if (!cachedir.Open(cachedirname)) {
wxLogDebug(_T("FFmpegSourceCacheCleaner: couldn't open cache directory %s"), cachedirname.c_str());
return (wxThread::ExitCode)1;
}
// sleep for a bit so we (hopefully) don't thrash the disk too much while indexing is in progress
wxThread::This()->Sleep(2000);
// the option is in megabytes, we need bytes
// shift left by 20 is CLEARLY more efficient than multiplying by 1048576
int64_t maxsize = Options.AsInt(_T("FFmpegSource max cache size")) << 20;
int64_t cursize = wxDir::GetTotalSize(cachedirname).GetValue();
int maxfiles = Options.AsInt(_T("FFmpegSource max cache files"));
if (!cachedir.HasFiles(_T("*.ffindex"))) {
wxLogDebug(_T("FFmpegSourceCacheCleaner: no index files in cache folder, exiting"));
return (wxThread::ExitCode)0;
}
int deleted = 0;
int numfiles = 0;
std::multimap<int64_t,wxFileName> cachefiles;
wxString curfn_str;
wxFileName curfn;
wxDateTime curatime;
// unusually paranoid sanity check
// (someone might have deleted the file(s) after we did HasFiles() above; does wxDir.Open() lock the dir?)
if (!cachedir.GetFirst(&curfn_str, _T("*.ffindex"), wxDIR_FILES)) {
wxLogDebug(_T("FFmpegSourceCacheCleaner: insanity/race condition/index dir fuckery detected, exiting"));
return (wxThread::ExitCode)1;
}
numfiles++;
curfn = wxFileName(cachedirname, curfn_str);
curfn.GetTimes(&curatime, NULL, NULL);
// FIXME: will break when the time_t's wrap around!!1!
cachefiles.insert(std::pair<int64_t,wxFileName>(curatime.GetTicks(),curfn));
while (cachedir.GetNext(&curfn_str)) {
curfn = wxFileName(cachedirname, curfn_str);
curfn.GetTimes(&curatime, NULL, NULL);
cachefiles.insert(std::pair<int64_t,wxFileName>(curatime.GetTicks(),curfn));
numfiles++;
wxThread::This()->Sleep(250);
}
if (numfiles <= maxfiles && cursize <= maxsize) {
wxLogDebug(_T("FFmpegSourceCacheCleaner: cache does not need cleaning (maxsize=%d, cursize=%d; maxfiles=%d, numfiles=%d), exiting"),
maxsize, cursize, maxfiles, numfiles);
return (wxThread::ExitCode)0;
}
for (std::multimap<int64_t,wxFileName>::iterator i = cachefiles.begin(); i != cachefiles.end(); i++) {
// stop cleaning?
if ((cursize <= maxsize && numfiles <= maxfiles) || numfiles <= 1)
break;
int64_t fsize = i->second.GetSize().GetValue();
if (!wxRemoveFile(i->second.GetFullPath())) {
wxLogDebug(_T("FFmpegSourceCacheCleaner: failed to remove file %s"),i->second.GetFullPath().c_str());
continue;
}
cursize -= fsize;
numfiles--;
deleted++;
wxThread::This()->Sleep(250);
}
wxLogDebug(_T("FFmpegSourceCacheCleaner: deleted %d files"), deleted);
wxLogDebug(_T("FFmpegSourceCacheCleaner: done, exiting"));
return (wxThread::ExitCode)0;
}
#endif WITH_FFMPEGSOURCE

View File

@ -38,12 +38,15 @@
///////////
// Headers
#include <wx/wxprec.h>
#include <wx/thread.h>
#include "include/aegisub/aegisub.h"
#include "../FFmpegSource2/ffms.h"
#include "dialog_progress.h"
class FFmpegSourceProvider {
friend class FFmpegSourceCacheCleaner;
public:
static const int FFMSTrackMaskAll = -1;
static const int FFMSTrackMaskNone = 0;
@ -53,9 +56,26 @@ public:
DialogProgress *ProgressDialog;
};
static wxMutex CleaningInProgress;
bool CleanCache();
static int FFMS_CC UpdateIndexingProgress(int State, int64_t Current, int64_t Total, void *Private);
FrameIndex *DoIndexing(FrameIndex *Index, wxString Filename, wxString Cachename, int Trackmask, bool IgnoreDecodeErrors);
wxString GetCacheFilename(const wxString& filename);
virtual FFmpegSourceProvider::~FFmpegSourceProvider() {}
};
class FFmpegSourceCacheCleaner : public wxThread {
private:
FFmpegSourceProvider *parent;
public:
FFmpegSourceCacheCleaner(FFmpegSourceProvider *par);
~FFmpegSourceCacheCleaner() {};
wxThread::ExitCode Entry();
};
#endif /* WITH_FFMPEGSOURCE */

View File

@ -180,6 +180,8 @@ void OptionsManager::LoadDefaults(bool onlyDefaults,bool doOverride) {
SetText(_T("Subtitles Provider"),_T(DEFAULT_PROVIDER_SUBTITLE));
#endif
SetBool(_T("Video Use Pixel Shaders"),false,1700);
SetInt(_T("FFmpegSource max cache size"),42);
SetInt(_T("FFmpegSource max cache files"),20);
// Audio Options
SetModificationType(MOD_AUTOMATIC);

View File

@ -39,6 +39,7 @@
///////////
// Headers
#include <wx/utils.h>
#include "include/aegisub/aegisub.h"
#include "video_provider_ffmpegsource.h"
#include "video_context.h"
@ -107,6 +108,14 @@ void FFmpegSourceVideoProvider::LoadVideo(Aegisub::String filename, double fps)
}
}
// update access time of index file so it won't get cleaned away
wxFileName(CacheName).Touch();
// we have now read the index and may proceed with cleaning the index cache
if (!CleanCache()) {
//do something?
}
// set thread count
int Threads = Options.AsInt(_T("FFmpegSource decoding threads"));
if (Threads < 1)