fuwafuwa
/
etherpad-lite
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Improve DB migration performance. 

Achieve 10x speedup in migrating a 12mb dirtyDB file by disabling
ueberDB caching in this special case.  Add some progress messages
to the migration script and rename it, as nothing in it is tied
to mysql.
This commit is contained in:
Blake Tölva 2015-11-21 23:15:46 -08:00
parent f774c2584f
commit c61aa20028
2 changed files with 38 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
bin/migrateDirtyDBtoMySQL.js
View File

@ -1,18 +0,0 @@
require("ep_etherpad-lite/node_modules/npm").load({}, function(er,npm) {
process.chdir(npm.root+'/..')
var settings = require("ep_etherpad-lite/node/utils/Settings");
var dirty = require("ep_etherpad-lite/node_modules/ueberDB/node_modules/dirty")('var/dirty.db');
var db = require("ep_etherpad-lite/node/db/DB");
db.init(function() {
db = db.db;
dirty.on("load", function() {
dirty.forEach(function(key, value) {
db.set(key, value);
});
});
});
});

38
bin/migrateDirtyDBtoRealDB.js Normal file
View File

@ -0,0 +1,38 @@
require("ep_etherpad-lite/node_modules/npm").load({}, function(er,npm) {
process.chdir(npm.root+'/..')
// This script requires that you have modified your settings.json file
// to work with a real database. Please make a backup of your dirty.db
// file before using this script, just to be safe.
var settings = require("ep_etherpad-lite/node/utils/Settings");
var dirty = require("dirty")('var/dirty.db');
var ueberDB = require("../src/node_modules/ueberDB");
var log4js = require("../src/node_modules/log4js");
var dbWrapperSettings = {
"cache": "0", // The cache slows things down when you're mostly writing.
};
var db = new ueberDB.database(settings.dbType, settings.dbSettings, dbWrapperSettings, log4js.getLogger("ueberDB"));
db.init(function() {
console.log("Waiting for dirtyDB to parse its file.");
dirty.on("load", function(length) {
console.log("Loaded " + length + " records, processing now.");
var remaining = length;
dirty.forEach(function(key, value) {
db.set(key, value, function(error) {
if (typeof error != 'undefined') {
console.log("Unexpected result handling: ", key, value, " was: ", error);
}
remaining -= 1;
var oldremaining = remaining;
if ((oldremaining % 100) == 0) {
console.log("Records not yet flushed to database: ", remaining);
}
});
});
console.log("Please wait for all records to flush to database, then kill this process.");
});
});
});