From c61aa200281bcdb3d02bb4f3244cd75de14236ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Blake=20T=C3=B6lva?= <blake@junelife.com>
Date: Sat, 21 Nov 2015 23:15:46 -0800
Subject: [PATCH] Improve DB migration performance.

Achieve 10x speedup in migrating a 12mb dirtyDB file by disabling
ueberDB caching in this special case.  Add some progress messages
to the migration script and rename it, as nothing in it is tied
to mysql.
---
 bin/migrateDirtyDBtoMySQL.js  | 18 -----------------
 bin/migrateDirtyDBtoRealDB.js | 38 +++++++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+), 18 deletions(-)
 delete mode 100644 bin/migrateDirtyDBtoMySQL.js
 create mode 100644 bin/migrateDirtyDBtoRealDB.js

diff --git a/bin/migrateDirtyDBtoMySQL.js b/bin/migrateDirtyDBtoMySQL.js
deleted file mode 100644
index b4913e76..00000000
--- a/bin/migrateDirtyDBtoMySQL.js
+++ /dev/null
@@ -1,18 +0,0 @@
-require("ep_etherpad-lite/node_modules/npm").load({}, function(er,npm) {
-
-  process.chdir(npm.root+'/..')
-
-  var settings = require("ep_etherpad-lite/node/utils/Settings");
-  var dirty = require("ep_etherpad-lite/node_modules/ueberDB/node_modules/dirty")('var/dirty.db');
-  var db = require("ep_etherpad-lite/node/db/DB");
-
-  db.init(function() {
-    db = db.db;
-    dirty.on("load", function() {
-      dirty.forEach(function(key, value) {
-        db.set(key, value);
-      });
-    });
-  });
-
-});
diff --git a/bin/migrateDirtyDBtoRealDB.js b/bin/migrateDirtyDBtoRealDB.js
new file mode 100644
index 00000000..393c3081
--- /dev/null
+++ b/bin/migrateDirtyDBtoRealDB.js
@@ -0,0 +1,38 @@
+require("ep_etherpad-lite/node_modules/npm").load({}, function(er,npm) {
+
+  process.chdir(npm.root+'/..')
+
+  // This script requires that you have modified your settings.json file
+  // to work with a real database.  Please make a backup of your dirty.db
+  // file before using this script, just to be safe.
+
+  var settings = require("ep_etherpad-lite/node/utils/Settings");
+  var dirty = require("dirty")('var/dirty.db');
+  var ueberDB = require("../src/node_modules/ueberDB");
+  var log4js = require("../src/node_modules/log4js");
+  var dbWrapperSettings = {
+      "cache": "0",         // The cache slows things down when you're mostly writing.
+  };
+  var db = new ueberDB.database(settings.dbType, settings.dbSettings, dbWrapperSettings, log4js.getLogger("ueberDB"));
+
+  db.init(function() {
+      console.log("Waiting for dirtyDB to parse its file.");
+      dirty.on("load", function(length) {
+          console.log("Loaded " + length + " records, processing now.");
+          var remaining = length;
+          dirty.forEach(function(key, value) {
+              db.set(key, value, function(error) {
+                  if (typeof error != 'undefined') {
+                      console.log("Unexpected result handling: ", key, value, " was: ", error);
+                  }
+                  remaining -= 1;
+                  var oldremaining = remaining;
+                  if ((oldremaining % 100) == 0) {
+                      console.log("Records not yet flushed to database: ", remaining);
+                  }
+              });
+          });
+      console.log("Please wait for all records to flush to database, then kill this process.");
+    });
+  });
+});