refactor: use a new database for the store

BREAKING CHANGE: old file-based store data is lost

ormconfig.yml

@@ -10,3 +10,14 @@ library:
     - ./src/main/migrations/library/*.js
   cli:
     migrationsDir: ./src/main/migrations/library
+
+store:
+  type: sqlite
+  database: ./test-paths/typeorm/store.db
+  cache: true
+  entities:
+    - ./src/main/entities/store/*.js
+  migrations:
+    - ./src/main/migrations/store/*.js
+  cli:
+    migrationsDir: ./src/main/migrations/store

package.json

@@ -15,8 +15,9 @@
     "start": "electron . --enable-logging --dev",
     "rebuild": "electron-rebuild -f -b -t prod,dev,optional",
     "electron-version": "electron electron-version.js",
-    "typeorm:migrate": "npm run typeorm:migrate:library",
+    "typeorm:migrate": "npm run typeorm:migrate:library && npm run typeorm:migrate:store",
     "typeorm:migrate:library": "typeorm migration:run -c library",
+    "typeorm:migrate:store": "typeorm migration:run -c store",
     "build:webpack": "webpack --config webpack.config.js",
     "watch:webpack": "webpack --config webpack.config.js --mode development -w",
     "build:index": "node buildfile.js",

src/main/core/database.ts

@@ -5,6 +5,7 @@ import { appPath } from './app-path';
 
 export enum Databases {
   LIBRARY = 'library',
+  STORE = 'store',
 }
 
 type MyConnectionOptions = { [key in Databases]?: SqliteConnectionOptions };
@@ -13,6 +14,7 @@ const databasePath = path.resolve(appPath, 'database');
 const connectionOptions: MyConnectionOptions = Object.values(Databases).reduce(
   (prev: MyConnectionOptions, database: Databases) => {
     prev[database] = {
+      name: database,
       type: 'sqlite',
       database: path.resolve(databasePath, `${database}.db`),
       cache: true,

src/main/entities/store/store-value.ts

@@ -0,0 +1,19 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+/**
+ * This entity represents a single store value, identified by its key.
+ */
+@Entity()
+export class StoreValue {
+  /**
+   * the key
+   */
+  @PrimaryColumn()
+  public key: StoreKey;
+
+  /**
+   * the value
+   */
+  @Column('simple-json')
+  public value: any;
+}

src/main/migrations/store/1587511036078-initial_migration.ts

@@ -0,0 +1,21 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class initialMigration1587511036078 implements MigrationInterface {
+  name = 'initialMigration1587511036078';
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `CREATE TABLE "store_value" ("key" varchar PRIMARY KEY NOT NULL, "value" text NOT NULL)`,
+      undefined
+    );
+    await queryRunner.query(
+      `CREATE TABLE "query-result-cache" ("id" integer PRIMARY KEY AUTOINCREMENT NOT NULL, "identifier" varchar, "time" bigint NOT NULL, "duration" integer NOT NULL, "query" text NOT NULL, "result" text NOT NULL)`,
+      undefined
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP TABLE "query-result-cache"`, undefined);
+    await queryRunner.query(`DROP TABLE "store_value"`, undefined);
+  }
+}

src/main/modules/store/store-key.d.ts

@@ -0,0 +1,9 @@
+/**
+ * These keys are used by other modules, each module needs to define their key in here.
+ * These values need to be unique as they are used as primary keys in the database.
+ *
+ * When they are changed, a migration to the new key must be included in the commit.
+ */
+declare const enum StoreKey {
+  'COOKIES' = 'cookies',
+}

src/main/modules/store/store.spec.ts

@@ -2,13 +2,8 @@ import rewiremock from 'rewiremock';
 import '../../../../mocks/electron';
 
 import { expect } from 'chai';
-import fs from 'fs-extra';
 import 'mocha';
-import path from 'path';
-import { appPath } from '../../core/app-path';
-import { load, save, StoreKeys } from './store';
-
-const storeDirectory = path.resolve(appPath, 'store');
+import { load, save } from './store';
 
 describe('Store Service', function () {
   this.timeout(10000);
@@ -21,17 +16,6 @@ describe('Store Service', function () {
     rewiremock.disable();
   });
 
-  beforeEach(() => {
-    if (fs.existsSync(storeDirectory)) {
-      fs.removeSync(storeDirectory);
-    }
-  });
-
-  it('creates a store directory', () =>
-    save(StoreKeys.COOKIES, { some: 'data' }).then(() => {
-      expect(fs.existsSync(storeDirectory)).to.be.true;
-    }));
-
   it('loads saved data', () => {
     const testData: any = {
       something: 'gaga',
@@ -48,40 +32,14 @@ describe('Store Service', function () {
       },
     };
     const expectedJson = JSON.stringify(testData);
-    return save(StoreKeys.COOKIES, testData)
-      .then(() => load(StoreKeys.COOKIES))
+    return save(StoreKey.COOKIES, testData)
+      .then(() => load(StoreKey.COOKIES))
       .then((data) => {
         expect(JSON.stringify(data)).to.equal(expectedJson, 'store does not save and load data correctly');
-        return load(StoreKeys.COOKIES);
+        return load(StoreKey.COOKIES);
       })
       .then((data) => {
         expect(JSON.stringify(data)).to.equal(expectedJson, 'store does not load data correctly when loaded twice');
       });
   });
-
-  it('handles a deleted store directory', () => {
-    const testData = 'test_data';
-    return save(StoreKeys.COOKIES, testData)
-      .then(() => {
-        fs.removeSync(storeDirectory);
-        return load(StoreKeys.COOKIES);
-      })
-      .then((data) => {
-        expect(data).to.equal(testData, 'store does not load when store directory is deleted');
-      });
-  });
-
-  it('handles a deleted store file', () => {
-    const testData = 'test_data';
-    return save(StoreKeys.COOKIES, testData)
-      .then(() => {
-        fs.readdirSync(storeDirectory).forEach((file) => {
-          fs.unlinkSync(path.resolve(storeDirectory, file));
-        });
-        return load(StoreKeys.COOKIES);
-      })
-      .then((data) => {
-        expect(data).to.equal(testData, 'store does not load when store files are deleted');
-      });
-  });
 });

src/main/modules/store/store.ts

@@ -1,66 +1,26 @@
-import path from 'path';
-import fs from 'fs-extra';
-import { appPath } from '../../core/app-path';
+import { Databases, getConnection } from '../../core/database';
+import { StoreValue } from '../../entities/store/store-value';
 
-export const enum StoreKeys {
-  'COOKIES' = 'cookies',
+const CACHE_ID = 'store';
+
+export async function load(key: StoreKey): Promise<any> {
+  const c = await getConnection(Databases.STORE);
+  const repository = c.getRepository(StoreValue);
+  const storeValue = await repository.findOne(key, {
+    cache: {
+      id: CACHE_ID,
+      milliseconds: 0,
+    },
+  });
+  return storeValue.value;
 }
 
-type Store = {
-  [key in StoreKeys]?: any;
-};
-
-interface IStoreOptions {
-  path: string;
-}
-
-let store: Store = {};
-let synced = false;
-
-const options: IStoreOptions = {
-  path: path.resolve(appPath, 'store', 'store.json'),
-};
-const folder = path.dirname(options.path);
-
-function initDir(): Promise<void> {
-  if (!fs.existsSync(folder)) {
-    return fs.promises.mkdir(folder, { recursive: true }).then(() => writeUnsafe());
-  }
-  if (!fs.existsSync(options.path)) {
-    return writeUnsafe();
-  }
-  return Promise.resolve();
-}
-
-function writeUnsafe(): Promise<void> {
-  return fs.writeFile(options.path, JSON.stringify(store));
-}
-
-function read(): Promise<void> {
-  return initDir().then(() =>
-    fs.readFile(options.path).then((buf: Buffer) => {
-      store = JSON.parse(buf.toString());
-      synced = true;
-    })
-  );
-}
-
-function write(): Promise<void> {
-  return initDir().then(() =>
-    writeUnsafe().then(() => {
-      synced = false;
-    })
-  );
-}
-
-export function load(key: StoreKeys): Promise<any> {
-  if (synced) {
-    return Promise.resolve(store[key]);
-  }
-  return read().then(() => Promise.resolve(store[key]));
-}
-
-export function save(key: StoreKeys, data: any): Promise<void> {
-  store[key] = data;
-  return write();
+export async function save(key: StoreKey, data: any): Promise<void> {
+  const c = await getConnection(Databases.STORE);
+  const manager = c.manager;
+  const storeValue = new StoreValue();
+  storeValue.key = key;
+  storeValue.value = data;
+  await manager.save(storeValue);
+  await c.queryResultCache.remove([CACHE_ID]);
 }

src/main/modules/web-crawler/web-crawler.ts

@@ -2,7 +2,7 @@ import { injectable } from 'inversify';
 import { CookieJar } from 'jsdom';
 import nodeFetch, { RequestInit, Response } from 'node-fetch';
 import { Errors, RenaiError } from '../../core/error';
-import { load, save, StoreKeys } from '../store/store';
+import { load, save } from '../store/store';
 import { IWebCrawler } from './i-web-crawler';
 
 @injectable()
@@ -40,7 +40,7 @@ export class WebCrawler implements IWebCrawler {
 
   private init(): Promise<void> {
     if (!this.initialized) {
-      return load(StoreKeys.COOKIES).then((cookies: any) => {
+      return load(StoreKey.COOKIES).then((cookies: any) => {
         if (cookies !== undefined) {
           this.cookieJar = CookieJar.deserializeSync(cookies);
         }
@@ -56,7 +56,7 @@ export class WebCrawler implements IWebCrawler {
       header.forEach((cookie: string) => {
         this.cookieJar.setCookieSync(cookie, url);
       });
-      return save(StoreKeys.COOKIES, this.cookieJar.serializeSync()).catch((reason: any) => {
+      return save(StoreKey.COOKIES, this.cookieJar.serializeSync()).catch((reason: any) => {
         throw new RenaiError(Errors.ECOOKIESAVEFAIL, reason);
       });
     }