diff --git a/ormconfig.yml b/ormconfig.yml index bfda4a2..7e8cf3e 100644 --- a/ormconfig.yml +++ b/ormconfig.yml @@ -10,3 +10,14 @@ library: - ./src/main/migrations/library/*.js cli: migrationsDir: ./src/main/migrations/library + +store: + type: sqlite + database: ./test-paths/typeorm/store.db + cache: true + entities: + - ./src/main/entities/store/*.js + migrations: + - ./src/main/migrations/store/*.js + cli: + migrationsDir: ./src/main/migrations/store diff --git a/package.json b/package.json index 89def70..6f8f5d5 100644 --- a/package.json +++ b/package.json @@ -15,8 +15,9 @@ "start": "electron . --enable-logging --dev", "rebuild": "electron-rebuild -f -b -t prod,dev,optional", "electron-version": "electron electron-version.js", - "typeorm:migrate": "npm run typeorm:migrate:library", + "typeorm:migrate": "npm run typeorm:migrate:library && npm run typeorm:migrate:store", "typeorm:migrate:library": "typeorm migration:run -c library", + "typeorm:migrate:store": "typeorm migration:run -c store", "build:webpack": "webpack --config webpack.config.js", "watch:webpack": "webpack --config webpack.config.js --mode development -w", "build:index": "node buildfile.js", diff --git a/src/main/core/database.ts b/src/main/core/database.ts index fef89cc..350f0da 100644 --- a/src/main/core/database.ts +++ b/src/main/core/database.ts @@ -5,6 +5,7 @@ import { appPath } from './app-path'; export enum Databases { LIBRARY = 'library', + STORE = 'store', } type MyConnectionOptions = { [key in Databases]?: SqliteConnectionOptions }; @@ -13,6 +14,7 @@ const databasePath = path.resolve(appPath, 'database'); const connectionOptions: MyConnectionOptions = Object.values(Databases).reduce( (prev: MyConnectionOptions, database: Databases) => { prev[database] = { + name: database, type: 'sqlite', database: path.resolve(databasePath, `${database}.db`), cache: true, diff --git a/src/main/entities/store/store-value.ts b/src/main/entities/store/store-value.ts new file mode 100644 index 0000000..2980ade --- /dev/null +++ b/src/main/entities/store/store-value.ts @@ -0,0 +1,19 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +/** + * This entity represents a single store value, identified by its key. + */ +@Entity() +export class StoreValue { + /** + * the key + */ + @PrimaryColumn() + public key: StoreKey; + + /** + * the value + */ + @Column('simple-json') + public value: any; +} diff --git a/src/main/migrations/store/1587511036078-initial_migration.ts b/src/main/migrations/store/1587511036078-initial_migration.ts new file mode 100644 index 0000000..d75ddf9 --- /dev/null +++ b/src/main/migrations/store/1587511036078-initial_migration.ts @@ -0,0 +1,21 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class initialMigration1587511036078 implements MigrationInterface { + name = 'initialMigration1587511036078'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE "store_value" ("key" varchar PRIMARY KEY NOT NULL, "value" text NOT NULL)`, + undefined + ); + await queryRunner.query( + `CREATE TABLE "query-result-cache" ("id" integer PRIMARY KEY AUTOINCREMENT NOT NULL, "identifier" varchar, "time" bigint NOT NULL, "duration" integer NOT NULL, "query" text NOT NULL, "result" text NOT NULL)`, + undefined + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE "query-result-cache"`, undefined); + await queryRunner.query(`DROP TABLE "store_value"`, undefined); + } +} diff --git a/src/main/modules/store/store-key.d.ts b/src/main/modules/store/store-key.d.ts new file mode 100644 index 0000000..92fcd2d --- /dev/null +++ b/src/main/modules/store/store-key.d.ts @@ -0,0 +1,9 @@ +/** + * These keys are used by other modules, each module needs to define their key in here. + * These values need to be unique as they are used as primary keys in the database. + * + * When they are changed, a migration to the new key must be included in the commit. + */ +declare const enum StoreKey { + 'COOKIES' = 'cookies', +} diff --git a/src/main/modules/store/store.spec.ts b/src/main/modules/store/store.spec.ts index 3095158..d722ab5 100644 --- a/src/main/modules/store/store.spec.ts +++ b/src/main/modules/store/store.spec.ts @@ -2,13 +2,8 @@ import rewiremock from 'rewiremock'; import '../../../../mocks/electron'; import { expect } from 'chai'; -import fs from 'fs-extra'; import 'mocha'; -import path from 'path'; -import { appPath } from '../../core/app-path'; -import { load, save, StoreKeys } from './store'; - -const storeDirectory = path.resolve(appPath, 'store'); +import { load, save } from './store'; describe('Store Service', function () { this.timeout(10000); @@ -21,17 +16,6 @@ describe('Store Service', function () { rewiremock.disable(); }); - beforeEach(() => { - if (fs.existsSync(storeDirectory)) { - fs.removeSync(storeDirectory); - } - }); - - it('creates a store directory', () => - save(StoreKeys.COOKIES, { some: 'data' }).then(() => { - expect(fs.existsSync(storeDirectory)).to.be.true; - })); - it('loads saved data', () => { const testData: any = { something: 'gaga', @@ -48,40 +32,14 @@ describe('Store Service', function () { }, }; const expectedJson = JSON.stringify(testData); - return save(StoreKeys.COOKIES, testData) - .then(() => load(StoreKeys.COOKIES)) + return save(StoreKey.COOKIES, testData) + .then(() => load(StoreKey.COOKIES)) .then((data) => { expect(JSON.stringify(data)).to.equal(expectedJson, 'store does not save and load data correctly'); - return load(StoreKeys.COOKIES); + return load(StoreKey.COOKIES); }) .then((data) => { expect(JSON.stringify(data)).to.equal(expectedJson, 'store does not load data correctly when loaded twice'); }); }); - - it('handles a deleted store directory', () => { - const testData = 'test_data'; - return save(StoreKeys.COOKIES, testData) - .then(() => { - fs.removeSync(storeDirectory); - return load(StoreKeys.COOKIES); - }) - .then((data) => { - expect(data).to.equal(testData, 'store does not load when store directory is deleted'); - }); - }); - - it('handles a deleted store file', () => { - const testData = 'test_data'; - return save(StoreKeys.COOKIES, testData) - .then(() => { - fs.readdirSync(storeDirectory).forEach((file) => { - fs.unlinkSync(path.resolve(storeDirectory, file)); - }); - return load(StoreKeys.COOKIES); - }) - .then((data) => { - expect(data).to.equal(testData, 'store does not load when store files are deleted'); - }); - }); }); diff --git a/src/main/modules/store/store.ts b/src/main/modules/store/store.ts index dd58457..0d51520 100644 --- a/src/main/modules/store/store.ts +++ b/src/main/modules/store/store.ts @@ -1,66 +1,26 @@ -import path from 'path'; -import fs from 'fs-extra'; -import { appPath } from '../../core/app-path'; +import { Databases, getConnection } from '../../core/database'; +import { StoreValue } from '../../entities/store/store-value'; -export const enum StoreKeys { - 'COOKIES' = 'cookies', +const CACHE_ID = 'store'; + +export async function load(key: StoreKey): Promise { + const c = await getConnection(Databases.STORE); + const repository = c.getRepository(StoreValue); + const storeValue = await repository.findOne(key, { + cache: { + id: CACHE_ID, + milliseconds: 0, + }, + }); + return storeValue.value; } -type Store = { - [key in StoreKeys]?: any; -}; - -interface IStoreOptions { - path: string; -} - -let store: Store = {}; -let synced = false; - -const options: IStoreOptions = { - path: path.resolve(appPath, 'store', 'store.json'), -}; -const folder = path.dirname(options.path); - -function initDir(): Promise { - if (!fs.existsSync(folder)) { - return fs.promises.mkdir(folder, { recursive: true }).then(() => writeUnsafe()); - } - if (!fs.existsSync(options.path)) { - return writeUnsafe(); - } - return Promise.resolve(); -} - -function writeUnsafe(): Promise { - return fs.writeFile(options.path, JSON.stringify(store)); -} - -function read(): Promise { - return initDir().then(() => - fs.readFile(options.path).then((buf: Buffer) => { - store = JSON.parse(buf.toString()); - synced = true; - }) - ); -} - -function write(): Promise { - return initDir().then(() => - writeUnsafe().then(() => { - synced = false; - }) - ); -} - -export function load(key: StoreKeys): Promise { - if (synced) { - return Promise.resolve(store[key]); - } - return read().then(() => Promise.resolve(store[key])); -} - -export function save(key: StoreKeys, data: any): Promise { - store[key] = data; - return write(); +export async function save(key: StoreKey, data: any): Promise { + const c = await getConnection(Databases.STORE); + const manager = c.manager; + const storeValue = new StoreValue(); + storeValue.key = key; + storeValue.value = data; + await manager.save(storeValue); + await c.queryResultCache.remove([CACHE_ID]); } diff --git a/src/main/modules/web-crawler/web-crawler.ts b/src/main/modules/web-crawler/web-crawler.ts index 7cd1032..a09a5f1 100644 --- a/src/main/modules/web-crawler/web-crawler.ts +++ b/src/main/modules/web-crawler/web-crawler.ts @@ -2,7 +2,7 @@ import { injectable } from 'inversify'; import { CookieJar } from 'jsdom'; import nodeFetch, { RequestInit, Response } from 'node-fetch'; import { Errors, RenaiError } from '../../core/error'; -import { load, save, StoreKeys } from '../store/store'; +import { load, save } from '../store/store'; import { IWebCrawler } from './i-web-crawler'; @injectable() @@ -40,7 +40,7 @@ export class WebCrawler implements IWebCrawler { private init(): Promise { if (!this.initialized) { - return load(StoreKeys.COOKIES).then((cookies: any) => { + return load(StoreKey.COOKIES).then((cookies: any) => { if (cookies !== undefined) { this.cookieJar = CookieJar.deserializeSync(cookies); } @@ -56,7 +56,7 @@ export class WebCrawler implements IWebCrawler { header.forEach((cookie: string) => { this.cookieJar.setCookieSync(cookie, url); }); - return save(StoreKeys.COOKIES, this.cookieJar.serializeSync()).catch((reason: any) => { + return save(StoreKey.COOKIES, this.cookieJar.serializeSync()).catch((reason: any) => { throw new RenaiError(Errors.ECOOKIESAVEFAIL, reason); }); }