refactor: use a new database for the store
BREAKING CHANGE: old file-based store data is lost
This commit is contained in:
parent
9e5abaeb42
commit
288deee56f
|
@ -10,3 +10,14 @@ library:
|
||||||
- ./src/main/migrations/library/*.js
|
- ./src/main/migrations/library/*.js
|
||||||
cli:
|
cli:
|
||||||
migrationsDir: ./src/main/migrations/library
|
migrationsDir: ./src/main/migrations/library
|
||||||
|
|
||||||
|
store:
|
||||||
|
type: sqlite
|
||||||
|
database: ./test-paths/typeorm/store.db
|
||||||
|
cache: true
|
||||||
|
entities:
|
||||||
|
- ./src/main/entities/store/*.js
|
||||||
|
migrations:
|
||||||
|
- ./src/main/migrations/store/*.js
|
||||||
|
cli:
|
||||||
|
migrationsDir: ./src/main/migrations/store
|
||||||
|
|
|
@ -15,8 +15,9 @@
|
||||||
"start": "electron . --enable-logging --dev",
|
"start": "electron . --enable-logging --dev",
|
||||||
"rebuild": "electron-rebuild -f -b -t prod,dev,optional",
|
"rebuild": "electron-rebuild -f -b -t prod,dev,optional",
|
||||||
"electron-version": "electron electron-version.js",
|
"electron-version": "electron electron-version.js",
|
||||||
"typeorm:migrate": "npm run typeorm:migrate:library",
|
"typeorm:migrate": "npm run typeorm:migrate:library && npm run typeorm:migrate:store",
|
||||||
"typeorm:migrate:library": "typeorm migration:run -c library",
|
"typeorm:migrate:library": "typeorm migration:run -c library",
|
||||||
|
"typeorm:migrate:store": "typeorm migration:run -c store",
|
||||||
"build:webpack": "webpack --config webpack.config.js",
|
"build:webpack": "webpack --config webpack.config.js",
|
||||||
"watch:webpack": "webpack --config webpack.config.js --mode development -w",
|
"watch:webpack": "webpack --config webpack.config.js --mode development -w",
|
||||||
"build:index": "node buildfile.js",
|
"build:index": "node buildfile.js",
|
||||||
|
|
|
@ -5,6 +5,7 @@ import { appPath } from './app-path';
|
||||||
|
|
||||||
export enum Databases {
|
export enum Databases {
|
||||||
LIBRARY = 'library',
|
LIBRARY = 'library',
|
||||||
|
STORE = 'store',
|
||||||
}
|
}
|
||||||
|
|
||||||
type MyConnectionOptions = { [key in Databases]?: SqliteConnectionOptions };
|
type MyConnectionOptions = { [key in Databases]?: SqliteConnectionOptions };
|
||||||
|
@ -13,6 +14,7 @@ const databasePath = path.resolve(appPath, 'database');
|
||||||
const connectionOptions: MyConnectionOptions = Object.values(Databases).reduce(
|
const connectionOptions: MyConnectionOptions = Object.values(Databases).reduce(
|
||||||
(prev: MyConnectionOptions, database: Databases) => {
|
(prev: MyConnectionOptions, database: Databases) => {
|
||||||
prev[database] = {
|
prev[database] = {
|
||||||
|
name: database,
|
||||||
type: 'sqlite',
|
type: 'sqlite',
|
||||||
database: path.resolve(databasePath, `${database}.db`),
|
database: path.resolve(databasePath, `${database}.db`),
|
||||||
cache: true,
|
cache: true,
|
||||||
|
|
|
@ -0,0 +1,19 @@
|
||||||
|
import { Column, Entity, PrimaryColumn } from 'typeorm';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This entity represents a single store value, identified by its key.
|
||||||
|
*/
|
||||||
|
@Entity()
|
||||||
|
export class StoreValue {
|
||||||
|
/**
|
||||||
|
* the key
|
||||||
|
*/
|
||||||
|
@PrimaryColumn()
|
||||||
|
public key: StoreKey;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* the value
|
||||||
|
*/
|
||||||
|
@Column('simple-json')
|
||||||
|
public value: any;
|
||||||
|
}
|
|
@ -0,0 +1,21 @@
|
||||||
|
import { MigrationInterface, QueryRunner } from 'typeorm';
|
||||||
|
|
||||||
|
export class initialMigration1587511036078 implements MigrationInterface {
|
||||||
|
name = 'initialMigration1587511036078';
|
||||||
|
|
||||||
|
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||||
|
await queryRunner.query(
|
||||||
|
`CREATE TABLE "store_value" ("key" varchar PRIMARY KEY NOT NULL, "value" text NOT NULL)`,
|
||||||
|
undefined
|
||||||
|
);
|
||||||
|
await queryRunner.query(
|
||||||
|
`CREATE TABLE "query-result-cache" ("id" integer PRIMARY KEY AUTOINCREMENT NOT NULL, "identifier" varchar, "time" bigint NOT NULL, "duration" integer NOT NULL, "query" text NOT NULL, "result" text NOT NULL)`,
|
||||||
|
undefined
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||||
|
await queryRunner.query(`DROP TABLE "query-result-cache"`, undefined);
|
||||||
|
await queryRunner.query(`DROP TABLE "store_value"`, undefined);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,9 @@
|
||||||
|
/**
|
||||||
|
* These keys are used by other modules, each module needs to define their key in here.
|
||||||
|
* These values need to be unique as they are used as primary keys in the database.
|
||||||
|
*
|
||||||
|
* When they are changed, a migration to the new key must be included in the commit.
|
||||||
|
*/
|
||||||
|
declare const enum StoreKey {
|
||||||
|
'COOKIES' = 'cookies',
|
||||||
|
}
|
|
@ -2,13 +2,8 @@ import rewiremock from 'rewiremock';
|
||||||
import '../../../../mocks/electron';
|
import '../../../../mocks/electron';
|
||||||
|
|
||||||
import { expect } from 'chai';
|
import { expect } from 'chai';
|
||||||
import fs from 'fs-extra';
|
|
||||||
import 'mocha';
|
import 'mocha';
|
||||||
import path from 'path';
|
import { load, save } from './store';
|
||||||
import { appPath } from '../../core/app-path';
|
|
||||||
import { load, save, StoreKeys } from './store';
|
|
||||||
|
|
||||||
const storeDirectory = path.resolve(appPath, 'store');
|
|
||||||
|
|
||||||
describe('Store Service', function () {
|
describe('Store Service', function () {
|
||||||
this.timeout(10000);
|
this.timeout(10000);
|
||||||
|
@ -21,17 +16,6 @@ describe('Store Service', function () {
|
||||||
rewiremock.disable();
|
rewiremock.disable();
|
||||||
});
|
});
|
||||||
|
|
||||||
beforeEach(() => {
|
|
||||||
if (fs.existsSync(storeDirectory)) {
|
|
||||||
fs.removeSync(storeDirectory);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
it('creates a store directory', () =>
|
|
||||||
save(StoreKeys.COOKIES, { some: 'data' }).then(() => {
|
|
||||||
expect(fs.existsSync(storeDirectory)).to.be.true;
|
|
||||||
}));
|
|
||||||
|
|
||||||
it('loads saved data', () => {
|
it('loads saved data', () => {
|
||||||
const testData: any = {
|
const testData: any = {
|
||||||
something: 'gaga',
|
something: 'gaga',
|
||||||
|
@ -48,40 +32,14 @@ describe('Store Service', function () {
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
const expectedJson = JSON.stringify(testData);
|
const expectedJson = JSON.stringify(testData);
|
||||||
return save(StoreKeys.COOKIES, testData)
|
return save(StoreKey.COOKIES, testData)
|
||||||
.then(() => load(StoreKeys.COOKIES))
|
.then(() => load(StoreKey.COOKIES))
|
||||||
.then((data) => {
|
.then((data) => {
|
||||||
expect(JSON.stringify(data)).to.equal(expectedJson, 'store does not save and load data correctly');
|
expect(JSON.stringify(data)).to.equal(expectedJson, 'store does not save and load data correctly');
|
||||||
return load(StoreKeys.COOKIES);
|
return load(StoreKey.COOKIES);
|
||||||
})
|
})
|
||||||
.then((data) => {
|
.then((data) => {
|
||||||
expect(JSON.stringify(data)).to.equal(expectedJson, 'store does not load data correctly when loaded twice');
|
expect(JSON.stringify(data)).to.equal(expectedJson, 'store does not load data correctly when loaded twice');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('handles a deleted store directory', () => {
|
|
||||||
const testData = 'test_data';
|
|
||||||
return save(StoreKeys.COOKIES, testData)
|
|
||||||
.then(() => {
|
|
||||||
fs.removeSync(storeDirectory);
|
|
||||||
return load(StoreKeys.COOKIES);
|
|
||||||
})
|
|
||||||
.then((data) => {
|
|
||||||
expect(data).to.equal(testData, 'store does not load when store directory is deleted');
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it('handles a deleted store file', () => {
|
|
||||||
const testData = 'test_data';
|
|
||||||
return save(StoreKeys.COOKIES, testData)
|
|
||||||
.then(() => {
|
|
||||||
fs.readdirSync(storeDirectory).forEach((file) => {
|
|
||||||
fs.unlinkSync(path.resolve(storeDirectory, file));
|
|
||||||
});
|
|
||||||
return load(StoreKeys.COOKIES);
|
|
||||||
})
|
|
||||||
.then((data) => {
|
|
||||||
expect(data).to.equal(testData, 'store does not load when store files are deleted');
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|
|
@ -1,66 +1,26 @@
|
||||||
import path from 'path';
|
import { Databases, getConnection } from '../../core/database';
|
||||||
import fs from 'fs-extra';
|
import { StoreValue } from '../../entities/store/store-value';
|
||||||
import { appPath } from '../../core/app-path';
|
|
||||||
|
|
||||||
export const enum StoreKeys {
|
const CACHE_ID = 'store';
|
||||||
'COOKIES' = 'cookies',
|
|
||||||
|
export async function load(key: StoreKey): Promise<any> {
|
||||||
|
const c = await getConnection(Databases.STORE);
|
||||||
|
const repository = c.getRepository(StoreValue);
|
||||||
|
const storeValue = await repository.findOne(key, {
|
||||||
|
cache: {
|
||||||
|
id: CACHE_ID,
|
||||||
|
milliseconds: 0,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return storeValue.value;
|
||||||
}
|
}
|
||||||
|
|
||||||
type Store = {
|
export async function save(key: StoreKey, data: any): Promise<void> {
|
||||||
[key in StoreKeys]?: any;
|
const c = await getConnection(Databases.STORE);
|
||||||
};
|
const manager = c.manager;
|
||||||
|
const storeValue = new StoreValue();
|
||||||
interface IStoreOptions {
|
storeValue.key = key;
|
||||||
path: string;
|
storeValue.value = data;
|
||||||
}
|
await manager.save(storeValue);
|
||||||
|
await c.queryResultCache.remove([CACHE_ID]);
|
||||||
let store: Store = {};
|
|
||||||
let synced = false;
|
|
||||||
|
|
||||||
const options: IStoreOptions = {
|
|
||||||
path: path.resolve(appPath, 'store', 'store.json'),
|
|
||||||
};
|
|
||||||
const folder = path.dirname(options.path);
|
|
||||||
|
|
||||||
function initDir(): Promise<void> {
|
|
||||||
if (!fs.existsSync(folder)) {
|
|
||||||
return fs.promises.mkdir(folder, { recursive: true }).then(() => writeUnsafe());
|
|
||||||
}
|
|
||||||
if (!fs.existsSync(options.path)) {
|
|
||||||
return writeUnsafe();
|
|
||||||
}
|
|
||||||
return Promise.resolve();
|
|
||||||
}
|
|
||||||
|
|
||||||
function writeUnsafe(): Promise<void> {
|
|
||||||
return fs.writeFile(options.path, JSON.stringify(store));
|
|
||||||
}
|
|
||||||
|
|
||||||
function read(): Promise<void> {
|
|
||||||
return initDir().then(() =>
|
|
||||||
fs.readFile(options.path).then((buf: Buffer) => {
|
|
||||||
store = JSON.parse(buf.toString());
|
|
||||||
synced = true;
|
|
||||||
})
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function write(): Promise<void> {
|
|
||||||
return initDir().then(() =>
|
|
||||||
writeUnsafe().then(() => {
|
|
||||||
synced = false;
|
|
||||||
})
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function load(key: StoreKeys): Promise<any> {
|
|
||||||
if (synced) {
|
|
||||||
return Promise.resolve(store[key]);
|
|
||||||
}
|
|
||||||
return read().then(() => Promise.resolve(store[key]));
|
|
||||||
}
|
|
||||||
|
|
||||||
export function save(key: StoreKeys, data: any): Promise<void> {
|
|
||||||
store[key] = data;
|
|
||||||
return write();
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,7 +2,7 @@ import { injectable } from 'inversify';
|
||||||
import { CookieJar } from 'jsdom';
|
import { CookieJar } from 'jsdom';
|
||||||
import nodeFetch, { RequestInit, Response } from 'node-fetch';
|
import nodeFetch, { RequestInit, Response } from 'node-fetch';
|
||||||
import { Errors, RenaiError } from '../../core/error';
|
import { Errors, RenaiError } from '../../core/error';
|
||||||
import { load, save, StoreKeys } from '../store/store';
|
import { load, save } from '../store/store';
|
||||||
import { IWebCrawler } from './i-web-crawler';
|
import { IWebCrawler } from './i-web-crawler';
|
||||||
|
|
||||||
@injectable()
|
@injectable()
|
||||||
|
@ -40,7 +40,7 @@ export class WebCrawler implements IWebCrawler {
|
||||||
|
|
||||||
private init(): Promise<void> {
|
private init(): Promise<void> {
|
||||||
if (!this.initialized) {
|
if (!this.initialized) {
|
||||||
return load(StoreKeys.COOKIES).then((cookies: any) => {
|
return load(StoreKey.COOKIES).then((cookies: any) => {
|
||||||
if (cookies !== undefined) {
|
if (cookies !== undefined) {
|
||||||
this.cookieJar = CookieJar.deserializeSync(cookies);
|
this.cookieJar = CookieJar.deserializeSync(cookies);
|
||||||
}
|
}
|
||||||
|
@ -56,7 +56,7 @@ export class WebCrawler implements IWebCrawler {
|
||||||
header.forEach((cookie: string) => {
|
header.forEach((cookie: string) => {
|
||||||
this.cookieJar.setCookieSync(cookie, url);
|
this.cookieJar.setCookieSync(cookie, url);
|
||||||
});
|
});
|
||||||
return save(StoreKeys.COOKIES, this.cookieJar.serializeSync()).catch((reason: any) => {
|
return save(StoreKey.COOKIES, this.cookieJar.serializeSync()).catch((reason: any) => {
|
||||||
throw new RenaiError(Errors.ECOOKIESAVEFAIL, reason);
|
throw new RenaiError(Errors.ECOOKIESAVEFAIL, reason);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue