feat: install and use fs-extra instead of fs, implement groundwork for more sophisticated error reporting

also add tests and mocking framework
This commit is contained in:
Xymorot 2019-11-18 22:38:51 +01:00
parent 1392532b7e
commit fce8e95a0e
27 changed files with 276 additions and 64 deletions

5
.gitignore vendored
View File

@ -9,8 +9,13 @@ node_modules
/src/**/*.js.map
/tests/**/*.js
/tests/**/*.js.map
/mocks/**/*.js
/mocks/**/*.js.map
/frontend
# created by testing
/store-backup
# managed by application
/database
/store

View File

@ -20,7 +20,7 @@ This project uses [Conventional Commits](https://www.conventionalcommits.or) wit
- `remove`: removal of existing code/functionality
- `fix`: bugfixes
- `refactor`: code refactoring
- `test`: any of the above, but with tests
- `test`: any of the above, but with tests/mocks
- `update`: updating dependencies and associated code changes
- `config`: changing configuration (linters, build process)
- `doc`: documentation, including comments
@ -52,6 +52,13 @@ The testing framework of choice is [Mocha](https://mochajs.org/). Call `npm run
- HTTP server mocking is done by [nock](https://github.com/nock/nock)
- property based testing is made possible by [fast-check](https://github.com/dubzzz/fast-check)
#### Mocks
There are 2 ways in which mocks are defined/used:
0. for external modules, in [mocks](mocks), uses the [rewiremock](https://github.com/theKashey/rewiremock) package; use this only when there is some magic happening like for electron which normally runs in its own node process
1. for own modules, just beside their test file in [tests](tests); name the file `*.mock.ts` and use other files for orientation; use sparingly and only when not having a mock makes it more complex e.g. for modules which interact with the file system
#### Tagging
Mocha does [not have a seperate tagging feature](https://github.com/mochajs/mocha/wiki/Tagging), but it can filter via title. Us the following tags in your test titles:

View File

@ -4,6 +4,7 @@ const ignoreList = [
/^\/\.nyc_output($|\/)/,
/^\/database($|\/)/,
/^\/declarations($|\/)/,
/^\/mocks($|\/)/,
/^\/node_modules\/\.cache($|\/)/,
/^\/src\/.*\.(ts|js\.map)/,
/^\/store($|\/)/,

20
mocks/electron.ts Normal file
View File

@ -0,0 +1,20 @@
import { rewiremock } from './rewiremock';
import WebContents = Electron.WebContents;
const electronMock: DeepPartial<typeof Electron> = {
app: {
on() {},
},
BrowserWindow: class {
public webContents: DeepPartial<WebContents> = {
openDevTools() {},
};
public loadFile() {}
public on() {}
},
ipcMain: {
on() {},
},
};
rewiremock('electron').with(electronMock);

6
mocks/rewiremock.ts Normal file
View File

@ -0,0 +1,6 @@
import rewiremock from 'rewiremock';
rewiremock.overrideEntryPoint(module);
rewiremock.enable();
export { rewiremock };

6
mocks/tslint.json Normal file
View File

@ -0,0 +1,6 @@
{
"extends": ["../tslint.json"],
"rules": {
"typedef": false
}
}

58
package-lock.json generated
View File

@ -511,6 +511,15 @@
"integrity": "sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g==",
"dev": true
},
"@types/fs-extra": {
"version": "8.0.1",
"resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-8.0.1.tgz",
"integrity": "sha512-J00cVDALmi/hJOYsunyT52Hva5TnJeKP5yd1r+mH/ZU0mbYZflR0Z5kw5kITtKTRYMhm1JMClOFYdHnQszEvqw==",
"dev": true,
"requires": {
"@types/node": "*"
}
},
"@types/glob": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.1.1.tgz",
@ -2176,6 +2185,12 @@
"integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs=",
"dev": true
},
"compare-module-exports": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/compare-module-exports/-/compare-module-exports-2.1.0.tgz",
"integrity": "sha512-3Lc0sTIuX1jmY2K2RrXRJOND6KsRTX2D4v3+eu1PDptsuJZVK4LZc852eZa9I+avj0NrUKlTNgqvccNOH6mbGg==",
"dev": true
},
"compare-version": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/compare-version/-/compare-version-0.1.2.tgz",
@ -4180,7 +4195,6 @@
"version": "8.1.0",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz",
"integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==",
"dev": true,
"requires": {
"graceful-fs": "^4.2.0",
"jsonfile": "^4.0.0",
@ -5061,10 +5075,9 @@
}
},
"graceful-fs": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.0.tgz",
"integrity": "sha512-jpSvDPV4Cq/bgtpndIWbI5hmYxhQGHPC4d4cqBPb4DLniCfhJokdXhwhaDuLBGLQdvvRum/UiX6ECVIPvDXqdg==",
"dev": true
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.2.tgz",
"integrity": "sha512-IItsdsea19BoLC7ELy13q1iJFNmd7ofZH5+X/pJr90/nRoPEX0DJo1dHDbgtYWOhJhcCgMDTOw84RZ72q6lB+Q=="
},
"grapheme-splitter": {
"version": "1.0.4",
@ -6285,7 +6298,6 @@
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz",
"integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=",
"dev": true,
"requires": {
"graceful-fs": "^4.1.6"
}
@ -8763,6 +8775,22 @@
"integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==",
"dev": true
},
"rewiremock": {
"version": "3.13.9",
"resolved": "https://registry.npmjs.org/rewiremock/-/rewiremock-3.13.9.tgz",
"integrity": "sha512-FDk5uCyvfwgYZtZ9MKdpg6QiSSdjB/a/vU5luKjoJddaqcZz5+u4dXhc3Qf4vNMvDXvnOyodNd1riE5yeqoxaA==",
"dev": true,
"requires": {
"babel-runtime": "^6.26.0",
"compare-module-exports": "^2.1.0",
"lodash.some": "^4.6.0",
"lodash.template": "^4.4.0",
"node-libs-browser": "^2.1.0",
"path-parse": "^1.0.5",
"wipe-node-cache": "^2.1.0",
"wipe-webpack-cache": "^2.1.0"
}
},
"rgb2hex": {
"version": "0.1.9",
"resolved": "https://registry.npmjs.org/rgb2hex/-/rgb2hex-0.1.9.tgz",
@ -10388,8 +10416,7 @@
"universalify": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",
"integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==",
"dev": true
"integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg=="
},
"unset-value": {
"version": "1.0.0",
@ -11121,6 +11148,21 @@
"string-width": "^1.0.2 || 2"
}
},
"wipe-node-cache": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/wipe-node-cache/-/wipe-node-cache-2.1.0.tgz",
"integrity": "sha512-Vdash0WV9Di/GeYW9FJrAZcPjGK4dO7M/Be/sJybguEgcM7As0uwLyvewZYqdlepoh7Rj4ZJKEdo8uX83PeNIw==",
"dev": true
},
"wipe-webpack-cache": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/wipe-webpack-cache/-/wipe-webpack-cache-2.1.0.tgz",
"integrity": "sha512-OXzQMGpA7MnQQ8AG+uMl5mWR2ezy6fw1+DMHY+wzYP1qkF1jrek87psLBmhZEj+er4efO/GD4R8jXWFierobaA==",
"dev": true,
"requires": {
"wipe-node-cache": "^2.1.0"
}
},
"wordwrap": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz",

View File

@ -18,19 +18,22 @@
"coverage:fast": "nyc npm run test:fast",
"coverage": "nyc npm run test",
"lint": "npm run eslint && npm run tslint",
"eslint-check": "eslint --print-config gulpfile.js | eslint-config-prettier-check",
"eslint:check": "eslint --print-config gulpfile.js | eslint-config-prettier-check",
"eslint": "eslint .",
"tslint-check": "tslint-config-prettier-check ./tslint.json",
"tslint": "tslint -t stylish -p tsconfig.json",
"eslint:fix": "eslint . --fix",
"tslint:check": "tslint-config-prettier-check ./tslint.json",
"tslint": "tslint -p tsconfig.json -t stylish",
"tslint:fix": "tslint -p tsconfig.json --fix",
"prettier": "prettier --ignore-path .gitignore -c **/*.{html,handlebars,json,{c,sc,sa,le}ss,yml,svelte,md,ts,js}",
"prettier:write": "prettier --ignore-path .gitignore --write **/*.{html,handlebars,json,{c,sc,sa,le}ss,yml,svelte,md,ts,js}",
"prettier:fix": "prettier --ignore-path .gitignore --write **/*.{html,handlebars,json,{c,sc,sa,le}ss,yml,svelte,md,ts,js}",
"forge:start": "electron-forge start",
"forge:make": "electron-forge --platform win32 --arch x64 make",
"forge": "npm run build && npm run forge:make",
"precommit": "npm run prettier && npm run eslint-check && npm run tslint-check && npm run lint && npm run coverage:fast",
"precommit": "npm run prettier && npm run eslint:check && npm run tslint:check && npm run lint && npm run coverage:fast",
"prepush": "npm run coverage"
},
"dependencies": {
"fs-extra": "^8.1.0",
"jsdom": "^15.1.1",
"node-fetch": "^2.6.0",
"sqlite3": "^4.1.0",
@ -41,6 +44,7 @@
"@electron-forge/cli": "^6.0.0-beta.45",
"@electron-forge/maker-squirrel": "^6.0.0-beta.45",
"@types/chai": "^4.2.3",
"@types/fs-extra": "^8.0.1",
"@types/gulp": "^4.0.6",
"@types/jsdom": "latest",
"@types/minimist": "latest",
@ -65,6 +69,7 @@
"nock": "^11.4.0",
"nyc": "^14.1.1",
"prettier": "latest",
"rewiremock": "^3.13.9",
"sinon": "^7.5.0",
"spectron": "^8.0.0",
"svelte": "^3.12.1",

View File

@ -6,7 +6,7 @@ import './main/services/database';
import * as session from './main/services/session';
import BrowserWindowConstructorOptions = Electron.BrowserWindowConstructorOptions;
let mainWindow: Electron.BrowserWindow;
export let mainWindow: Electron.BrowserWindow;
async function createWindow(): Promise<void> {
session.setHeaders();

View File

@ -1,8 +1,9 @@
import { ipcMain } from 'electron';
import { isLoggedIn, login } from '../services/nhentai-crawler';
import IpcMainEvent = Electron.IpcMainEvent;
import { mainWindow } from '../../main';
import { isLoggedIn, login } from '../services/nhentai-crawler';
const ipcServer: IIpcServer = {
export const ipcServer: IIpcServer = {
answer: (channel: IpcChannels, handler: (data?: any) => Promise<any>): void => {
ipcMain.on(channel, (event: IpcMainEvent, payload: IIpcPayload) => {
handler(payload.data)
@ -24,6 +25,9 @@ const ipcServer: IIpcServer = {
});
});
},
send: (channel: IpcChannels, data: any): void => {
mainWindow.webContents.send(channel, data);
},
};
ipcServer.answer(IpcChannels.LOGIN, (credentials: ICredentials) => {

View File

@ -17,7 +17,7 @@ Object.values(Databases).forEach((database: Databases) => {
return connection.runMigrations();
})
.catch((reason: any) => {
throwError(reason);
throwError(reason, true);
});
});

View File

@ -1,9 +1,12 @@
// if you would like to add some logging or something similar, start here
import { ipcServer } from '../controllers/api';
export function throwError(error: any): void {
if (error instanceof Error) {
throw error;
} else {
throw new Error(error);
export function throwError(error: any, isFatal: boolean = false): void {
let errorInstance = error;
if (!(errorInstance instanceof Error)) {
errorInstance = new Error(error);
}
if (isFatal) {
throw errorInstance;
}
ipcServer.send(IpcChannels.ERROR, errorInstance);
}

View File

@ -1,4 +1,4 @@
import fs, { promises as fsp } from 'fs';
import fs from 'fs-extra';
import path from 'path';
export const enum StoreKeys {
@ -23,7 +23,7 @@ const folder = path.dirname(options.path);
function initDir(): Promise<void> {
if (!fs.existsSync(folder)) {
return fsp.mkdir(folder).then(() => writeUnsave());
return fs.promises.mkdir(folder, { recursive: true }).then(() => writeUnsave());
}
if (!fs.existsSync(options.path)) {
return writeUnsave();
@ -32,12 +32,12 @@ function initDir(): Promise<void> {
}
function writeUnsave(): Promise<void> {
return fsp.writeFile(options.path, JSON.stringify(store));
return fs.writeFile(options.path, JSON.stringify(store));
}
function read(): Promise<void> {
return initDir().then(() => {
return fsp.readFile(options.path).then((buf: Buffer) => {
return fs.readFile(options.path).then((buf: Buffer) => {
store = JSON.parse(buf.toString());
synced = true;
});

View File

@ -1,44 +1,43 @@
import { CookieJar } from 'jsdom';
import nodeFetch, { RequestInit, Response } from 'node-fetch';
import { Errors, RenaiError } from '../../types/error';
import { throwError } from './error';
import { load, save, StoreKeys } from './store';
export let cookieJar: CookieJar = new CookieJar();
let error: Error;
let initialized = false;
function init(): void {
load(StoreKeys.COOKIES)
.then((cookies: any) => {
function init(): Promise<void> {
if (!initialized) {
return load(StoreKeys.COOKIES).then((cookies: any) => {
if (cookies !== undefined) {
cookieJar = CookieJar.deserializeSync(cookies);
}
})
.catch((reason: any) => {
error = new RenaiError(Errors.EINITFAIL, reason);
initialized = true;
});
} else {
return Promise.resolve();
}
}
export function fetch(url: string, requestInit: RequestInit = {}): Promise<Response> {
if (error !== undefined) {
return Promise.reject(error);
}
const cookiedInit = {
...requestInit,
...{
headers: {
...requestInit.headers,
...{
Cookie: cookieJar.getCookieStringSync(url),
return init().then(() => {
const cookiedInit = {
...requestInit,
...{
headers: {
...requestInit.headers,
...{
Cookie: cookieJar.getCookieStringSync(url),
},
},
},
},
};
return nodeFetch(url, cookiedInit).then((res: Response) => {
setCookies(res.headers.raw()['set-cookie'], url).catch((reason: any) => {
error = new Error(reason);
};
return nodeFetch(url, cookiedInit).then((res: Response) => {
setCookies(res.headers.raw()['set-cookie'], url).catch();
return res;
});
return res;
});
}
@ -47,9 +46,9 @@ function setCookies(header: string[], url: string): Promise<void> {
header.forEach((cookie: string) => {
cookieJar.setCookieSync(cookie, url);
});
return save(StoreKeys.COOKIES, cookieJar.serializeSync());
return save(StoreKeys.COOKIES, cookieJar.serializeSync()).catch((reason: any) => {
throwError(new RenaiError(Errors.ECOOKIESAVEFAIL, reason));
});
}
return Promise.resolve();
}
init();

View File

@ -21,8 +21,8 @@ const ipcClient: IIpcClient = {
ipcRenderer.removeListener(channel, listener);
}
};
ipcRenderer.send(channel, payload);
ipcRenderer.on(channel, listener);
ipcRenderer.send(channel, payload);
});
},
};

View File

@ -0,0 +1,5 @@
type DeepPartial<T> = {
[P in keyof T]?: T[P] extends Array<infer U>
? Array<DeepPartial<U>>
: (T[P] extends ReadonlyArray<infer V> ? ReadonlyArray<DeepPartial<V>> : DeepPartial<T[P]>);
};

View File

@ -1,19 +1,19 @@
export const enum Errors {
ERROR = 'ERROR',
ENOLOGIN = 'ENOLOGIN',
ELOGINFAIL = 'ELOGINFAIL',
EINITFAIL = 'EINITFAIL',
ECOOKIESAVEFAIL = 'ECOOKIESAVEFAIL',
}
const messages = {
[Errors.ERROR]: 'generic error',
[Errors.ENOLOGIN]: 'no login form found',
[Errors.ELOGINFAIL]: 'login failed',
[Errors.EINITFAIL]: 'initialization failed',
[Errors.ECOOKIESAVEFAIL]: 'failed to save cookies',
};
export class RenaiError extends Error {
constructor(eno: Errors = Errors.ERROR, msg: string = '') {
constructor(eno: Errors, msg: string = '') {
super(`${messages[eno]}.${msg ? ` ${msg}` : ''}`);
}
}

View File

@ -1,4 +1,5 @@
const enum IpcChannels {
ERROR = 'ERROR',
LOGIN = 'LOGIN',
LOGGED_IN = 'LOGGED_IN',
}
@ -26,4 +27,5 @@ interface IIpcClient {
interface IIpcServer {
answer: (channel: IpcChannels, handler: (data?: any) => Promise<any>) => void;
send: (channel: IpcChannels, data: any) => void;
}

View File

@ -1,6 +1,9 @@
import rewiremock from 'rewiremock';
rewiremock.disable();
import { expect } from 'chai';
import * as electron from 'electron';
import { after, before, describe, it } from 'mocha';
import 'mocha';
import { Application } from 'spectron';
import packageJson from '../package.json';

View File

@ -0,0 +1,33 @@
const store = require('../../../src/main/services/store');
import { load, save } from '../../../src/main/services/store';
interface IStoreMock extends IMock {
original: {
load: typeof load;
save: typeof save;
};
mock: {
load: (mock: typeof load) => void;
save: (mock: typeof save) => void;
};
restore: () => void;
}
export const storeMock: IStoreMock = {
original: {
load: store.load,
save: store.save,
},
mock: {
load(mock) {
store.load = mock;
},
save(mock) {
store.save = mock;
},
},
restore() {
store.load = this.original.load;
store.save = this.original.save;
},
};

View File

@ -0,0 +1,40 @@
import rewiremock from 'rewiremock';
import '../../../mocks/electron';
import { expect } from 'chai';
import fs from 'fs-extra';
import 'mocha';
import path from 'path';
import { save, StoreKeys } from '../../../src/main/services/store';
const storeDirectory = path.resolve('store');
const storeBackupDirectory = path.resolve('store-backup');
describe('Store Service', function() {
this.timeout(10000);
before(() => {
rewiremock.enable();
if (fs.existsSync(storeDirectory)) {
fs.removeSync(storeBackupDirectory);
fs.moveSync(storeDirectory, storeBackupDirectory);
}
});
after(() => {
rewiremock.disable();
if (fs.existsSync(storeBackupDirectory)) {
fs.removeSync(storeDirectory);
fs.moveSync(storeBackupDirectory, storeDirectory);
}
});
it('creates a store directory', () => {
if (fs.existsSync(storeDirectory)) {
fs.removeSync(storeDirectory);
}
return save(StoreKeys.COOKIES, { some: 'data' }).then(() => {
expect(fs.existsSync(storeDirectory)).to.be.true;
});
});
});

View File

@ -1,13 +1,30 @@
import rewiremock from 'rewiremock';
import '../../../mocks/electron';
import { expect } from 'chai';
import { afterEach, beforeEach, describe, it } from 'mocha';
import { CookieJar } from 'jsdom';
import 'mocha';
import nock from 'nock';
import { Response } from 'node-fetch';
import sinon from 'sinon';
import { fetch } from '../../../src/main/services/web-crawler';
import { storeMock } from './store.mock';
describe('Web Crawler', function() {
this.timeout(2000);
before(() => {
rewiremock.enable();
storeMock.mock.load(() => {
return Promise.resolve(new CookieJar().serializeSync());
});
storeMock.mock.save(() => {
return Promise.resolve();
});
});
beforeEach(() => {
if (!nock.isActive()) {
nock.activate();
@ -18,6 +35,11 @@ describe('Web Crawler', function() {
nock.cleanAll();
});
after(() => {
rewiremock.disable();
storeMock.restore();
});
it('fetches websites', async () => {
const callback = sinon.spy();

7
tests/mock.ts Normal file
View File

@ -0,0 +1,7 @@
interface IMock {
original: object;
mock: {
[key: string]: (mock: any) => void;
};
restore: () => void;
}

View File

@ -1,6 +1,6 @@
import { expect } from 'chai';
import fc from 'fast-check';
import { describe, it } from 'mocha';
import 'mocha';
import { c, s, t } from '../../../src/renderer/services/utils';
describe('Frontend Utils', function() {

View File

@ -1,5 +1,5 @@
import { expect } from 'chai';
import { describe, it } from 'mocha';
import 'mocha';
import { uuid } from '../../src/services/uuid';
describe('UUID Service', function() {

View File

@ -2,6 +2,8 @@
"extends": ["../tslint.json"],
"rules": {
"no-magic-numbers": false,
"typedef": false
"typedef": false,
"no-unused-expression": false,
"no-var-requires": false
}
}

View File

@ -12,5 +12,5 @@
"emitDecoratorMetadata": true,
"lib": ["es2018", "dom"]
},
"include": ["declarations/**/*.ts", "src/**/*.ts", "tests/**/*.ts"]
"include": ["declarations/**/*.ts", "src/**/*.ts", "tests/**/*.ts", "mocks/**/*.ts"]
}