From 82a97d0f406942e7acc8ffe8b9450f4ededfa186 Mon Sep 17 00:00:00 2001 From: Xymorot Date: Mon, 4 Jan 2021 23:40:15 +0100 Subject: [PATCH] fix: implement mutex for the nhentai app window so multiple calls to getting the favorites just do the thing one after another This commit also fixes some other bugs and cleans up related code. --- .../modules/app-window/i-site-app-window.d.ts | 8 ++ .../modules/app-window/site-app-window.ts | 27 ++++ src/main/modules/mutex/i-mutex.d.ts | 19 +++ src/main/modules/mutex/simple-mutex.spec.ts | 88 ++++++++++++ src/main/modules/mutex/simple-mutex.ts | 52 +++++++ .../modules/nhentai/i-nhentai-app-window.d.ts | 4 +- .../modules/nhentai/nhentai-app-window.ts | 129 ++++++++---------- .../modules/nhentai/nhentai-ipc-controller.ts | 2 +- src/main/modules/nhentai/nhentai-util.ts | 4 +- 9 files changed, 254 insertions(+), 79 deletions(-) create mode 100644 src/main/modules/app-window/i-site-app-window.d.ts create mode 100644 src/main/modules/app-window/site-app-window.ts create mode 100644 src/main/modules/mutex/i-mutex.d.ts create mode 100644 src/main/modules/mutex/simple-mutex.spec.ts create mode 100644 src/main/modules/mutex/simple-mutex.ts diff --git a/src/main/modules/app-window/i-site-app-window.d.ts b/src/main/modules/app-window/i-site-app-window.d.ts new file mode 100644 index 0000000..bff88b9 --- /dev/null +++ b/src/main/modules/app-window/i-site-app-window.d.ts @@ -0,0 +1,8 @@ +import { IUrlAppWindow } from './i-url-app-window'; + +export interface ISiteAppWindow extends IUrlAppWindow { + /** + * @see IMutex.acquire + */ + acquireLock(): Promise; +} diff --git a/src/main/modules/app-window/site-app-window.ts b/src/main/modules/app-window/site-app-window.ts new file mode 100644 index 0000000..1309c66 --- /dev/null +++ b/src/main/modules/app-window/site-app-window.ts @@ -0,0 +1,27 @@ +import { BrowserWindowConstructorOptions, LoadURLOptions } from 'electron'; +import { SimpleMutex } from '../mutex/simple-mutex'; +import { ISessionHelper } from '../session/i-session-helper'; +import { ISiteAppWindow } from './i-site-app-window'; +import { UrlAppWindow } from './url-app-window'; + +/** + * This class represents an app window of a site which need to be crawled via the built-in chromium of Electron. + * It offers a lock so that multiple calls do executed simultaneously on the same chromium window. + */ +export abstract class SiteAppWindow extends UrlAppWindow implements ISiteAppWindow { + private windowLock: IMutex; + + protected constructor( + sessionHelper: ISessionHelper, + uri: string, + options: BrowserWindowConstructorOptions = {}, + loadOptions: LoadURLOptions = {} + ) { + super(sessionHelper, uri, options, loadOptions); + this.windowLock = new SimpleMutex(); + } + + public acquireLock(): Promise { + return this.windowLock.acquire(); + } +} diff --git a/src/main/modules/mutex/i-mutex.d.ts b/src/main/modules/mutex/i-mutex.d.ts new file mode 100644 index 0000000..b6a9fe4 --- /dev/null +++ b/src/main/modules/mutex/i-mutex.d.ts @@ -0,0 +1,19 @@ +/** + * A mutex (mutual exclusion) is a lock which enforces limited access to a resource in asynchronous/multi-threaded environments. + * + * Acquiring this lock returns a release function function (via promise) which needs to be called to release the lock again. + */ +interface IMutex { + /** + * acquires the lock and returns a Promise with the release function to be called when the lock shall be released. + * This release function needs to be called or the lock will never release and execution of subsequent consumers will not take place. + * Always think about possible error states and release the lock accordingly. + */ + acquire(): Promise; + + isLocked(): boolean; +} + +declare namespace Mutex { + type ReleaseFunction = () => void; +} diff --git a/src/main/modules/mutex/simple-mutex.spec.ts b/src/main/modules/mutex/simple-mutex.spec.ts new file mode 100644 index 0000000..0aecb93 --- /dev/null +++ b/src/main/modules/mutex/simple-mutex.spec.ts @@ -0,0 +1,88 @@ +import { promisify } from 'util'; +import { expect } from 'chai'; +import { describe } from 'mocha'; +import { SimpleMutex } from './simple-mutex'; + +describe('Simple Mutex', () => { + it('executes code after acquiring lock', () => { + const mutex = new SimpleMutex(); + + return mutex.acquire().then(() => { + const x = 1; + expect(x).eq(1); + }); + }); + + it('correctly locks a resource', async () => { + const mutex = new SimpleMutex(); + + let inUse = false; + + const useResource = () => { + expect(inUse).to.be.false; + inUse = true; + return promisify(setTimeout)(1).then(() => { + inUse = false; + }); + }; + + const acquireOne = mutex.acquire(); + const acquireTwo = mutex.acquire(); + const acquireThree = mutex.acquire(); + void acquireOne.then((release) => + useResource().then(() => { + release(); + }) + ); + void acquireTwo.then((release) => + useResource().then(() => { + release(); + }) + ); + return acquireThree.then((release) => + useResource().then(() => { + release(); + }) + ); + }); + + it('executes consumers in the right order', async () => { + const mutex = new SimpleMutex(); + + let counter = 0; + + const useResource = (expectValue: number) => { + counter++; + expect(counter).to.equal(expectValue); + return promisify(setTimeout)(1); + }; + + const acquireOne = mutex.acquire(); + const acquireTwo = mutex.acquire(); + const acquireThree = mutex.acquire(); + void acquireOne.then((release) => + useResource(1).then(() => { + release(); + }) + ); + void acquireTwo.then((release) => + useResource(2).then(() => { + release(); + }) + ); + return acquireThree.then((release) => + useResource(3).then(() => { + release(); + }) + ); + }); + + it('correctly informs if it is currently locked', async () => { + const mutex = new SimpleMutex(); + + const release = await mutex.acquire(); + expect(mutex.isLocked()).to.be.true; + release(); + expect(mutex.isLocked()).to.be.false; + }); +}); diff --git a/src/main/modules/mutex/simple-mutex.ts b/src/main/modules/mutex/simple-mutex.ts new file mode 100644 index 0000000..f982ff5 --- /dev/null +++ b/src/main/modules/mutex/simple-mutex.ts @@ -0,0 +1,52 @@ +/** + * This class implements a simple mutex using a semaphore. + */ +export class SimpleMutex implements IMutex { + /** + * This queue is an array of promise resolve functions. + * Calling them signals the corresponding consumer that the lock is now free. + * The resolve functions resolve with the release function. + */ + private queue: Array<(release: Mutex.ReleaseFunction) => void> = []; + + /** + * This variable is the semaphore, true if locked, false if not. + */ + private locked: boolean = false; + + public acquire(): Promise { + return new Promise((resolve) => { + this.queue.push(resolve); + this.dispatch(); + }); + } + + public isLocked(): boolean { + return this.locked; + } + + /** + * This function locks the mutex and calls the next resolve function from the start of the queue. + * The resolve function is called with a release function as parameter which unlocks the mutex and calls the function again + */ + private dispatch(): void { + if (this.locked) { + return; + } + const nextResolve = this.queue.shift(); + if (!nextResolve) { + return; + } + this.locked = true; + + // this is the function which gets called by the consumer to release the lock + // it also dispatches the next consumer (recursive call) + const releaseFunction = (): void => { + this.locked = false; + this.dispatch(); + }; + + // lock the resource and resolve the promise so the consumer can do its thing + nextResolve(releaseFunction); + } +} diff --git a/src/main/modules/nhentai/i-nhentai-app-window.d.ts b/src/main/modules/nhentai/i-nhentai-app-window.d.ts index 4c40cc3..f1409b6 100644 --- a/src/main/modules/nhentai/i-nhentai-app-window.d.ts +++ b/src/main/modules/nhentai/i-nhentai-app-window.d.ts @@ -1,5 +1,5 @@ -import { IUrlAppWindow } from '../app-window/i-url-app-window'; +import { ISiteAppWindow } from '../app-window/i-site-app-window'; -export interface INhentaiAppWindow extends IUrlAppWindow { +export interface INhentaiAppWindow extends ISiteAppWindow { getFavorites(): Promise; } diff --git a/src/main/modules/nhentai/nhentai-app-window.ts b/src/main/modules/nhentai/nhentai-app-window.ts index acb4d8f..899c700 100644 --- a/src/main/modules/nhentai/nhentai-app-window.ts +++ b/src/main/modules/nhentai/nhentai-app-window.ts @@ -6,7 +6,7 @@ import { URL } from 'url'; import { createReadStream, remove } from 'fs-extra'; import { injectable } from 'inversify'; import { inject } from '../../core/inject'; -import { UrlAppWindow } from '../app-window/url-app-window'; +import { SiteAppWindow } from '../app-window/site-app-window'; import { WindowClosedError } from '../app-window/window-closed-error'; import { ISessionHelper } from '../session/i-session-helper'; import { INhentaiAppWindow } from './i-nhentai-app-window'; @@ -25,51 +25,52 @@ import { const waitInterval = 2000; @injectable() -export class NhentaiAppWindow extends UrlAppWindow implements INhentaiAppWindow { +export class NhentaiAppWindow extends SiteAppWindow implements INhentaiAppWindow { public constructor(@inject('session-helper') sessionHelper: ISessionHelper) { super(sessionHelper, nhentaiUrl); } - /** - * @throws WindowClosedError when this._window is null - */ public async getFavorites(): Promise { - const error = new WindowClosedError(); - - if (this.isClosed()) { - await this.open(); - } - - if (!(await this.isLoggedIn())) { - await this.login(); - } - - if (!this._window) { - throw error; - } - - const bookUrls: string[] = []; - for await (const wc of this.getFavoritePageWebContentsGenerator()) { - bookUrls.push( - ...((await wc.executeJavaScript( - `Array.from(document.querySelectorAll('${coverLinkSelector}')).map((el) => el.href)` - )) as string[]) - ); - } - - const readable = Readable.from( - (async function* (thisArg): AsyncGenerator { - for (const bookUrl of bookUrls) { - yield await thisArg.getBookTorrent(bookUrl); - } - })(this), - { - objectMode: true, + const release = await this.acquireLock(); + try { + if (this.isClosed()) { + await this.open(); } - ); - readable.once('close', this.close); - return readable; + if (!(await this.isLoggedIn())) { + await this.login(); + } + + const bookUrls: string[] = []; + for await (const wc of this.getFavoritePageWebContentsGenerator()) { + bookUrls.push( + ...((await wc.executeJavaScript( + `Array.from(document.querySelectorAll('${coverLinkSelector}')).map((el) => el.href)` + )) as string[]) + ); + } + + const readable = Readable.from( + (async function* (thisArg): AsyncGenerator { + for (const bookUrl of bookUrls) { + yield await thisArg.getBookTorrent(bookUrl); + } + return; + })(this), + { + objectMode: true, + } + ); + readable.once('end', () => { + this.close(); + release(); + }); + + return readable; + } catch (e) { + release(); + throw e; + } } protected getCsp(): IContentSecurityPolicy { @@ -101,9 +102,6 @@ export class NhentaiAppWindow extends UrlAppWindow implements INhentaiAppWindow } } - /** - * @throws WindowClosedError when this._window is null - */ private async isLoggedIn(): Promise { if (!this._window) { throw new WindowClosedError(); @@ -141,42 +139,25 @@ export class NhentaiAppWindow extends UrlAppWindow implements INhentaiAppWindow } private async *getFavoritePageWebContentsGenerator(): AsyncGenerator { - const error = new WindowClosedError(); if (!this._window) { - throw error; + throw new WindowClosedError(); } - await this.loadUrlSafe(getFavoritePageUrl(1)); - yield this._window.webContents; - do { - try { - const hasNextPage = (await this._window.webContents.executeJavaScript( - `!!document.querySelector('${nextFavoritePageSelector}')` - )) as boolean; - if (hasNextPage) { - yield new Promise((resolve) => { - if (this._window) { - this._window.webContents.once('did-finish-load', () => { - if (this._window) { - resolve(this._window.webContents); - } else { - throw error; - } - }); - void this._window.webContents.executeJavaScript( - `document.querySelector('${nextFavoritePageSelector}').click()` - ); - } else { - throw error; - } - }); - } else { - break; - } - } catch { + await this.loadUrlSafe(getFavoritePageUrl()); + while (true) { + yield this._window.webContents; + const hasNextPage = (await this._window.webContents.executeJavaScript( + `!!document.querySelector('${nextFavoritePageSelector}')` + )) as boolean; + if (hasNextPage) { + const nextPageHref = (await this._window.webContents.executeJavaScript( + `document.querySelector('${nextFavoritePageSelector}').href` + )) as string; + await this.loadUrlSafe(nextPageHref); + } else { break; } - } while (true); - return undefined; + } + return; } private async getBookTorrent(bookUrl: string): Promise { diff --git a/src/main/modules/nhentai/nhentai-ipc-controller.ts b/src/main/modules/nhentai/nhentai-ipc-controller.ts index 000fe62..f99e6b9 100644 --- a/src/main/modules/nhentai/nhentai-ipc-controller.ts +++ b/src/main/modules/nhentai/nhentai-ipc-controller.ts @@ -38,7 +38,7 @@ export class NhentaiIpcController implements IIpcController { favorite.torrentFile.pipe(writable); }); - favoritesStream.once('close', resolve); + favoritesStream.once('end', resolve); }); } diff --git a/src/main/modules/nhentai/nhentai-util.ts b/src/main/modules/nhentai/nhentai-util.ts index 5970354..9eccd36 100644 --- a/src/main/modules/nhentai/nhentai-util.ts +++ b/src/main/modules/nhentai/nhentai-util.ts @@ -12,8 +12,8 @@ export const downloadLinkId = 'download'; export const nextFavoritePageSelector = 'a.next'; export const coverLinkSelector = 'a.cover'; -export function getFavoritePageUrl(page: number): string { - return `${url + paths.favorites}?page=${page}`; +export function getFavoritePageUrl(page?: number): string { + return `${url + paths.favorites}${page ? `?page=${page}` : ''}`; } export function getGalleryId(bookUrl: string): string {