import { injectable } from 'inversify'; import { CookieJar } from 'jsdom'; import nodeFetch, { RequestInit, Response } from 'node-fetch'; import { Errors, RenaiError } from '../../core/error'; import { load, save, StoreKeys } from '../store/store'; import { IWebCrawler } from './i-web-crawler'; @injectable() export class WebCrawler implements IWebCrawler { public cookieJar: CookieJar; private initialized: boolean; public constructor() { this.initialized = false; this.cookieJar = new CookieJar(); } public fetch(url: string, requestInit: RequestInit = {}): Promise { return this.init().then(() => { const cookiedInit = { ...requestInit, ...{ headers: { ...requestInit.headers, ...{ Cookie: this.cookieJar.getCookieStringSync(url), }, }, }, }; return nodeFetch(url, cookiedInit).then((res: Response) => { this.setCookies(res.headers.raw()['set-cookie'], url).catch((reason: any) => { throw new RenaiError(Errors.ECOOKIESAVEFAIL, reason); }); return res; }); }); } private init(): Promise { if (!this.initialized) { return load(StoreKeys.COOKIES).then((cookies: any) => { if (cookies !== undefined) { this.cookieJar = CookieJar.deserializeSync(cookies); } this.initialized = true; }); } else { return Promise.resolve(); } } private setCookies(header: string[], url: string): Promise { if (header) { header.forEach((cookie: string) => { this.cookieJar.setCookieSync(cookie, url); }); return save(StoreKeys.COOKIES, this.cookieJar.serializeSync()).catch((reason: any) => { throw new RenaiError(Errors.ECOOKIESAVEFAIL, reason); }); } return Promise.resolve(); } }