66 lines
1.9 KiB
TypeScript
66 lines
1.9 KiB
TypeScript
import { injectable } from 'inversify';
|
|
import { CookieJar } from 'jsdom';
|
|
import nodeFetch, { RequestInit, Response } from 'node-fetch';
|
|
import { Errors, RenaiError } from '../../core/error';
|
|
import { load, save, StoreKeys } from '../store/store';
|
|
import { IWebCrawler } from './i-web-crawler';
|
|
|
|
@injectable()
|
|
export class WebCrawler implements IWebCrawler {
|
|
public cookieJar: CookieJar;
|
|
|
|
private initialized: boolean;
|
|
|
|
public constructor() {
|
|
this.initialized = false;
|
|
this.cookieJar = new CookieJar();
|
|
}
|
|
|
|
public fetch(url: string, requestInit: RequestInit = {}): Promise<Response> {
|
|
return this.init().then(() => {
|
|
const cookiedInit = {
|
|
...requestInit,
|
|
...{
|
|
headers: {
|
|
...requestInit.headers,
|
|
...{
|
|
Cookie: this.cookieJar.getCookieStringSync(url),
|
|
},
|
|
},
|
|
},
|
|
};
|
|
return nodeFetch(url, cookiedInit).then((res: Response) => {
|
|
this.setCookies(res.headers.raw()['set-cookie'], url).catch((reason: any) => {
|
|
throw new RenaiError(Errors.ECOOKIESAVEFAIL, reason);
|
|
});
|
|
return res;
|
|
});
|
|
});
|
|
}
|
|
|
|
private init(): Promise<void> {
|
|
if (!this.initialized) {
|
|
return load(StoreKeys.COOKIES).then((cookies: any) => {
|
|
if (cookies !== undefined) {
|
|
this.cookieJar = CookieJar.deserializeSync(cookies);
|
|
}
|
|
this.initialized = true;
|
|
});
|
|
} else {
|
|
return Promise.resolve();
|
|
}
|
|
}
|
|
|
|
private setCookies(header: string[], url: string): Promise<void> {
|
|
if (header) {
|
|
header.forEach((cookie: string) => {
|
|
this.cookieJar.setCookieSync(cookie, url);
|
|
});
|
|
return save(StoreKeys.COOKIES, this.cookieJar.serializeSync()).catch((reason: any) => {
|
|
throw new RenaiError(Errors.ECOOKIESAVEFAIL, reason);
|
|
});
|
|
}
|
|
return Promise.resolve();
|
|
}
|
|
}
|