Sharkey/packages/backend/src/core/HttpRequestService.ts

162 lines
3.8 KiB
TypeScript
Raw Normal View History

2022-09-17 21:27:08 +03:00
import * as http from 'node:http';
import * as https from 'node:https';
import CacheableLookup from 'cacheable-lookup';
import fetch from 'node-fetch';
2022-09-17 21:27:08 +03:00
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
import { Inject, Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
2022-09-20 23:33:11 +03:00
import type { Config } from '@/config.js';
2022-09-17 21:27:08 +03:00
import { StatusError } from '@/misc/status-error.js';
import { bindThis } from '@/decorators.js';
import type { Response } from 'node-fetch';
import type { URL } from 'node:url';
2022-09-17 21:27:08 +03:00
@Injectable()
export class HttpRequestService {
2022-09-17 21:27:08 +03:00
/**
* Get http non-proxy agent
2022-09-17 21:27:08 +03:00
*/
private http: http.Agent;
2022-09-17 21:27:08 +03:00
/**
* Get https non-proxy agent
2022-09-17 21:27:08 +03:00
*/
private https: https.Agent;
2022-09-17 21:27:08 +03:00
/**
* Get http proxy or non-proxy agent
2022-09-17 21:27:08 +03:00
*/
public httpAgent: http.Agent;
/**
* Get https proxy or non-proxy agent
*/
2022-09-17 21:27:08 +03:00
public httpsAgent: https.Agent;
constructor(
@Inject(DI.config)
private config: Config,
) {
const cache = new CacheableLookup({
2022-09-17 21:27:08 +03:00
maxTtl: 3600, // 1hours
errorTtl: 30, // 30secs
lookup: false, // nativeのdns.lookupにfallbackしない
});
2022-09-18 21:11:50 +03:00
this.http = new http.Agent({
2022-09-17 21:27:08 +03:00
keepAlive: true,
keepAliveMsecs: 30 * 1000,
lookup: cache.lookup,
2022-09-17 21:27:08 +03:00
} as http.AgentOptions);
2022-09-18 21:11:50 +03:00
this.https = new https.Agent({
2022-09-17 21:27:08 +03:00
keepAlive: true,
keepAliveMsecs: 30 * 1000,
lookup: cache.lookup,
2022-09-17 21:27:08 +03:00
} as https.AgentOptions);
const maxSockets = Math.max(256, config.deliverJobConcurrency ?? 128);
2022-09-17 21:27:08 +03:00
this.httpAgent = config.proxy
? new HttpProxyAgent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
maxSockets,
2022-09-17 21:27:08 +03:00
maxFreeSockets: 256,
scheduling: 'lifo',
proxy: config.proxy,
})
2022-09-18 21:11:50 +03:00
: this.http;
2022-09-17 21:27:08 +03:00
this.httpsAgent = config.proxy
? new HttpsProxyAgent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
maxSockets,
2022-09-17 21:27:08 +03:00
maxFreeSockets: 256,
scheduling: 'lifo',
proxy: config.proxy,
})
2022-09-18 21:11:50 +03:00
: this.https;
2022-09-17 21:27:08 +03:00
}
/**
* Get agent by URL
2022-09-17 21:27:08 +03:00
* @param url URL
* @param bypassProxy Allways bypass proxy
*/
@bindThis
public getAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent {
2022-09-17 21:27:08 +03:00
if (bypassProxy || (this.config.proxyBypassHosts || []).includes(url.hostname)) {
2022-09-18 21:11:50 +03:00
return url.protocol === 'http:' ? this.http : this.https;
2022-09-17 21:27:08 +03:00
} else {
return url.protocol === 'http:' ? this.httpAgent : this.httpsAgent;
}
}
@bindThis
2023-02-01 13:13:22 +02:00
public async getJson<T = unknown>(url: string, accept = 'application/json, */*', headers?: Record<string, string>): Promise<T> {
const res = await this.send(url, {
method: 'GET',
headers: Object.assign({
Accept: accept,
}, headers ?? {}),
timeout: 5000,
size: 1024 * 256,
});
2023-02-01 13:13:22 +02:00
return await res.json() as T;
}
@bindThis
public async getHtml(url: string, accept = 'text/html, */*', headers?: Record<string, string>): Promise<string> {
const res = await this.send(url, {
method: 'GET',
headers: Object.assign({
Accept: accept,
}, headers ?? {}),
timeout: 5000,
});
return await res.text();
}
2022-09-17 21:27:08 +03:00
@bindThis
public async send(url: string, args: {
method?: string,
body?: string,
headers?: Record<string, string>,
timeout?: number,
size?: number,
} = {}, extra: {
throwErrorWhenResponseNotOk: boolean;
} = {
throwErrorWhenResponseNotOk: true,
}): Promise<Response> {
const timeout = args.timeout ?? 5000;
const controller = new AbortController();
setTimeout(() => {
controller.abort();
}, timeout);
const res = await fetch(url, {
method: args.method ?? 'GET',
headers: {
'User-Agent': this.config.userAgent,
...(args.headers ?? {})
},
body: args.body,
size: args.size ?? 10 * 1024 * 1024,
agent: (url) => this.getAgentByUrl(url),
signal: controller.signal,
});
2022-09-17 21:27:08 +03:00
if (!res.ok && extra.throwErrorWhenResponseNotOk) {
throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
}
2022-09-17 21:27:08 +03:00
return res;
2022-09-17 21:27:08 +03:00
}
}