perf(backend): Use undici instead of node-fetch and got (#9459)

* Implement? HttpFetchService

* ✌️

* remove node-fetch

* fix

* refactor

* fix

* gateway timeout

* UndiciFetcherクラスを追加 (仮コミット, ビルドもstartもさせていない)

* fix

* add logger and fix url preview

* fix ip check

* enhance logger and error handling

* fix

* fix

* clean up

* Use custom fetcher for ApRequest / ApResolver

* bypassProxyはproxyBypassHostsに判断を委譲するように

* set maxRedirections (default 3, ApRequest/ApResolver: 0)

* fix comment

* handle error s3 upload

* add debug message

* no return await

* Revert "no return await"

This reverts commit b5b0dc58a342393d260492e3a6f58304372f53b2.

* reduce maxSockets

* apResolverのUndiciFetcherを廃止しapRequestのものを使う、 add ap logger

* Revert "apResolverのUndiciFetcherを廃止しapRequestのものを使う、 add ap logger"

This reverts commit 997243915c8e1f8472da64f607f88c36cb1d5cb4.

* add logger

* fix

* change logger name

* safe

* デフォルトでUser-Agentを設定
This commit is contained in:
tamaina 2023-01-12 21:03:02 +09:00 committed by GitHub
parent 2470afaa2e
commit 978a9bbb3b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 444 additions and 255 deletions

View file

@ -122,10 +122,12 @@ id: 'aid'
# Proxy for HTTP/HTTPS # Proxy for HTTP/HTTPS
#proxy: http://127.0.0.1:3128 #proxy: http://127.0.0.1:3128
#proxyBypassHosts: [ proxyBypassHosts:
# 'example.com', - api.deepl.com
# '192.0.2.8' - api-free.deepl.com
#] - www.recaptcha.net
- hcaptcha.com
- challenges.cloudflare.com
# Proxy for SMTP/SMTPS # Proxy for SMTP/SMTPS
#proxySmtp: http://127.0.0.1:3128 # use HTTP/1.1 CONNECT #proxySmtp: http://127.0.0.1:3128 # use HTTP/1.1 CONNECT

View file

@ -77,7 +77,6 @@
"misskey-js": "0.0.14", "misskey-js": "0.0.14",
"ms": "3.0.0-canary.1", "ms": "3.0.0-canary.1",
"nested-property": "4.0.0", "nested-property": "4.0.0",
"node-fetch": "3.3.0",
"nodemailer": "6.8.0", "nodemailer": "6.8.0",
"nsfwjs": "2.4.2", "nsfwjs": "2.4.2",
"oauth": "^0.10.0", "oauth": "^0.10.0",
@ -118,6 +117,7 @@
"twemoji-parser": "14.0.0", "twemoji-parser": "14.0.0",
"typeorm": "0.3.11", "typeorm": "0.3.11",
"ulid": "2.3.0", "ulid": "2.3.0",
"undici": "^5.14.0",
"unzipper": "0.10.11", "unzipper": "0.10.11",
"uuid": "9.0.0", "uuid": "9.0.0",
"vary": "1.1.2", "vary": "1.1.2",
@ -180,6 +180,7 @@
"execa": "6.1.0", "execa": "6.1.0",
"jest": "29.3.1", "jest": "29.3.1",
"jest-mock": "^29.3.1", "jest-mock": "^29.3.1",
"node-fetch": "3.3.0",
"typescript": "4.9.4" "typescript": "4.9.4"
} }
} }

View file

@ -1,7 +1,4 @@
import { Inject, Injectable } from '@nestjs/common'; import { Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
import type { UsersRepository } from '@/models/index.js';
import type { Config } from '@/config.js';
import { HttpRequestService } from '@/core/HttpRequestService.js'; import { HttpRequestService } from '@/core/HttpRequestService.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
@ -13,9 +10,6 @@ type CaptchaResponse = {
@Injectable() @Injectable()
export class CaptchaService { export class CaptchaService {
constructor( constructor(
@Inject(DI.config)
private config: Config,
private httpRequestService: HttpRequestService, private httpRequestService: HttpRequestService,
) { ) {
} }
@ -27,16 +21,16 @@ export class CaptchaService {
response, response,
}); });
const res = await fetch(url, { const res = await this.httpRequestService.fetch(
method: 'POST', url,
body: params, {
headers: { method: 'POST',
'User-Agent': this.config.userAgent, body: params,
}, },
// TODO {
//timeout: 10 * 1000, noOkError: true,
agent: (url, bypassProxy) => this.httpRequestService.getAgentByUrl(url, bypassProxy), }
}).catch(err => { ).catch(err => {
throw `${err.message ?? err}`; throw `${err.message ?? err}`;
}); });

View file

@ -8,11 +8,12 @@ import got, * as Got from 'got';
import chalk from 'chalk'; import chalk from 'chalk';
import { DI } from '@/di-symbols.js'; import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js'; import type { Config } from '@/config.js';
import { HttpRequestService } from '@/core/HttpRequestService.js'; import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js';
import { createTemp } from '@/misc/create-temp.js'; import { createTemp } from '@/misc/create-temp.js';
import { StatusError } from '@/misc/status-error.js'; import { StatusError } from '@/misc/status-error.js';
import { LoggerService } from '@/core/LoggerService.js'; import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js'; import type Logger from '@/logger.js';
import { buildConnector } from 'undici';
const pipeline = util.promisify(stream.pipeline); const pipeline = util.promisify(stream.pipeline);
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
@ -20,6 +21,7 @@ import { bindThis } from '@/decorators.js';
@Injectable() @Injectable()
export class DownloadService { export class DownloadService {
private logger: Logger; private logger: Logger;
private undiciFetcher: UndiciFetcher;
constructor( constructor(
@Inject(DI.config) @Inject(DI.config)
@ -29,70 +31,42 @@ export class DownloadService {
private loggerService: LoggerService, private loggerService: LoggerService,
) { ) {
this.logger = this.loggerService.getLogger('download'); this.logger = this.loggerService.getLogger('download');
this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption(
{
connect: process.env.NODE_ENV === 'development' ?
this.httpRequestService.clientDefaults.connect
:
this.httpRequestService.getConnectorWithIpCheck(
buildConnector({
...this.httpRequestService.clientDefaults.connect,
}),
(ip) => !this.isPrivateIp(ip)
),
bodyTimeout: 30 * 1000,
},
{
connect: this.httpRequestService.clientDefaults.connect,
}
), this.logger);
} }
@bindThis @bindThis
public async downloadUrl(url: string, path: string): Promise<void> { public async downloadUrl(url: string, path: string): Promise<void> {
this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`); this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
const timeout = 30 * 1000; const timeout = 30 * 1000;
const operationTimeout = 60 * 1000; const operationTimeout = 60 * 1000;
const maxSize = this.config.maxFileSize ?? 262144000; const maxSize = this.config.maxFileSize ?? 262144000;
const req = got.stream(url, { const response = await this.undiciFetcher.fetch(url);
headers: {
'User-Agent': this.config.userAgent, if (response.body === null) {
}, throw new StatusError('No body', 400, 'No body');
timeout: {
lookup: timeout,
connect: timeout,
secureConnect: timeout,
socket: timeout, // read timeout
response: timeout,
send: timeout,
request: operationTimeout, // whole operation timeout
},
agent: {
http: this.httpRequestService.httpAgent,
https: this.httpRequestService.httpsAgent,
},
http2: false, // default
retry: {
limit: 0,
},
}).on('response', (res: Got.Response) => {
if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !this.config.proxy && res.ip) {
if (this.isPrivateIp(res.ip)) {
this.logger.warn(`Blocked address: ${res.ip}`);
req.destroy();
}
}
const contentLength = res.headers['content-length'];
if (contentLength != null) {
const size = Number(contentLength);
if (size > maxSize) {
this.logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`);
req.destroy();
}
}
}).on('downloadProgress', (progress: Got.Progress) => {
if (progress.transferred > maxSize) {
this.logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`);
req.destroy();
}
});
try {
await pipeline(req, fs.createWriteStream(path));
} catch (e) {
if (e instanceof Got.HTTPError) {
throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage);
} else {
throw e;
}
} }
await pipeline(stream.Readable.fromWeb(response.body), fs.createWriteStream(path));
this.logger.succ(`Download finished: ${chalk.cyan(url)}`); this.logger.succ(`Download finished: ${chalk.cyan(url)}`);
} }
@ -114,7 +88,7 @@ export class DownloadService {
cleanup(); cleanup();
} }
} }
@bindThis @bindThis
private isPrivateIp(ip: string): boolean { private isPrivateIp(ip: string): boolean {
for (const net of this.config.allowedPrivateNetworks ?? []) { for (const net of this.config.allowedPrivateNetworks ?? []) {
@ -124,6 +98,6 @@ export class DownloadService {
} }
} }
return PrivateIp(ip); return PrivateIp(ip) ?? false;
} }
} }

View file

@ -375,8 +375,19 @@ export class DriveService {
partSize: s3.endpoint.hostname === 'storage.googleapis.com' ? 500 * 1024 * 1024 : 8 * 1024 * 1024, partSize: s3.endpoint.hostname === 'storage.googleapis.com' ? 500 * 1024 * 1024 : 8 * 1024 * 1024,
}); });
const result = await upload.promise(); await upload.promise()
if (result) this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`); .then(
result => {
if (result) {
this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`);
} else {
this.registerLogger.error(`Upload Result Empty: key = ${key}, filename = ${filename}`);
}
},
err => {
this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err);
}
);
} }
@bindThis @bindThis
@ -462,6 +473,8 @@ export class DriveService {
} }
} }
this.registerLogger.debug(`ADD DRIVE FILE: user ${user?.id ?? 'not set'}, name ${detectedName}, tmp ${path}`);
//#region Check drive usage //#region Check drive usage
if (user && !isLink) { if (user && !isLink) {
const usage = await this.driveFileEntityService.calcDriveUsageOf(user); const usage = await this.driveFileEntityService.calcDriveUsageOf(user);

View file

@ -1,7 +1,6 @@
import { URL } from 'node:url'; import { URL } from 'node:url';
import { Inject, Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import { JSDOM } from 'jsdom'; import { JSDOM } from 'jsdom';
import fetch from 'node-fetch';
import tinycolor from 'tinycolor2'; import tinycolor from 'tinycolor2';
import type { Instance } from '@/models/entities/Instance.js'; import type { Instance } from '@/models/entities/Instance.js';
import type { InstancesRepository } from '@/models/index.js'; import type { InstancesRepository } from '@/models/index.js';
@ -191,11 +190,7 @@ export class FetchInstanceMetadataService {
const faviconUrl = url + '/favicon.ico'; const faviconUrl = url + '/favicon.ico';
const favicon = await fetch(faviconUrl, { const favicon = await this.httpRequestService.fetch(faviconUrl, {}, { noOkError: true });
// TODO
//timeout: 10000,
agent: url => this.httpRequestService.getAgentByUrl(url),
});
if (favicon.ok) { if (favicon.ok) {
return faviconUrl; return faviconUrl;

View file

@ -1,67 +1,257 @@
import * as http from 'node:http'; import * as http from 'node:http';
import * as https from 'node:https'; import * as https from 'node:https';
import CacheableLookup from 'cacheable-lookup'; import CacheableLookup from 'cacheable-lookup';
import fetch from 'node-fetch';
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'; import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
import { Inject, Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js'; import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js'; import type { Config } from '@/config.js';
import { StatusError } from '@/misc/status-error.js'; import { StatusError } from '@/misc/status-error.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import type { Response } from 'node-fetch'; import * as undici from 'undici';
import type { URL } from 'node:url'; import { LookupFunction } from 'node:net';
import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
// true to allow, false to deny
export type IpChecker = (ip: string) => boolean;
/*
* Child class to create and save Agent for fetch.
* You should construct this when you want
* to change timeout, size limit, socket connect function, etc.
*/
export class UndiciFetcher {
/**
* Get http non-proxy agent (undici)
*/
public nonProxiedAgent: undici.Agent;
/**
* Get http proxy or non-proxy agent (undici)
*/
public agent: undici.ProxyAgent | undici.Agent;
private proxyBypassHosts: string[];
private userAgent: string | undefined;
private logger: Logger | undefined;
constructor(
args: {
agentOptions: undici.Agent.Options;
proxy?: {
uri: string;
options?: undici.Agent.Options; // Override of agentOptions
},
proxyBypassHosts?: string[];
userAgent?: string;
},
logger?: Logger,
) {
this.logger = logger;
this.logger?.debug('UndiciFetcher constructor', args);
this.proxyBypassHosts = args.proxyBypassHosts ?? [];
this.userAgent = args.userAgent;
this.nonProxiedAgent = new undici.Agent({
...args.agentOptions,
connect: (process.env.NODE_ENV !== 'production' && typeof args.agentOptions.connect !== 'function')
? (options, cb) => {
// Custom connector for debug
undici.buildConnector(args.agentOptions.connect as undici.buildConnector.BuildOptions)(options, (err, socket) => {
this.logger?.debug('Socket connector called', socket);
if (err) {
this.logger?.debug(`Socket error`, err);
cb(new Error(`Error while socket connecting\n${err}`), null);
return;
}
this.logger?.debug(`Socket connected: port ${socket.localPort} => remote ${socket.remoteAddress}`);
cb(null, socket);
});
} : args.agentOptions.connect,
});
this.agent = args.proxy
? new undici.ProxyAgent({
...args.agentOptions,
...args.proxy.options,
uri: args.proxy.uri,
connect: (process.env.NODE_ENV !== 'production' && typeof (args.proxy?.options?.connect ?? args.agentOptions.connect) !== 'function')
? (options, cb) => {
// Custom connector for debug
undici.buildConnector((args.proxy?.options?.connect ?? args.agentOptions.connect) as undici.buildConnector.BuildOptions)(options, (err, socket) => {
this.logger?.debug('Socket connector called (secure)', socket);
if (err) {
this.logger?.debug(`Socket error`, err);
cb(new Error(`Error while socket connecting\n${err}`), null);
return;
}
this.logger?.debug(`Socket connected (secure): port ${socket.localPort} => remote ${socket.remoteAddress}`);
cb(null, socket);
});
} : (args.proxy?.options?.connect ?? args.agentOptions.connect),
})
: this.nonProxiedAgent;
}
/**
* Get agent by URL
* @param url URL
* @param bypassProxy Allways bypass proxy
*/
@bindThis
public getAgentByUrl(url: URL, bypassProxy = false): undici.Agent | undici.ProxyAgent {
if (bypassProxy || this.proxyBypassHosts.includes(url.hostname)) {
return this.nonProxiedAgent;
} else {
return this.agent;
}
}
@bindThis
public async fetch(
url: string | URL,
options: undici.RequestInit = {},
privateOptions: { noOkError?: boolean; bypassProxy?: boolean; } = { noOkError: false, bypassProxy: false }
): Promise<undici.Response> {
const res = await undici.fetch(url, {
dispatcher: this.getAgentByUrl(new URL(url), privateOptions.bypassProxy),
...options,
headers: {
'User-Agent': this.userAgent ?? '',
...(options.headers ?? {}),
},
}).catch((err) => {
this.logger?.error('fetch error', err);
throw new StatusError('Resource Unreachable', 500, 'Resource Unreachable');
});
if (!res.ok && !privateOptions.noOkError) {
throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
}
return res;
}
@bindThis
public async getJson<T extends unknown>(url: string, accept = 'application/json, */*', headers?: Record<string, string>): Promise<T> {
const res = await this.fetch(
url,
{
headers: Object.assign({
Accept: accept,
}, headers ?? {}),
}
);
return await res.json() as T;
}
@bindThis
public async getHtml(url: string, accept = 'text/html, */*', headers?: Record<string, string>): Promise<string> {
const res = await this.fetch(
url,
{
headers: Object.assign({
Accept: accept,
}, headers ?? {}),
}
);
return await res.text();
}
}
@Injectable() @Injectable()
export class HttpRequestService { export class HttpRequestService {
/** public defaultFetcher: UndiciFetcher;
* Get http non-proxy agent public fetch: UndiciFetcher['fetch'];
*/ public getHtml: UndiciFetcher['getHtml'];
public defaultJsonFetcher: UndiciFetcher;
public getJson: UndiciFetcher['getJson'];
//#region for old http/https, only used in S3Service
// http non-proxy agent
private http: http.Agent; private http: http.Agent;
/** // https non-proxy agent
* Get https non-proxy agent
*/
private https: https.Agent; private https: https.Agent;
/** // http proxy or non-proxy agent
* Get http proxy or non-proxy agent
*/
public httpAgent: http.Agent; public httpAgent: http.Agent;
/** // https proxy or non-proxy agent
* Get https proxy or non-proxy agent
*/
public httpsAgent: https.Agent; public httpsAgent: https.Agent;
//#endregion
public readonly dnsCache: CacheableLookup;
public readonly clientDefaults: undici.Agent.Options;
private maxSockets: number;
private logger: Logger;
constructor( constructor(
@Inject(DI.config) @Inject(DI.config)
private config: Config, private config: Config,
private loggerService: LoggerService,
) { ) {
const cache = new CacheableLookup({ this.logger = this.loggerService.getLogger('http-request');
this.dnsCache = new CacheableLookup({
maxTtl: 3600, // 1hours maxTtl: 3600, // 1hours
errorTtl: 30, // 30secs errorTtl: 30, // 30secs
lookup: false, // nativeのdns.lookupにfallbackしない lookup: false, // nativeのdns.lookupにfallbackしない
}); });
this.clientDefaults = {
keepAliveTimeout: 30 * 1000,
keepAliveMaxTimeout: 10 * 60 * 1000,
keepAliveTimeoutThreshold: 1 * 1000,
strictContentLength: true,
headersTimeout: 10 * 1000,
bodyTimeout: 10 * 1000,
maxHeaderSize: 16364, // default
maxResponseSize: 10 * 1024 * 1024,
maxRedirections: 3,
connect: {
timeout: 10 * 1000, // コネクションが確立するまでのタイムアウト
maxCachedSessions: 300, // TLSセッションのキャッシュ数 https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L80
lookup: this.dnsCache.lookup as LookupFunction, // https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L98
},
}
this.maxSockets = Math.max(64, this.config.deliverJobConcurrency ?? 128);
this.defaultFetcher = new UndiciFetcher(this.getStandardUndiciFetcherOption(), this.logger);
this.fetch = this.defaultFetcher.fetch;
this.getHtml = this.defaultFetcher.getHtml;
this.defaultJsonFetcher = new UndiciFetcher(this.getStandardUndiciFetcherOption({
maxResponseSize: 1024 * 256,
}), this.logger);
this.getJson = this.defaultJsonFetcher.getJson;
//#region for old http/https, only used in S3Service
this.http = new http.Agent({ this.http = new http.Agent({
keepAlive: true, keepAlive: true,
keepAliveMsecs: 30 * 1000, keepAliveMsecs: 30 * 1000,
lookup: cache.lookup, lookup: this.dnsCache.lookup,
} as http.AgentOptions); } as http.AgentOptions);
this.https = new https.Agent({ this.https = new https.Agent({
keepAlive: true, keepAlive: true,
keepAliveMsecs: 30 * 1000, keepAliveMsecs: 30 * 1000,
lookup: cache.lookup, lookup: this.dnsCache.lookup,
} as https.AgentOptions); } as https.AgentOptions);
const maxSockets = Math.max(256, config.deliverJobConcurrency ?? 128);
this.httpAgent = config.proxy this.httpAgent = config.proxy
? new HttpProxyAgent({ ? new HttpProxyAgent({
keepAlive: true, keepAlive: true,
keepAliveMsecs: 30 * 1000, keepAliveMsecs: 30 * 1000,
maxSockets, maxSockets: this.maxSockets,
maxFreeSockets: 256, maxFreeSockets: 256,
scheduling: 'lifo', scheduling: 'lifo',
proxy: config.proxy, proxy: config.proxy,
@ -72,21 +262,42 @@ export class HttpRequestService {
? new HttpsProxyAgent({ ? new HttpsProxyAgent({
keepAlive: true, keepAlive: true,
keepAliveMsecs: 30 * 1000, keepAliveMsecs: 30 * 1000,
maxSockets, maxSockets: this.maxSockets,
maxFreeSockets: 256, maxFreeSockets: 256,
scheduling: 'lifo', scheduling: 'lifo',
proxy: config.proxy, proxy: config.proxy,
}) })
: this.https; : this.https;
//#endregion
}
@bindThis
public getStandardUndiciFetcherOption(opts: undici.Agent.Options = {}, proxyOpts: undici.Agent.Options = {}) {
return {
agentOptions: {
...this.clientDefaults,
...opts,
},
...(this.config.proxy ? {
proxy: {
uri: this.config.proxy,
options: {
connections: this.maxSockets,
...proxyOpts,
}
}
} : {}),
userAgent: this.config.userAgent,
}
} }
/** /**
* Get agent by URL * Get http agent by URL
* @param url URL * @param url URL
* @param bypassProxy Allways bypass proxy * @param bypassProxy Allways bypass proxy
*/ */
@bindThis @bindThis
public getAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent { public getHttpAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent {
if (bypassProxy || (this.config.proxyBypassHosts || []).includes(url.hostname)) { if (bypassProxy || (this.config.proxyBypassHosts || []).includes(url.hostname)) {
return url.protocol === 'http:' ? this.http : this.https; return url.protocol === 'http:' ? this.http : this.https;
} else { } else {
@ -94,67 +305,37 @@ export class HttpRequestService {
} }
} }
/**
* check ip
*/
@bindThis @bindThis
public async getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record<string, string>): Promise<unknown> { public getConnectorWithIpCheck(connector: undici.buildConnector.connector, checkIp: IpChecker): undici.buildConnector.connectorAsync {
const res = await this.getResponse({ return (options, cb) => {
url, connector(options, (err, socket) => {
method: 'GET', this.logger.debug('Socket connector (with ip checker) called', socket);
headers: Object.assign({ if (err) {
'User-Agent': this.config.userAgent, this.logger.error(`Socket error`, err)
Accept: accept, cb(new Error(`Error while socket connecting\n${err}`), null);
}, headers ?? {}), return;
timeout, }
size: 1024 * 256,
});
return await res.json(); if (socket.remoteAddress == undefined) {
} this.logger.error(`Socket error: remoteAddress is undefined`);
cb(new Error('remoteAddress is undefined (maybe socket destroyed)'), null);
return;
}
@bindThis // allow
public async getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: Record<string, string>): Promise<string> { if (checkIp(socket.remoteAddress)) {
const res = await this.getResponse({ this.logger.debug(`Socket connected (ip ok): ${socket.localPort} => ${socket.remoteAddress}`);
url, cb(null, socket);
method: 'GET', return;
headers: Object.assign({ }
'User-Agent': this.config.userAgent,
Accept: accept,
}, headers ?? {}),
timeout,
});
return await res.text(); this.logger.error('IP is not allowed', socket);
} cb(new StatusError('IP is not allowed', 403, 'IP is not allowed'), null);
socket.destroy();
@bindThis });
public async getResponse(args: { };
url: string,
method: string,
body?: string,
headers: Record<string, string>,
timeout?: number,
size?: number,
}): Promise<Response> {
const timeout = args.timeout ?? 10 * 1000;
const controller = new AbortController();
setTimeout(() => {
controller.abort();
}, timeout * 6);
const res = await fetch(args.url, {
method: args.method,
headers: args.headers,
body: args.body,
timeout,
size: args.size ?? 10 * 1024 * 1024,
agent: (url) => this.getAgentByUrl(url),
signal: controller.signal,
});
if (!res.ok) {
throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
}
return res;
} }
} }

View file

@ -33,7 +33,7 @@ export class S3Service {
? false ? false
: meta.objectStorageS3ForcePathStyle, : meta.objectStorageS3ForcePathStyle,
httpOptions: { httpOptions: {
agent: this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy), agent: this.httpRequestService.getHttpAgentByUrl(new URL(u), !meta.objectStorageUseProxy),
}, },
}); });
} }

View file

@ -30,7 +30,7 @@ export class WebfingerService {
public async webfinger(query: string): Promise<IWebFinger> { public async webfinger(query: string): Promise<IWebFinger> {
const url = this.genUrl(query); const url = this.genUrl(query);
return await this.httpRequestService.getJson(url, 'application/jrd+json, application/json') as IWebFinger; return await this.httpRequestService.getJson<IWebFinger>(url, 'application/jrd+json, application/json');
} }
@bindThis @bindThis

View file

@ -5,8 +5,10 @@ import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js'; import type { Config } from '@/config.js';
import type { User } from '@/models/entities/User.js'; import type { User } from '@/models/entities/User.js';
import { UserKeypairStoreService } from '@/core/UserKeypairStoreService.js'; import { UserKeypairStoreService } from '@/core/UserKeypairStoreService.js';
import { HttpRequestService } from '@/core/HttpRequestService.js'; import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js';
import { LoggerService } from '@/core/LoggerService.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import type Logger from '@/logger.js';
type Request = { type Request = {
url: string; url: string;
@ -28,13 +30,21 @@ type PrivateKey = {
@Injectable() @Injectable()
export class ApRequestService { export class ApRequestService {
private undiciFetcher: UndiciFetcher;
private logger: Logger;
constructor( constructor(
@Inject(DI.config) @Inject(DI.config)
private config: Config, private config: Config,
private userKeypairStoreService: UserKeypairStoreService, private userKeypairStoreService: UserKeypairStoreService,
private httpRequestService: HttpRequestService, private httpRequestService: HttpRequestService,
private loggerService: LoggerService,
) { ) {
this.logger = this.loggerService?.getLogger('ap-request'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる
this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption({
maxRedirections: 0,
}), this.logger );
} }
@bindThis @bindThis
@ -148,16 +158,17 @@ export class ApRequestService {
url, url,
body, body,
additionalHeaders: { additionalHeaders: {
'User-Agent': this.config.userAgent,
}, },
}); });
await this.httpRequestService.getResponse({ await this.undiciFetcher.fetch(
url, url,
method: req.request.method, {
headers: req.request.headers, method: req.request.method,
body, headers: req.request.headers,
}); body,
}
);
} }
/** /**
@ -176,15 +187,16 @@ export class ApRequestService {
}, },
url, url,
additionalHeaders: { additionalHeaders: {
'User-Agent': this.config.userAgent,
}, },
}); });
const res = await this.httpRequestService.getResponse({ const res = await this.httpRequestService.fetch(
url, url,
method: req.request.method, {
headers: req.request.headers, method: req.request.method,
}); headers: req.request.headers,
}
);
return await res.json(); return await res.json();
} }

View file

@ -4,7 +4,7 @@ import { InstanceActorService } from '@/core/InstanceActorService.js';
import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository } from '@/models/index.js'; import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository } from '@/models/index.js';
import type { Config } from '@/config.js'; import type { Config } from '@/config.js';
import { MetaService } from '@/core/MetaService.js'; import { MetaService } from '@/core/MetaService.js';
import { HttpRequestService } from '@/core/HttpRequestService.js'; import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js';
import { DI } from '@/di-symbols.js'; import { DI } from '@/di-symbols.js';
import { UtilityService } from '@/core/UtilityService.js'; import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
@ -12,11 +12,15 @@ import { isCollectionOrOrderedCollection } from './type.js';
import { ApDbResolverService } from './ApDbResolverService.js'; import { ApDbResolverService } from './ApDbResolverService.js';
import { ApRendererService } from './ApRendererService.js'; import { ApRendererService } from './ApRendererService.js';
import { ApRequestService } from './ApRequestService.js'; import { ApRequestService } from './ApRequestService.js';
import { LoggerService } from '@/core/LoggerService.js';
import type { IObject, ICollection, IOrderedCollection } from './type.js'; import type { IObject, ICollection, IOrderedCollection } from './type.js';
import type Logger from '@/logger.js';
export class Resolver { export class Resolver {
private history: Set<string>; private history: Set<string>;
private user?: ILocalUser; private user?: ILocalUser;
private undiciFetcher: UndiciFetcher;
private logger: Logger;
constructor( constructor(
private config: Config, private config: Config,
@ -31,9 +35,14 @@ export class Resolver {
private httpRequestService: HttpRequestService, private httpRequestService: HttpRequestService,
private apRendererService: ApRendererService, private apRendererService: ApRendererService,
private apDbResolverService: ApDbResolverService, private apDbResolverService: ApDbResolverService,
private loggerService: LoggerService,
private recursionLimit = 100, private recursionLimit = 100,
) { ) {
this.history = new Set(); this.history = new Set();
this.logger = this.loggerService?.getLogger('ap-resolve'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる
this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption({
maxRedirections: 0,
}), this.logger);
} }
@bindThis @bindThis
@ -96,8 +105,8 @@ export class Resolver {
} }
const object = (this.user const object = (this.user
? await this.apRequestService.signedGet(value, this.user) ? await this.apRequestService.signedGet(value, this.user) as IObject
: await this.httpRequestService.getJson(value, 'application/activity+json, application/ld+json')) as IObject; : await this.undiciFetcher.getJson<IObject>(value, 'application/activity+json, application/ld+json'));
if (object == null || ( if (object == null || (
Array.isArray(object['@context']) ? Array.isArray(object['@context']) ?

View file

@ -1,6 +1,5 @@
import * as crypto from 'node:crypto'; import * as crypto from 'node:crypto';
import { Inject, Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import fetch from 'node-fetch';
import { HttpRequestService } from '@/core/HttpRequestService.js'; import { HttpRequestService } from '@/core/HttpRequestService.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { CONTEXTS } from './misc/contexts.js'; import { CONTEXTS } from './misc/contexts.js';
@ -116,14 +115,19 @@ class LdSignature {
@bindThis @bindThis
private async fetchDocument(url: string) { private async fetchDocument(url: string) {
const json = await fetch(url, { const json = await this.httpRequestService.fetch(
headers: { url,
Accept: 'application/ld+json, application/json', {
headers: {
Accept: 'application/ld+json, application/json',
},
// TODO
//timeout: this.loderTimeout,
}, },
// TODO {
//timeout: this.loderTimeout, noOkError: true,
agent: u => u.protocol === 'http:' ? this.httpRequestService.httpAgent : this.httpRequestService.httpsAgent, }
}).then(res => { ).then(res => {
if (!res.ok) { if (!res.ok) {
throw `${res.status} ${res.statusText}`; throw `${res.status} ${res.statusText}`;
} else { } else {

View file

@ -33,24 +33,26 @@ export class WebhookDeliverProcessorService {
try { try {
this.logger.debug(`delivering ${job.data.webhookId}`); this.logger.debug(`delivering ${job.data.webhookId}`);
const res = await this.httpRequestService.getResponse({ const res = await this.httpRequestService.fetch(
url: job.data.to, job.data.to,
method: 'POST', {
headers: { method: 'POST',
'User-Agent': 'Misskey-Hooks', headers: {
'X-Misskey-Host': this.config.host, 'User-Agent': 'Misskey-Hooks',
'X-Misskey-Hook-Id': job.data.webhookId, 'X-Misskey-Host': this.config.host,
'X-Misskey-Hook-Secret': job.data.secret, 'X-Misskey-Hook-Id': job.data.webhookId,
}, 'X-Misskey-Hook-Secret': job.data.secret,
body: JSON.stringify({ },
hookId: job.data.webhookId, body: JSON.stringify({
userId: job.data.userId, hookId: job.data.webhookId,
eventId: job.data.eventId, userId: job.data.userId,
createdAt: job.data.createdAt, eventId: job.data.eventId,
type: job.data.type, createdAt: job.data.createdAt,
body: job.data.content, type: job.data.type,
}), body: job.data.content,
}); }),
}
);
this.webhooksRepository.update({ id: job.data.webhookId }, { this.webhooksRepository.update({ id: job.data.webhookId }, {
latestSentAt: new Date(), latestSentAt: new Date(),

View file

@ -33,15 +33,16 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
private httpRequestService: HttpRequestService, private httpRequestService: HttpRequestService,
) { ) {
super(meta, paramDef, async (ps, me) => { super(meta, paramDef, async (ps, me) => {
const res = await this.httpRequestService.getResponse({ const res = await this.httpRequestService.fetch(
url: ps.url, ps.url,
method: 'GET', {
headers: Object.assign({ method: 'GET',
'User-Agent': config.userAgent, headers: {
Accept: 'application/rss+xml, */*', Accept: 'application/rss+xml, */*',
}), },
timeout: 5000, // timeout: 5000,
}); }
);
const text = await res.text(); const text = await res.text();

View file

@ -1,5 +1,4 @@
import { URLSearchParams } from 'node:url'; import { URLSearchParams } from 'node:url';
import fetch from 'node-fetch';
import { Inject, Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import type { NotesRepository } from '@/models/index.js'; import type { NotesRepository } from '@/models/index.js';
import { Endpoint } from '@/server/api/endpoint-base.js'; import { Endpoint } from '@/server/api/endpoint-base.js';
@ -84,25 +83,27 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
const endpoint = instance.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate'; const endpoint = instance.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate';
const res = await fetch(endpoint, { const res = await this.httpRequestService.fetch(
method: 'POST', endpoint,
headers: { {
'Content-Type': 'application/x-www-form-urlencoded', method: 'POST',
'User-Agent': config.userAgent, headers: {
Accept: 'application/json, */*', 'Content-Type': 'application/x-www-form-urlencoded',
Accept: 'application/json, */*',
},
body: params.toString(),
}, },
body: params, {
// TODO noOkError: false,
//timeout: 10000, }
agent: (url) => this.httpRequestService.getAgentByUrl(url), );
});
const json = (await res.json()) as { const json = (await res.json()) as {
translations: { translations: {
detected_source_language: string; detected_source_language: string;
text: string; text: string;
}[]; }[];
}; };
return { return {
sourceLang: json.translations[0].detected_source_language, sourceLang: json.translations[0].detected_source_language,

View file

@ -181,7 +181,7 @@ export class DiscordServerService {
} }
})); }));
const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', 10 * 1000, { const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', {
'Authorization': `Bearer ${accessToken}`, 'Authorization': `Bearer ${accessToken}`,
})) as Record<string, unknown>; })) as Record<string, unknown>;
@ -249,7 +249,7 @@ export class DiscordServerService {
} }
})); }));
const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', 10 * 1000, { const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', {
'Authorization': `Bearer ${accessToken}`, 'Authorization': `Bearer ${accessToken}`,
})) as Record<string, unknown>; })) as Record<string, unknown>;
if (typeof id !== 'string' || typeof username !== 'string' || typeof discriminator !== 'string') { if (typeof id !== 'string' || typeof username !== 'string' || typeof discriminator !== 'string') {

View file

@ -174,7 +174,7 @@ export class GithubServerService {
} }
})); }));
const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, { const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', {
'Authorization': `bearer ${accessToken}`, 'Authorization': `bearer ${accessToken}`,
})) as Record<string, unknown>; })) as Record<string, unknown>;
if (typeof login !== 'string' || typeof id !== 'string') { if (typeof login !== 'string' || typeof id !== 'string') {
@ -223,7 +223,7 @@ export class GithubServerService {
} }
})); }));
const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, { const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', {
'Authorization': `bearer ${accessToken}`, 'Authorization': `bearer ${accessToken}`,
})) as Record<string, unknown>; })) as Record<string, unknown>;

View file

@ -63,9 +63,8 @@ export class UrlPreviewService {
this.logger.info(meta.summalyProxy this.logger.info(meta.summalyProxy
? `(Proxy) Getting preview of ${url}@${lang} ...` ? `(Proxy) Getting preview of ${url}@${lang} ...`
: `Getting preview of ${url}@${lang} ...`); : `Getting preview of ${url}@${lang} ...`);
try { try {
const summary = meta.summalyProxy ? await this.httpRequestService.getJson(`${meta.summalyProxy}?${query({ const summary = meta.summalyProxy ? await this.httpRequestService.getJson<ReturnType<typeof summaly.default>>(`${meta.summalyProxy}?${query({
url: url, url: url,
lang: lang ?? 'ja-JP', lang: lang ?? 'ja-JP',
})}`) : await summaly.default(url, { })}`) : await summaly.default(url, {

View file

@ -4272,6 +4272,7 @@ __metadata:
typeorm: 0.3.11 typeorm: 0.3.11
typescript: 4.9.4 typescript: 4.9.4
ulid: 2.3.0 ulid: 2.3.0
undici: ^5.14.0
unzipper: 0.10.11 unzipper: 0.10.11
uuid: 9.0.0 uuid: 9.0.0
vary: 1.1.2 vary: 1.1.2
@ -16665,6 +16666,15 @@ __metadata:
languageName: node languageName: node
linkType: hard linkType: hard
"undici@npm:^5.14.0, undici@npm:^5.5.1":
version: 5.14.0
resolution: "undici@npm:5.14.0"
dependencies:
busboy: ^1.6.0
checksum: 7a076e44d84b25844b4eb657034437b8b9bb91f17d347de474fdea1d4263ce7ae9406db79cd30de5642519277b4893f43073258bcc8fed420b295da3fdd11b26
languageName: node
linkType: hard
"undici@npm:^5.2.0": "undici@npm:^5.2.0":
version: 5.13.0 version: 5.13.0
resolution: "undici@npm:5.13.0" resolution: "undici@npm:5.13.0"
@ -16674,15 +16684,6 @@ __metadata:
languageName: node languageName: node
linkType: hard linkType: hard
"undici@npm:^5.5.1":
version: 5.14.0
resolution: "undici@npm:5.14.0"
dependencies:
busboy: ^1.6.0
checksum: 7a076e44d84b25844b4eb657034437b8b9bb91f17d347de474fdea1d4263ce7ae9406db79cd30de5642519277b4893f43073258bcc8fed420b295da3fdd11b26
languageName: node
linkType: hard
"union-value@npm:^1.0.0": "union-value@npm:^1.0.0":
version: 1.0.1 version: 1.0.1
resolution: "union-value@npm:1.0.1" resolution: "union-value@npm:1.0.1"