mirror of
https://git.joinsharkey.org/Sharkey/Sharkey.git
synced 2024-11-08 23:33:09 +02:00
perf(backend): Use undici instead of node-fetch and got (#9459)
* Implement? HttpFetchService
* ✌️
* remove node-fetch
* fix
* refactor
* fix
* gateway timeout
* UndiciFetcherクラスを追加 (仮コミット, ビルドもstartもさせていない)
* fix
* add logger and fix url preview
* fix ip check
* enhance logger and error handling
* fix
* fix
* clean up
* Use custom fetcher for ApRequest / ApResolver
* bypassProxyはproxyBypassHostsに判断を委譲するように
* set maxRedirections (default 3, ApRequest/ApResolver: 0)
* fix comment
* handle error s3 upload
* add debug message
* no return await
* Revert "no return await"
This reverts commit b5b0dc58a342393d260492e3a6f58304372f53b2.
* reduce maxSockets
* apResolverのUndiciFetcherを廃止しapRequestのものを使う、 add ap logger
* Revert "apResolverのUndiciFetcherを廃止しapRequestのものを使う、 add ap logger"
This reverts commit 997243915c8e1f8472da64f607f88c36cb1d5cb4.
* add logger
* fix
* change logger name
* safe
* デフォルトでUser-Agentを設定
This commit is contained in:
parent
2470afaa2e
commit
978a9bbb3b
19 changed files with 444 additions and 255 deletions
|
@ -122,10 +122,12 @@ id: 'aid'
|
||||||
# Proxy for HTTP/HTTPS
|
# Proxy for HTTP/HTTPS
|
||||||
#proxy: http://127.0.0.1:3128
|
#proxy: http://127.0.0.1:3128
|
||||||
|
|
||||||
#proxyBypassHosts: [
|
proxyBypassHosts:
|
||||||
# 'example.com',
|
- api.deepl.com
|
||||||
# '192.0.2.8'
|
- api-free.deepl.com
|
||||||
#]
|
- www.recaptcha.net
|
||||||
|
- hcaptcha.com
|
||||||
|
- challenges.cloudflare.com
|
||||||
|
|
||||||
# Proxy for SMTP/SMTPS
|
# Proxy for SMTP/SMTPS
|
||||||
#proxySmtp: http://127.0.0.1:3128 # use HTTP/1.1 CONNECT
|
#proxySmtp: http://127.0.0.1:3128 # use HTTP/1.1 CONNECT
|
||||||
|
|
|
@ -77,7 +77,6 @@
|
||||||
"misskey-js": "0.0.14",
|
"misskey-js": "0.0.14",
|
||||||
"ms": "3.0.0-canary.1",
|
"ms": "3.0.0-canary.1",
|
||||||
"nested-property": "4.0.0",
|
"nested-property": "4.0.0",
|
||||||
"node-fetch": "3.3.0",
|
|
||||||
"nodemailer": "6.8.0",
|
"nodemailer": "6.8.0",
|
||||||
"nsfwjs": "2.4.2",
|
"nsfwjs": "2.4.2",
|
||||||
"oauth": "^0.10.0",
|
"oauth": "^0.10.0",
|
||||||
|
@ -118,6 +117,7 @@
|
||||||
"twemoji-parser": "14.0.0",
|
"twemoji-parser": "14.0.0",
|
||||||
"typeorm": "0.3.11",
|
"typeorm": "0.3.11",
|
||||||
"ulid": "2.3.0",
|
"ulid": "2.3.0",
|
||||||
|
"undici": "^5.14.0",
|
||||||
"unzipper": "0.10.11",
|
"unzipper": "0.10.11",
|
||||||
"uuid": "9.0.0",
|
"uuid": "9.0.0",
|
||||||
"vary": "1.1.2",
|
"vary": "1.1.2",
|
||||||
|
@ -180,6 +180,7 @@
|
||||||
"execa": "6.1.0",
|
"execa": "6.1.0",
|
||||||
"jest": "29.3.1",
|
"jest": "29.3.1",
|
||||||
"jest-mock": "^29.3.1",
|
"jest-mock": "^29.3.1",
|
||||||
|
"node-fetch": "3.3.0",
|
||||||
"typescript": "4.9.4"
|
"typescript": "4.9.4"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,4 @@
|
||||||
import { Inject, Injectable } from '@nestjs/common';
|
import { Injectable } from '@nestjs/common';
|
||||||
import { DI } from '@/di-symbols.js';
|
|
||||||
import type { UsersRepository } from '@/models/index.js';
|
|
||||||
import type { Config } from '@/config.js';
|
|
||||||
import { HttpRequestService } from '@/core/HttpRequestService.js';
|
import { HttpRequestService } from '@/core/HttpRequestService.js';
|
||||||
import { bindThis } from '@/decorators.js';
|
import { bindThis } from '@/decorators.js';
|
||||||
|
|
||||||
|
@ -13,9 +10,6 @@ type CaptchaResponse = {
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class CaptchaService {
|
export class CaptchaService {
|
||||||
constructor(
|
constructor(
|
||||||
@Inject(DI.config)
|
|
||||||
private config: Config,
|
|
||||||
|
|
||||||
private httpRequestService: HttpRequestService,
|
private httpRequestService: HttpRequestService,
|
||||||
) {
|
) {
|
||||||
}
|
}
|
||||||
|
@ -27,16 +21,16 @@ export class CaptchaService {
|
||||||
response,
|
response,
|
||||||
});
|
});
|
||||||
|
|
||||||
const res = await fetch(url, {
|
const res = await this.httpRequestService.fetch(
|
||||||
method: 'POST',
|
url,
|
||||||
body: params,
|
{
|
||||||
headers: {
|
method: 'POST',
|
||||||
'User-Agent': this.config.userAgent,
|
body: params,
|
||||||
},
|
},
|
||||||
// TODO
|
{
|
||||||
//timeout: 10 * 1000,
|
noOkError: true,
|
||||||
agent: (url, bypassProxy) => this.httpRequestService.getAgentByUrl(url, bypassProxy),
|
}
|
||||||
}).catch(err => {
|
).catch(err => {
|
||||||
throw `${err.message ?? err}`;
|
throw `${err.message ?? err}`;
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
@ -8,11 +8,12 @@ import got, * as Got from 'got';
|
||||||
import chalk from 'chalk';
|
import chalk from 'chalk';
|
||||||
import { DI } from '@/di-symbols.js';
|
import { DI } from '@/di-symbols.js';
|
||||||
import type { Config } from '@/config.js';
|
import type { Config } from '@/config.js';
|
||||||
import { HttpRequestService } from '@/core/HttpRequestService.js';
|
import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js';
|
||||||
import { createTemp } from '@/misc/create-temp.js';
|
import { createTemp } from '@/misc/create-temp.js';
|
||||||
import { StatusError } from '@/misc/status-error.js';
|
import { StatusError } from '@/misc/status-error.js';
|
||||||
import { LoggerService } from '@/core/LoggerService.js';
|
import { LoggerService } from '@/core/LoggerService.js';
|
||||||
import type Logger from '@/logger.js';
|
import type Logger from '@/logger.js';
|
||||||
|
import { buildConnector } from 'undici';
|
||||||
|
|
||||||
const pipeline = util.promisify(stream.pipeline);
|
const pipeline = util.promisify(stream.pipeline);
|
||||||
import { bindThis } from '@/decorators.js';
|
import { bindThis } from '@/decorators.js';
|
||||||
|
@ -20,6 +21,7 @@ import { bindThis } from '@/decorators.js';
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class DownloadService {
|
export class DownloadService {
|
||||||
private logger: Logger;
|
private logger: Logger;
|
||||||
|
private undiciFetcher: UndiciFetcher;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
@Inject(DI.config)
|
@Inject(DI.config)
|
||||||
|
@ -29,70 +31,42 @@ export class DownloadService {
|
||||||
private loggerService: LoggerService,
|
private loggerService: LoggerService,
|
||||||
) {
|
) {
|
||||||
this.logger = this.loggerService.getLogger('download');
|
this.logger = this.loggerService.getLogger('download');
|
||||||
|
|
||||||
|
this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption(
|
||||||
|
{
|
||||||
|
connect: process.env.NODE_ENV === 'development' ?
|
||||||
|
this.httpRequestService.clientDefaults.connect
|
||||||
|
:
|
||||||
|
this.httpRequestService.getConnectorWithIpCheck(
|
||||||
|
buildConnector({
|
||||||
|
...this.httpRequestService.clientDefaults.connect,
|
||||||
|
}),
|
||||||
|
(ip) => !this.isPrivateIp(ip)
|
||||||
|
),
|
||||||
|
bodyTimeout: 30 * 1000,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
connect: this.httpRequestService.clientDefaults.connect,
|
||||||
|
}
|
||||||
|
), this.logger);
|
||||||
}
|
}
|
||||||
|
|
||||||
@bindThis
|
@bindThis
|
||||||
public async downloadUrl(url: string, path: string): Promise<void> {
|
public async downloadUrl(url: string, path: string): Promise<void> {
|
||||||
this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
|
this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
|
||||||
|
|
||||||
const timeout = 30 * 1000;
|
const timeout = 30 * 1000;
|
||||||
const operationTimeout = 60 * 1000;
|
const operationTimeout = 60 * 1000;
|
||||||
const maxSize = this.config.maxFileSize ?? 262144000;
|
const maxSize = this.config.maxFileSize ?? 262144000;
|
||||||
|
|
||||||
const req = got.stream(url, {
|
const response = await this.undiciFetcher.fetch(url);
|
||||||
headers: {
|
|
||||||
'User-Agent': this.config.userAgent,
|
if (response.body === null) {
|
||||||
},
|
throw new StatusError('No body', 400, 'No body');
|
||||||
timeout: {
|
|
||||||
lookup: timeout,
|
|
||||||
connect: timeout,
|
|
||||||
secureConnect: timeout,
|
|
||||||
socket: timeout, // read timeout
|
|
||||||
response: timeout,
|
|
||||||
send: timeout,
|
|
||||||
request: operationTimeout, // whole operation timeout
|
|
||||||
},
|
|
||||||
agent: {
|
|
||||||
http: this.httpRequestService.httpAgent,
|
|
||||||
https: this.httpRequestService.httpsAgent,
|
|
||||||
},
|
|
||||||
http2: false, // default
|
|
||||||
retry: {
|
|
||||||
limit: 0,
|
|
||||||
},
|
|
||||||
}).on('response', (res: Got.Response) => {
|
|
||||||
if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !this.config.proxy && res.ip) {
|
|
||||||
if (this.isPrivateIp(res.ip)) {
|
|
||||||
this.logger.warn(`Blocked address: ${res.ip}`);
|
|
||||||
req.destroy();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const contentLength = res.headers['content-length'];
|
|
||||||
if (contentLength != null) {
|
|
||||||
const size = Number(contentLength);
|
|
||||||
if (size > maxSize) {
|
|
||||||
this.logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`);
|
|
||||||
req.destroy();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}).on('downloadProgress', (progress: Got.Progress) => {
|
|
||||||
if (progress.transferred > maxSize) {
|
|
||||||
this.logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`);
|
|
||||||
req.destroy();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
try {
|
|
||||||
await pipeline(req, fs.createWriteStream(path));
|
|
||||||
} catch (e) {
|
|
||||||
if (e instanceof Got.HTTPError) {
|
|
||||||
throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage);
|
|
||||||
} else {
|
|
||||||
throw e;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
await pipeline(stream.Readable.fromWeb(response.body), fs.createWriteStream(path));
|
||||||
|
|
||||||
this.logger.succ(`Download finished: ${chalk.cyan(url)}`);
|
this.logger.succ(`Download finished: ${chalk.cyan(url)}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -114,7 +88,7 @@ export class DownloadService {
|
||||||
cleanup();
|
cleanup();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@bindThis
|
@bindThis
|
||||||
private isPrivateIp(ip: string): boolean {
|
private isPrivateIp(ip: string): boolean {
|
||||||
for (const net of this.config.allowedPrivateNetworks ?? []) {
|
for (const net of this.config.allowedPrivateNetworks ?? []) {
|
||||||
|
@ -124,6 +98,6 @@ export class DownloadService {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return PrivateIp(ip);
|
return PrivateIp(ip) ?? false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -375,8 +375,19 @@ export class DriveService {
|
||||||
partSize: s3.endpoint.hostname === 'storage.googleapis.com' ? 500 * 1024 * 1024 : 8 * 1024 * 1024,
|
partSize: s3.endpoint.hostname === 'storage.googleapis.com' ? 500 * 1024 * 1024 : 8 * 1024 * 1024,
|
||||||
});
|
});
|
||||||
|
|
||||||
const result = await upload.promise();
|
await upload.promise()
|
||||||
if (result) this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`);
|
.then(
|
||||||
|
result => {
|
||||||
|
if (result) {
|
||||||
|
this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`);
|
||||||
|
} else {
|
||||||
|
this.registerLogger.error(`Upload Result Empty: key = ${key}, filename = ${filename}`);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
err => {
|
||||||
|
this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err);
|
||||||
|
}
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@bindThis
|
@bindThis
|
||||||
|
@ -462,6 +473,8 @@ export class DriveService {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
this.registerLogger.debug(`ADD DRIVE FILE: user ${user?.id ?? 'not set'}, name ${detectedName}, tmp ${path}`);
|
||||||
|
|
||||||
//#region Check drive usage
|
//#region Check drive usage
|
||||||
if (user && !isLink) {
|
if (user && !isLink) {
|
||||||
const usage = await this.driveFileEntityService.calcDriveUsageOf(user);
|
const usage = await this.driveFileEntityService.calcDriveUsageOf(user);
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
import { URL } from 'node:url';
|
import { URL } from 'node:url';
|
||||||
import { Inject, Injectable } from '@nestjs/common';
|
import { Inject, Injectable } from '@nestjs/common';
|
||||||
import { JSDOM } from 'jsdom';
|
import { JSDOM } from 'jsdom';
|
||||||
import fetch from 'node-fetch';
|
|
||||||
import tinycolor from 'tinycolor2';
|
import tinycolor from 'tinycolor2';
|
||||||
import type { Instance } from '@/models/entities/Instance.js';
|
import type { Instance } from '@/models/entities/Instance.js';
|
||||||
import type { InstancesRepository } from '@/models/index.js';
|
import type { InstancesRepository } from '@/models/index.js';
|
||||||
|
@ -191,11 +190,7 @@ export class FetchInstanceMetadataService {
|
||||||
|
|
||||||
const faviconUrl = url + '/favicon.ico';
|
const faviconUrl = url + '/favicon.ico';
|
||||||
|
|
||||||
const favicon = await fetch(faviconUrl, {
|
const favicon = await this.httpRequestService.fetch(faviconUrl, {}, { noOkError: true });
|
||||||
// TODO
|
|
||||||
//timeout: 10000,
|
|
||||||
agent: url => this.httpRequestService.getAgentByUrl(url),
|
|
||||||
});
|
|
||||||
|
|
||||||
if (favicon.ok) {
|
if (favicon.ok) {
|
||||||
return faviconUrl;
|
return faviconUrl;
|
||||||
|
|
|
@ -1,67 +1,257 @@
|
||||||
import * as http from 'node:http';
|
import * as http from 'node:http';
|
||||||
import * as https from 'node:https';
|
import * as https from 'node:https';
|
||||||
import CacheableLookup from 'cacheable-lookup';
|
import CacheableLookup from 'cacheable-lookup';
|
||||||
import fetch from 'node-fetch';
|
|
||||||
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
|
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
|
||||||
import { Inject, Injectable } from '@nestjs/common';
|
import { Inject, Injectable } from '@nestjs/common';
|
||||||
import { DI } from '@/di-symbols.js';
|
import { DI } from '@/di-symbols.js';
|
||||||
import type { Config } from '@/config.js';
|
import type { Config } from '@/config.js';
|
||||||
import { StatusError } from '@/misc/status-error.js';
|
import { StatusError } from '@/misc/status-error.js';
|
||||||
import { bindThis } from '@/decorators.js';
|
import { bindThis } from '@/decorators.js';
|
||||||
import type { Response } from 'node-fetch';
|
import * as undici from 'undici';
|
||||||
import type { URL } from 'node:url';
|
import { LookupFunction } from 'node:net';
|
||||||
|
import { LoggerService } from '@/core/LoggerService.js';
|
||||||
|
import type Logger from '@/logger.js';
|
||||||
|
|
||||||
|
// true to allow, false to deny
|
||||||
|
export type IpChecker = (ip: string) => boolean;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Child class to create and save Agent for fetch.
|
||||||
|
* You should construct this when you want
|
||||||
|
* to change timeout, size limit, socket connect function, etc.
|
||||||
|
*/
|
||||||
|
export class UndiciFetcher {
|
||||||
|
/**
|
||||||
|
* Get http non-proxy agent (undici)
|
||||||
|
*/
|
||||||
|
public nonProxiedAgent: undici.Agent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get http proxy or non-proxy agent (undici)
|
||||||
|
*/
|
||||||
|
public agent: undici.ProxyAgent | undici.Agent;
|
||||||
|
|
||||||
|
private proxyBypassHosts: string[];
|
||||||
|
private userAgent: string | undefined;
|
||||||
|
|
||||||
|
private logger: Logger | undefined;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
args: {
|
||||||
|
agentOptions: undici.Agent.Options;
|
||||||
|
proxy?: {
|
||||||
|
uri: string;
|
||||||
|
options?: undici.Agent.Options; // Override of agentOptions
|
||||||
|
},
|
||||||
|
proxyBypassHosts?: string[];
|
||||||
|
userAgent?: string;
|
||||||
|
},
|
||||||
|
logger?: Logger,
|
||||||
|
) {
|
||||||
|
this.logger = logger;
|
||||||
|
this.logger?.debug('UndiciFetcher constructor', args);
|
||||||
|
|
||||||
|
this.proxyBypassHosts = args.proxyBypassHosts ?? [];
|
||||||
|
this.userAgent = args.userAgent;
|
||||||
|
|
||||||
|
this.nonProxiedAgent = new undici.Agent({
|
||||||
|
...args.agentOptions,
|
||||||
|
connect: (process.env.NODE_ENV !== 'production' && typeof args.agentOptions.connect !== 'function')
|
||||||
|
? (options, cb) => {
|
||||||
|
// Custom connector for debug
|
||||||
|
undici.buildConnector(args.agentOptions.connect as undici.buildConnector.BuildOptions)(options, (err, socket) => {
|
||||||
|
this.logger?.debug('Socket connector called', socket);
|
||||||
|
if (err) {
|
||||||
|
this.logger?.debug(`Socket error`, err);
|
||||||
|
cb(new Error(`Error while socket connecting\n${err}`), null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.logger?.debug(`Socket connected: port ${socket.localPort} => remote ${socket.remoteAddress}`);
|
||||||
|
cb(null, socket);
|
||||||
|
});
|
||||||
|
} : args.agentOptions.connect,
|
||||||
|
});
|
||||||
|
|
||||||
|
this.agent = args.proxy
|
||||||
|
? new undici.ProxyAgent({
|
||||||
|
...args.agentOptions,
|
||||||
|
...args.proxy.options,
|
||||||
|
|
||||||
|
uri: args.proxy.uri,
|
||||||
|
|
||||||
|
connect: (process.env.NODE_ENV !== 'production' && typeof (args.proxy?.options?.connect ?? args.agentOptions.connect) !== 'function')
|
||||||
|
? (options, cb) => {
|
||||||
|
// Custom connector for debug
|
||||||
|
undici.buildConnector((args.proxy?.options?.connect ?? args.agentOptions.connect) as undici.buildConnector.BuildOptions)(options, (err, socket) => {
|
||||||
|
this.logger?.debug('Socket connector called (secure)', socket);
|
||||||
|
if (err) {
|
||||||
|
this.logger?.debug(`Socket error`, err);
|
||||||
|
cb(new Error(`Error while socket connecting\n${err}`), null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.logger?.debug(`Socket connected (secure): port ${socket.localPort} => remote ${socket.remoteAddress}`);
|
||||||
|
cb(null, socket);
|
||||||
|
});
|
||||||
|
} : (args.proxy?.options?.connect ?? args.agentOptions.connect),
|
||||||
|
})
|
||||||
|
: this.nonProxiedAgent;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get agent by URL
|
||||||
|
* @param url URL
|
||||||
|
* @param bypassProxy Allways bypass proxy
|
||||||
|
*/
|
||||||
|
@bindThis
|
||||||
|
public getAgentByUrl(url: URL, bypassProxy = false): undici.Agent | undici.ProxyAgent {
|
||||||
|
if (bypassProxy || this.proxyBypassHosts.includes(url.hostname)) {
|
||||||
|
return this.nonProxiedAgent;
|
||||||
|
} else {
|
||||||
|
return this.agent;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@bindThis
|
||||||
|
public async fetch(
|
||||||
|
url: string | URL,
|
||||||
|
options: undici.RequestInit = {},
|
||||||
|
privateOptions: { noOkError?: boolean; bypassProxy?: boolean; } = { noOkError: false, bypassProxy: false }
|
||||||
|
): Promise<undici.Response> {
|
||||||
|
const res = await undici.fetch(url, {
|
||||||
|
dispatcher: this.getAgentByUrl(new URL(url), privateOptions.bypassProxy),
|
||||||
|
...options,
|
||||||
|
headers: {
|
||||||
|
'User-Agent': this.userAgent ?? '',
|
||||||
|
...(options.headers ?? {}),
|
||||||
|
},
|
||||||
|
}).catch((err) => {
|
||||||
|
this.logger?.error('fetch error', err);
|
||||||
|
throw new StatusError('Resource Unreachable', 500, 'Resource Unreachable');
|
||||||
|
});
|
||||||
|
if (!res.ok && !privateOptions.noOkError) {
|
||||||
|
throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
@bindThis
|
||||||
|
public async getJson<T extends unknown>(url: string, accept = 'application/json, */*', headers?: Record<string, string>): Promise<T> {
|
||||||
|
const res = await this.fetch(
|
||||||
|
url,
|
||||||
|
{
|
||||||
|
headers: Object.assign({
|
||||||
|
Accept: accept,
|
||||||
|
}, headers ?? {}),
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
return await res.json() as T;
|
||||||
|
}
|
||||||
|
|
||||||
|
@bindThis
|
||||||
|
public async getHtml(url: string, accept = 'text/html, */*', headers?: Record<string, string>): Promise<string> {
|
||||||
|
const res = await this.fetch(
|
||||||
|
url,
|
||||||
|
{
|
||||||
|
headers: Object.assign({
|
||||||
|
Accept: accept,
|
||||||
|
}, headers ?? {}),
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
return await res.text();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class HttpRequestService {
|
export class HttpRequestService {
|
||||||
/**
|
public defaultFetcher: UndiciFetcher;
|
||||||
* Get http non-proxy agent
|
public fetch: UndiciFetcher['fetch'];
|
||||||
*/
|
public getHtml: UndiciFetcher['getHtml'];
|
||||||
|
public defaultJsonFetcher: UndiciFetcher;
|
||||||
|
public getJson: UndiciFetcher['getJson'];
|
||||||
|
|
||||||
|
//#region for old http/https, only used in S3Service
|
||||||
|
// http non-proxy agent
|
||||||
private http: http.Agent;
|
private http: http.Agent;
|
||||||
|
|
||||||
/**
|
// https non-proxy agent
|
||||||
* Get https non-proxy agent
|
|
||||||
*/
|
|
||||||
private https: https.Agent;
|
private https: https.Agent;
|
||||||
|
|
||||||
/**
|
// http proxy or non-proxy agent
|
||||||
* Get http proxy or non-proxy agent
|
|
||||||
*/
|
|
||||||
public httpAgent: http.Agent;
|
public httpAgent: http.Agent;
|
||||||
|
|
||||||
/**
|
// https proxy or non-proxy agent
|
||||||
* Get https proxy or non-proxy agent
|
|
||||||
*/
|
|
||||||
public httpsAgent: https.Agent;
|
public httpsAgent: https.Agent;
|
||||||
|
//#endregion
|
||||||
|
|
||||||
|
public readonly dnsCache: CacheableLookup;
|
||||||
|
public readonly clientDefaults: undici.Agent.Options;
|
||||||
|
private maxSockets: number;
|
||||||
|
|
||||||
|
private logger: Logger;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
@Inject(DI.config)
|
@Inject(DI.config)
|
||||||
private config: Config,
|
private config: Config,
|
||||||
|
private loggerService: LoggerService,
|
||||||
) {
|
) {
|
||||||
const cache = new CacheableLookup({
|
this.logger = this.loggerService.getLogger('http-request');
|
||||||
|
|
||||||
|
this.dnsCache = new CacheableLookup({
|
||||||
maxTtl: 3600, // 1hours
|
maxTtl: 3600, // 1hours
|
||||||
errorTtl: 30, // 30secs
|
errorTtl: 30, // 30secs
|
||||||
lookup: false, // nativeのdns.lookupにfallbackしない
|
lookup: false, // nativeのdns.lookupにfallbackしない
|
||||||
});
|
});
|
||||||
|
|
||||||
|
this.clientDefaults = {
|
||||||
|
keepAliveTimeout: 30 * 1000,
|
||||||
|
keepAliveMaxTimeout: 10 * 60 * 1000,
|
||||||
|
keepAliveTimeoutThreshold: 1 * 1000,
|
||||||
|
strictContentLength: true,
|
||||||
|
headersTimeout: 10 * 1000,
|
||||||
|
bodyTimeout: 10 * 1000,
|
||||||
|
maxHeaderSize: 16364, // default
|
||||||
|
maxResponseSize: 10 * 1024 * 1024,
|
||||||
|
maxRedirections: 3,
|
||||||
|
connect: {
|
||||||
|
timeout: 10 * 1000, // コネクションが確立するまでのタイムアウト
|
||||||
|
maxCachedSessions: 300, // TLSセッションのキャッシュ数 https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L80
|
||||||
|
lookup: this.dnsCache.lookup as LookupFunction, // https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L98
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
this.maxSockets = Math.max(64, this.config.deliverJobConcurrency ?? 128);
|
||||||
|
|
||||||
|
this.defaultFetcher = new UndiciFetcher(this.getStandardUndiciFetcherOption(), this.logger);
|
||||||
|
|
||||||
|
this.fetch = this.defaultFetcher.fetch;
|
||||||
|
this.getHtml = this.defaultFetcher.getHtml;
|
||||||
|
|
||||||
|
this.defaultJsonFetcher = new UndiciFetcher(this.getStandardUndiciFetcherOption({
|
||||||
|
maxResponseSize: 1024 * 256,
|
||||||
|
}), this.logger);
|
||||||
|
|
||||||
|
this.getJson = this.defaultJsonFetcher.getJson;
|
||||||
|
|
||||||
|
//#region for old http/https, only used in S3Service
|
||||||
this.http = new http.Agent({
|
this.http = new http.Agent({
|
||||||
keepAlive: true,
|
keepAlive: true,
|
||||||
keepAliveMsecs: 30 * 1000,
|
keepAliveMsecs: 30 * 1000,
|
||||||
lookup: cache.lookup,
|
lookup: this.dnsCache.lookup,
|
||||||
} as http.AgentOptions);
|
} as http.AgentOptions);
|
||||||
|
|
||||||
this.https = new https.Agent({
|
this.https = new https.Agent({
|
||||||
keepAlive: true,
|
keepAlive: true,
|
||||||
keepAliveMsecs: 30 * 1000,
|
keepAliveMsecs: 30 * 1000,
|
||||||
lookup: cache.lookup,
|
lookup: this.dnsCache.lookup,
|
||||||
} as https.AgentOptions);
|
} as https.AgentOptions);
|
||||||
|
|
||||||
const maxSockets = Math.max(256, config.deliverJobConcurrency ?? 128);
|
|
||||||
|
|
||||||
this.httpAgent = config.proxy
|
this.httpAgent = config.proxy
|
||||||
? new HttpProxyAgent({
|
? new HttpProxyAgent({
|
||||||
keepAlive: true,
|
keepAlive: true,
|
||||||
keepAliveMsecs: 30 * 1000,
|
keepAliveMsecs: 30 * 1000,
|
||||||
maxSockets,
|
maxSockets: this.maxSockets,
|
||||||
maxFreeSockets: 256,
|
maxFreeSockets: 256,
|
||||||
scheduling: 'lifo',
|
scheduling: 'lifo',
|
||||||
proxy: config.proxy,
|
proxy: config.proxy,
|
||||||
|
@ -72,21 +262,42 @@ export class HttpRequestService {
|
||||||
? new HttpsProxyAgent({
|
? new HttpsProxyAgent({
|
||||||
keepAlive: true,
|
keepAlive: true,
|
||||||
keepAliveMsecs: 30 * 1000,
|
keepAliveMsecs: 30 * 1000,
|
||||||
maxSockets,
|
maxSockets: this.maxSockets,
|
||||||
maxFreeSockets: 256,
|
maxFreeSockets: 256,
|
||||||
scheduling: 'lifo',
|
scheduling: 'lifo',
|
||||||
proxy: config.proxy,
|
proxy: config.proxy,
|
||||||
})
|
})
|
||||||
: this.https;
|
: this.https;
|
||||||
|
//#endregion
|
||||||
|
}
|
||||||
|
|
||||||
|
@bindThis
|
||||||
|
public getStandardUndiciFetcherOption(opts: undici.Agent.Options = {}, proxyOpts: undici.Agent.Options = {}) {
|
||||||
|
return {
|
||||||
|
agentOptions: {
|
||||||
|
...this.clientDefaults,
|
||||||
|
...opts,
|
||||||
|
},
|
||||||
|
...(this.config.proxy ? {
|
||||||
|
proxy: {
|
||||||
|
uri: this.config.proxy,
|
||||||
|
options: {
|
||||||
|
connections: this.maxSockets,
|
||||||
|
...proxyOpts,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} : {}),
|
||||||
|
userAgent: this.config.userAgent,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get agent by URL
|
* Get http agent by URL
|
||||||
* @param url URL
|
* @param url URL
|
||||||
* @param bypassProxy Allways bypass proxy
|
* @param bypassProxy Allways bypass proxy
|
||||||
*/
|
*/
|
||||||
@bindThis
|
@bindThis
|
||||||
public getAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent {
|
public getHttpAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent {
|
||||||
if (bypassProxy || (this.config.proxyBypassHosts || []).includes(url.hostname)) {
|
if (bypassProxy || (this.config.proxyBypassHosts || []).includes(url.hostname)) {
|
||||||
return url.protocol === 'http:' ? this.http : this.https;
|
return url.protocol === 'http:' ? this.http : this.https;
|
||||||
} else {
|
} else {
|
||||||
|
@ -94,67 +305,37 @@ export class HttpRequestService {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* check ip
|
||||||
|
*/
|
||||||
@bindThis
|
@bindThis
|
||||||
public async getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record<string, string>): Promise<unknown> {
|
public getConnectorWithIpCheck(connector: undici.buildConnector.connector, checkIp: IpChecker): undici.buildConnector.connectorAsync {
|
||||||
const res = await this.getResponse({
|
return (options, cb) => {
|
||||||
url,
|
connector(options, (err, socket) => {
|
||||||
method: 'GET',
|
this.logger.debug('Socket connector (with ip checker) called', socket);
|
||||||
headers: Object.assign({
|
if (err) {
|
||||||
'User-Agent': this.config.userAgent,
|
this.logger.error(`Socket error`, err)
|
||||||
Accept: accept,
|
cb(new Error(`Error while socket connecting\n${err}`), null);
|
||||||
}, headers ?? {}),
|
return;
|
||||||
timeout,
|
}
|
||||||
size: 1024 * 256,
|
|
||||||
});
|
|
||||||
|
|
||||||
return await res.json();
|
if (socket.remoteAddress == undefined) {
|
||||||
}
|
this.logger.error(`Socket error: remoteAddress is undefined`);
|
||||||
|
cb(new Error('remoteAddress is undefined (maybe socket destroyed)'), null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
@bindThis
|
// allow
|
||||||
public async getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: Record<string, string>): Promise<string> {
|
if (checkIp(socket.remoteAddress)) {
|
||||||
const res = await this.getResponse({
|
this.logger.debug(`Socket connected (ip ok): ${socket.localPort} => ${socket.remoteAddress}`);
|
||||||
url,
|
cb(null, socket);
|
||||||
method: 'GET',
|
return;
|
||||||
headers: Object.assign({
|
}
|
||||||
'User-Agent': this.config.userAgent,
|
|
||||||
Accept: accept,
|
|
||||||
}, headers ?? {}),
|
|
||||||
timeout,
|
|
||||||
});
|
|
||||||
|
|
||||||
return await res.text();
|
this.logger.error('IP is not allowed', socket);
|
||||||
}
|
cb(new StatusError('IP is not allowed', 403, 'IP is not allowed'), null);
|
||||||
|
socket.destroy();
|
||||||
@bindThis
|
});
|
||||||
public async getResponse(args: {
|
};
|
||||||
url: string,
|
|
||||||
method: string,
|
|
||||||
body?: string,
|
|
||||||
headers: Record<string, string>,
|
|
||||||
timeout?: number,
|
|
||||||
size?: number,
|
|
||||||
}): Promise<Response> {
|
|
||||||
const timeout = args.timeout ?? 10 * 1000;
|
|
||||||
|
|
||||||
const controller = new AbortController();
|
|
||||||
setTimeout(() => {
|
|
||||||
controller.abort();
|
|
||||||
}, timeout * 6);
|
|
||||||
|
|
||||||
const res = await fetch(args.url, {
|
|
||||||
method: args.method,
|
|
||||||
headers: args.headers,
|
|
||||||
body: args.body,
|
|
||||||
timeout,
|
|
||||||
size: args.size ?? 10 * 1024 * 1024,
|
|
||||||
agent: (url) => this.getAgentByUrl(url),
|
|
||||||
signal: controller.signal,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!res.ok) {
|
|
||||||
throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
|
|
||||||
}
|
|
||||||
|
|
||||||
return res;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -33,7 +33,7 @@ export class S3Service {
|
||||||
? false
|
? false
|
||||||
: meta.objectStorageS3ForcePathStyle,
|
: meta.objectStorageS3ForcePathStyle,
|
||||||
httpOptions: {
|
httpOptions: {
|
||||||
agent: this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy),
|
agent: this.httpRequestService.getHttpAgentByUrl(new URL(u), !meta.objectStorageUseProxy),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
|
@ -30,7 +30,7 @@ export class WebfingerService {
|
||||||
public async webfinger(query: string): Promise<IWebFinger> {
|
public async webfinger(query: string): Promise<IWebFinger> {
|
||||||
const url = this.genUrl(query);
|
const url = this.genUrl(query);
|
||||||
|
|
||||||
return await this.httpRequestService.getJson(url, 'application/jrd+json, application/json') as IWebFinger;
|
return await this.httpRequestService.getJson<IWebFinger>(url, 'application/jrd+json, application/json');
|
||||||
}
|
}
|
||||||
|
|
||||||
@bindThis
|
@bindThis
|
||||||
|
|
|
@ -5,8 +5,10 @@ import { DI } from '@/di-symbols.js';
|
||||||
import type { Config } from '@/config.js';
|
import type { Config } from '@/config.js';
|
||||||
import type { User } from '@/models/entities/User.js';
|
import type { User } from '@/models/entities/User.js';
|
||||||
import { UserKeypairStoreService } from '@/core/UserKeypairStoreService.js';
|
import { UserKeypairStoreService } from '@/core/UserKeypairStoreService.js';
|
||||||
import { HttpRequestService } from '@/core/HttpRequestService.js';
|
import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js';
|
||||||
|
import { LoggerService } from '@/core/LoggerService.js';
|
||||||
import { bindThis } from '@/decorators.js';
|
import { bindThis } from '@/decorators.js';
|
||||||
|
import type Logger from '@/logger.js';
|
||||||
|
|
||||||
type Request = {
|
type Request = {
|
||||||
url: string;
|
url: string;
|
||||||
|
@ -28,13 +30,21 @@ type PrivateKey = {
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class ApRequestService {
|
export class ApRequestService {
|
||||||
|
private undiciFetcher: UndiciFetcher;
|
||||||
|
private logger: Logger;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
@Inject(DI.config)
|
@Inject(DI.config)
|
||||||
private config: Config,
|
private config: Config,
|
||||||
|
|
||||||
private userKeypairStoreService: UserKeypairStoreService,
|
private userKeypairStoreService: UserKeypairStoreService,
|
||||||
private httpRequestService: HttpRequestService,
|
private httpRequestService: HttpRequestService,
|
||||||
|
private loggerService: LoggerService,
|
||||||
) {
|
) {
|
||||||
|
this.logger = this.loggerService?.getLogger('ap-request'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる
|
||||||
|
this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption({
|
||||||
|
maxRedirections: 0,
|
||||||
|
}), this.logger );
|
||||||
}
|
}
|
||||||
|
|
||||||
@bindThis
|
@bindThis
|
||||||
|
@ -148,16 +158,17 @@ export class ApRequestService {
|
||||||
url,
|
url,
|
||||||
body,
|
body,
|
||||||
additionalHeaders: {
|
additionalHeaders: {
|
||||||
'User-Agent': this.config.userAgent,
|
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
await this.httpRequestService.getResponse({
|
await this.undiciFetcher.fetch(
|
||||||
url,
|
url,
|
||||||
method: req.request.method,
|
{
|
||||||
headers: req.request.headers,
|
method: req.request.method,
|
||||||
body,
|
headers: req.request.headers,
|
||||||
});
|
body,
|
||||||
|
}
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -176,15 +187,16 @@ export class ApRequestService {
|
||||||
},
|
},
|
||||||
url,
|
url,
|
||||||
additionalHeaders: {
|
additionalHeaders: {
|
||||||
'User-Agent': this.config.userAgent,
|
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
const res = await this.httpRequestService.getResponse({
|
const res = await this.httpRequestService.fetch(
|
||||||
url,
|
url,
|
||||||
method: req.request.method,
|
{
|
||||||
headers: req.request.headers,
|
method: req.request.method,
|
||||||
});
|
headers: req.request.headers,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
return await res.json();
|
return await res.json();
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,7 +4,7 @@ import { InstanceActorService } from '@/core/InstanceActorService.js';
|
||||||
import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository } from '@/models/index.js';
|
import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository } from '@/models/index.js';
|
||||||
import type { Config } from '@/config.js';
|
import type { Config } from '@/config.js';
|
||||||
import { MetaService } from '@/core/MetaService.js';
|
import { MetaService } from '@/core/MetaService.js';
|
||||||
import { HttpRequestService } from '@/core/HttpRequestService.js';
|
import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js';
|
||||||
import { DI } from '@/di-symbols.js';
|
import { DI } from '@/di-symbols.js';
|
||||||
import { UtilityService } from '@/core/UtilityService.js';
|
import { UtilityService } from '@/core/UtilityService.js';
|
||||||
import { bindThis } from '@/decorators.js';
|
import { bindThis } from '@/decorators.js';
|
||||||
|
@ -12,11 +12,15 @@ import { isCollectionOrOrderedCollection } from './type.js';
|
||||||
import { ApDbResolverService } from './ApDbResolverService.js';
|
import { ApDbResolverService } from './ApDbResolverService.js';
|
||||||
import { ApRendererService } from './ApRendererService.js';
|
import { ApRendererService } from './ApRendererService.js';
|
||||||
import { ApRequestService } from './ApRequestService.js';
|
import { ApRequestService } from './ApRequestService.js';
|
||||||
|
import { LoggerService } from '@/core/LoggerService.js';
|
||||||
import type { IObject, ICollection, IOrderedCollection } from './type.js';
|
import type { IObject, ICollection, IOrderedCollection } from './type.js';
|
||||||
|
import type Logger from '@/logger.js';
|
||||||
|
|
||||||
export class Resolver {
|
export class Resolver {
|
||||||
private history: Set<string>;
|
private history: Set<string>;
|
||||||
private user?: ILocalUser;
|
private user?: ILocalUser;
|
||||||
|
private undiciFetcher: UndiciFetcher;
|
||||||
|
private logger: Logger;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private config: Config,
|
private config: Config,
|
||||||
|
@ -31,9 +35,14 @@ export class Resolver {
|
||||||
private httpRequestService: HttpRequestService,
|
private httpRequestService: HttpRequestService,
|
||||||
private apRendererService: ApRendererService,
|
private apRendererService: ApRendererService,
|
||||||
private apDbResolverService: ApDbResolverService,
|
private apDbResolverService: ApDbResolverService,
|
||||||
|
private loggerService: LoggerService,
|
||||||
private recursionLimit = 100,
|
private recursionLimit = 100,
|
||||||
) {
|
) {
|
||||||
this.history = new Set();
|
this.history = new Set();
|
||||||
|
this.logger = this.loggerService?.getLogger('ap-resolve'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる
|
||||||
|
this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption({
|
||||||
|
maxRedirections: 0,
|
||||||
|
}), this.logger);
|
||||||
}
|
}
|
||||||
|
|
||||||
@bindThis
|
@bindThis
|
||||||
|
@ -96,8 +105,8 @@ export class Resolver {
|
||||||
}
|
}
|
||||||
|
|
||||||
const object = (this.user
|
const object = (this.user
|
||||||
? await this.apRequestService.signedGet(value, this.user)
|
? await this.apRequestService.signedGet(value, this.user) as IObject
|
||||||
: await this.httpRequestService.getJson(value, 'application/activity+json, application/ld+json')) as IObject;
|
: await this.undiciFetcher.getJson<IObject>(value, 'application/activity+json, application/ld+json'));
|
||||||
|
|
||||||
if (object == null || (
|
if (object == null || (
|
||||||
Array.isArray(object['@context']) ?
|
Array.isArray(object['@context']) ?
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
import * as crypto from 'node:crypto';
|
import * as crypto from 'node:crypto';
|
||||||
import { Inject, Injectable } from '@nestjs/common';
|
import { Inject, Injectable } from '@nestjs/common';
|
||||||
import fetch from 'node-fetch';
|
|
||||||
import { HttpRequestService } from '@/core/HttpRequestService.js';
|
import { HttpRequestService } from '@/core/HttpRequestService.js';
|
||||||
import { bindThis } from '@/decorators.js';
|
import { bindThis } from '@/decorators.js';
|
||||||
import { CONTEXTS } from './misc/contexts.js';
|
import { CONTEXTS } from './misc/contexts.js';
|
||||||
|
@ -116,14 +115,19 @@ class LdSignature {
|
||||||
|
|
||||||
@bindThis
|
@bindThis
|
||||||
private async fetchDocument(url: string) {
|
private async fetchDocument(url: string) {
|
||||||
const json = await fetch(url, {
|
const json = await this.httpRequestService.fetch(
|
||||||
headers: {
|
url,
|
||||||
Accept: 'application/ld+json, application/json',
|
{
|
||||||
|
headers: {
|
||||||
|
Accept: 'application/ld+json, application/json',
|
||||||
|
},
|
||||||
|
// TODO
|
||||||
|
//timeout: this.loderTimeout,
|
||||||
},
|
},
|
||||||
// TODO
|
{
|
||||||
//timeout: this.loderTimeout,
|
noOkError: true,
|
||||||
agent: u => u.protocol === 'http:' ? this.httpRequestService.httpAgent : this.httpRequestService.httpsAgent,
|
}
|
||||||
}).then(res => {
|
).then(res => {
|
||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
throw `${res.status} ${res.statusText}`;
|
throw `${res.status} ${res.statusText}`;
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -33,24 +33,26 @@ export class WebhookDeliverProcessorService {
|
||||||
try {
|
try {
|
||||||
this.logger.debug(`delivering ${job.data.webhookId}`);
|
this.logger.debug(`delivering ${job.data.webhookId}`);
|
||||||
|
|
||||||
const res = await this.httpRequestService.getResponse({
|
const res = await this.httpRequestService.fetch(
|
||||||
url: job.data.to,
|
job.data.to,
|
||||||
method: 'POST',
|
{
|
||||||
headers: {
|
method: 'POST',
|
||||||
'User-Agent': 'Misskey-Hooks',
|
headers: {
|
||||||
'X-Misskey-Host': this.config.host,
|
'User-Agent': 'Misskey-Hooks',
|
||||||
'X-Misskey-Hook-Id': job.data.webhookId,
|
'X-Misskey-Host': this.config.host,
|
||||||
'X-Misskey-Hook-Secret': job.data.secret,
|
'X-Misskey-Hook-Id': job.data.webhookId,
|
||||||
},
|
'X-Misskey-Hook-Secret': job.data.secret,
|
||||||
body: JSON.stringify({
|
},
|
||||||
hookId: job.data.webhookId,
|
body: JSON.stringify({
|
||||||
userId: job.data.userId,
|
hookId: job.data.webhookId,
|
||||||
eventId: job.data.eventId,
|
userId: job.data.userId,
|
||||||
createdAt: job.data.createdAt,
|
eventId: job.data.eventId,
|
||||||
type: job.data.type,
|
createdAt: job.data.createdAt,
|
||||||
body: job.data.content,
|
type: job.data.type,
|
||||||
}),
|
body: job.data.content,
|
||||||
});
|
}),
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
this.webhooksRepository.update({ id: job.data.webhookId }, {
|
this.webhooksRepository.update({ id: job.data.webhookId }, {
|
||||||
latestSentAt: new Date(),
|
latestSentAt: new Date(),
|
||||||
|
|
|
@ -33,15 +33,16 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
|
||||||
private httpRequestService: HttpRequestService,
|
private httpRequestService: HttpRequestService,
|
||||||
) {
|
) {
|
||||||
super(meta, paramDef, async (ps, me) => {
|
super(meta, paramDef, async (ps, me) => {
|
||||||
const res = await this.httpRequestService.getResponse({
|
const res = await this.httpRequestService.fetch(
|
||||||
url: ps.url,
|
ps.url,
|
||||||
method: 'GET',
|
{
|
||||||
headers: Object.assign({
|
method: 'GET',
|
||||||
'User-Agent': config.userAgent,
|
headers: {
|
||||||
Accept: 'application/rss+xml, */*',
|
Accept: 'application/rss+xml, */*',
|
||||||
}),
|
},
|
||||||
timeout: 5000,
|
// timeout: 5000,
|
||||||
});
|
}
|
||||||
|
);
|
||||||
|
|
||||||
const text = await res.text();
|
const text = await res.text();
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,4 @@
|
||||||
import { URLSearchParams } from 'node:url';
|
import { URLSearchParams } from 'node:url';
|
||||||
import fetch from 'node-fetch';
|
|
||||||
import { Inject, Injectable } from '@nestjs/common';
|
import { Inject, Injectable } from '@nestjs/common';
|
||||||
import type { NotesRepository } from '@/models/index.js';
|
import type { NotesRepository } from '@/models/index.js';
|
||||||
import { Endpoint } from '@/server/api/endpoint-base.js';
|
import { Endpoint } from '@/server/api/endpoint-base.js';
|
||||||
|
@ -84,25 +83,27 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
|
||||||
|
|
||||||
const endpoint = instance.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate';
|
const endpoint = instance.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate';
|
||||||
|
|
||||||
const res = await fetch(endpoint, {
|
const res = await this.httpRequestService.fetch(
|
||||||
method: 'POST',
|
endpoint,
|
||||||
headers: {
|
{
|
||||||
'Content-Type': 'application/x-www-form-urlencoded',
|
method: 'POST',
|
||||||
'User-Agent': config.userAgent,
|
headers: {
|
||||||
Accept: 'application/json, */*',
|
'Content-Type': 'application/x-www-form-urlencoded',
|
||||||
|
Accept: 'application/json, */*',
|
||||||
|
},
|
||||||
|
body: params.toString(),
|
||||||
},
|
},
|
||||||
body: params,
|
{
|
||||||
// TODO
|
noOkError: false,
|
||||||
//timeout: 10000,
|
}
|
||||||
agent: (url) => this.httpRequestService.getAgentByUrl(url),
|
);
|
||||||
});
|
|
||||||
|
|
||||||
const json = (await res.json()) as {
|
const json = (await res.json()) as {
|
||||||
translations: {
|
translations: {
|
||||||
detected_source_language: string;
|
detected_source_language: string;
|
||||||
text: string;
|
text: string;
|
||||||
}[];
|
}[];
|
||||||
};
|
};
|
||||||
|
|
||||||
return {
|
return {
|
||||||
sourceLang: json.translations[0].detected_source_language,
|
sourceLang: json.translations[0].detected_source_language,
|
||||||
|
|
|
@ -181,7 +181,7 @@ export class DiscordServerService {
|
||||||
}
|
}
|
||||||
}));
|
}));
|
||||||
|
|
||||||
const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', 10 * 1000, {
|
const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', {
|
||||||
'Authorization': `Bearer ${accessToken}`,
|
'Authorization': `Bearer ${accessToken}`,
|
||||||
})) as Record<string, unknown>;
|
})) as Record<string, unknown>;
|
||||||
|
|
||||||
|
@ -249,7 +249,7 @@ export class DiscordServerService {
|
||||||
}
|
}
|
||||||
}));
|
}));
|
||||||
|
|
||||||
const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', 10 * 1000, {
|
const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', {
|
||||||
'Authorization': `Bearer ${accessToken}`,
|
'Authorization': `Bearer ${accessToken}`,
|
||||||
})) as Record<string, unknown>;
|
})) as Record<string, unknown>;
|
||||||
if (typeof id !== 'string' || typeof username !== 'string' || typeof discriminator !== 'string') {
|
if (typeof id !== 'string' || typeof username !== 'string' || typeof discriminator !== 'string') {
|
||||||
|
|
|
@ -174,7 +174,7 @@ export class GithubServerService {
|
||||||
}
|
}
|
||||||
}));
|
}));
|
||||||
|
|
||||||
const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, {
|
const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', {
|
||||||
'Authorization': `bearer ${accessToken}`,
|
'Authorization': `bearer ${accessToken}`,
|
||||||
})) as Record<string, unknown>;
|
})) as Record<string, unknown>;
|
||||||
if (typeof login !== 'string' || typeof id !== 'string') {
|
if (typeof login !== 'string' || typeof id !== 'string') {
|
||||||
|
@ -223,7 +223,7 @@ export class GithubServerService {
|
||||||
}
|
}
|
||||||
}));
|
}));
|
||||||
|
|
||||||
const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, {
|
const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', {
|
||||||
'Authorization': `bearer ${accessToken}`,
|
'Authorization': `bearer ${accessToken}`,
|
||||||
})) as Record<string, unknown>;
|
})) as Record<string, unknown>;
|
||||||
|
|
||||||
|
|
|
@ -63,9 +63,8 @@ export class UrlPreviewService {
|
||||||
this.logger.info(meta.summalyProxy
|
this.logger.info(meta.summalyProxy
|
||||||
? `(Proxy) Getting preview of ${url}@${lang} ...`
|
? `(Proxy) Getting preview of ${url}@${lang} ...`
|
||||||
: `Getting preview of ${url}@${lang} ...`);
|
: `Getting preview of ${url}@${lang} ...`);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const summary = meta.summalyProxy ? await this.httpRequestService.getJson(`${meta.summalyProxy}?${query({
|
const summary = meta.summalyProxy ? await this.httpRequestService.getJson<ReturnType<typeof summaly.default>>(`${meta.summalyProxy}?${query({
|
||||||
url: url,
|
url: url,
|
||||||
lang: lang ?? 'ja-JP',
|
lang: lang ?? 'ja-JP',
|
||||||
})}`) : await summaly.default(url, {
|
})}`) : await summaly.default(url, {
|
||||||
|
|
19
yarn.lock
19
yarn.lock
|
@ -4272,6 +4272,7 @@ __metadata:
|
||||||
typeorm: 0.3.11
|
typeorm: 0.3.11
|
||||||
typescript: 4.9.4
|
typescript: 4.9.4
|
||||||
ulid: 2.3.0
|
ulid: 2.3.0
|
||||||
|
undici: ^5.14.0
|
||||||
unzipper: 0.10.11
|
unzipper: 0.10.11
|
||||||
uuid: 9.0.0
|
uuid: 9.0.0
|
||||||
vary: 1.1.2
|
vary: 1.1.2
|
||||||
|
@ -16665,6 +16666,15 @@ __metadata:
|
||||||
languageName: node
|
languageName: node
|
||||||
linkType: hard
|
linkType: hard
|
||||||
|
|
||||||
|
"undici@npm:^5.14.0, undici@npm:^5.5.1":
|
||||||
|
version: 5.14.0
|
||||||
|
resolution: "undici@npm:5.14.0"
|
||||||
|
dependencies:
|
||||||
|
busboy: ^1.6.0
|
||||||
|
checksum: 7a076e44d84b25844b4eb657034437b8b9bb91f17d347de474fdea1d4263ce7ae9406db79cd30de5642519277b4893f43073258bcc8fed420b295da3fdd11b26
|
||||||
|
languageName: node
|
||||||
|
linkType: hard
|
||||||
|
|
||||||
"undici@npm:^5.2.0":
|
"undici@npm:^5.2.0":
|
||||||
version: 5.13.0
|
version: 5.13.0
|
||||||
resolution: "undici@npm:5.13.0"
|
resolution: "undici@npm:5.13.0"
|
||||||
|
@ -16674,15 +16684,6 @@ __metadata:
|
||||||
languageName: node
|
languageName: node
|
||||||
linkType: hard
|
linkType: hard
|
||||||
|
|
||||||
"undici@npm:^5.5.1":
|
|
||||||
version: 5.14.0
|
|
||||||
resolution: "undici@npm:5.14.0"
|
|
||||||
dependencies:
|
|
||||||
busboy: ^1.6.0
|
|
||||||
checksum: 7a076e44d84b25844b4eb657034437b8b9bb91f17d347de474fdea1d4263ce7ae9406db79cd30de5642519277b4893f43073258bcc8fed420b295da3fdd11b26
|
|
||||||
languageName: node
|
|
||||||
linkType: hard
|
|
||||||
|
|
||||||
"union-value@npm:^1.0.0":
|
"union-value@npm:^1.0.0":
|
||||||
version: 1.0.1
|
version: 1.0.1
|
||||||
resolution: "union-value@npm:1.0.1"
|
resolution: "union-value@npm:1.0.1"
|
||||||
|
|
Loading…
Reference in a new issue