From 36b9a0d42fa4bd7036d5d4a11203006d0ceb59f0 Mon Sep 17 00:00:00 2001 From: MeiMei <30769358+mei23@users.noreply.github.com> Date: Sun, 12 Apr 2020 20:32:34 +0900 Subject: [PATCH] =?UTF-8?q?=E3=83=97=E3=83=AD=E3=82=AD=E3=82=B7=E3=81=AE?= =?UTF-8?q?=E9=99=A4=E5=A4=96=E3=83=9B=E3=82=B9=E3=83=88=20(#6244)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * プロキシの除外ホスト * オブジェクトストレージとの通信にProxyを使うかを選択できるように * fix lint * コメント Co-authored-by: rinsuki <428rinsuki+git@gmail.com> --- .config/example.yml | 5 ++ locales/ja-JP.yml | 2 + .../1586624197029-AddObjectStorageUseProxy.ts | 14 ++++++ src/client/pages/instance/settings.vue | 4 ++ src/config/types.ts | 1 + src/misc/download-url.ts | 4 +- src/misc/fetch.ts | 49 +++++++++++++++---- src/models/entities/meta.ts | 5 ++ src/remote/activitypub/request.ts | 4 +- src/server/api/endpoints/admin/update-meta.ts | 8 +++ src/server/api/endpoints/meta.ts | 1 + src/services/drive/s3.ts | 8 ++- 12 files changed, 89 insertions(+), 16 deletions(-) create mode 100644 migration/1586624197029-AddObjectStorageUseProxy.ts diff --git a/.config/example.yml b/.config/example.yml index 201082cce..1794dc9a8 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -142,6 +142,11 @@ id: 'aid' # Proxy for HTTP/HTTPS #proxy: http://127.0.0.1:3128 +#proxyBypassHosts: [ +# 'example.com', +# '192.0.2.8' +#] + # Proxy for SMTP/SMTPS #proxySmtp: http://127.0.0.1:3128 # use HTTP/1.1 CONNECT #proxySmtp: socks4://127.0.0.1:1080 # use SOCKS4 diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 29a4c2d3d..082709153 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -454,6 +454,8 @@ objectStorageRegion: "Region" objectStorageRegionDesc: "'xx-east-1'のようなregionを指定してください。使用サービスにregionの概念がない場合は、空または'us-east-1'にしてください。" objectStorageUseSSL: "SSLを使用する" objectStorageUseSSLDesc: "API接続にhttpsを使用しない場合はオフにしてください" +objectStorageUseProxy: "Proxyを利用する" +objectStorageUseProxyDesc: "API接続にproxyを利用しない場合はオフにしてください" serverLogs: "サーバーログ" deleteAll: "全て削除" showFixedPostForm: "タイムライン上部に投稿フォームを表示する" diff --git a/migration/1586624197029-AddObjectStorageUseProxy.ts b/migration/1586624197029-AddObjectStorageUseProxy.ts new file mode 100644 index 000000000..deadf9483 --- /dev/null +++ b/migration/1586624197029-AddObjectStorageUseProxy.ts @@ -0,0 +1,14 @@ +import {MigrationInterface, QueryRunner} from 'typeorm'; + +export class AddObjectStorageUseProxy1586624197029 implements MigrationInterface { + name = 'AddObjectStorageUseProxy1586624197029' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageUseProxy" boolean NOT NULL DEFAULT true`, undefined); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageUseProxy"`, undefined); + } + +} diff --git a/src/client/pages/instance/settings.vue b/src/client/pages/instance/settings.vue index f0a123f27..f7db4aa10 100644 --- a/src/client/pages/instance/settings.vue +++ b/src/client/pages/instance/settings.vue @@ -116,6 +116,7 @@ Secret key {{ $t('objectStorageUseSSL') }} + {{ $t('objectStorageUseProxy') }}
@@ -249,6 +250,7 @@ export default Vue.extend({ objectStorageAccessKey: null, objectStorageSecretKey: null, objectStorageUseSSL: false, + objectStorageUseProxy: false, enableTwitterIntegration: false, twitterConsumerKey: null, twitterConsumerSecret: null, @@ -303,6 +305,7 @@ export default Vue.extend({ this.objectStorageAccessKey = this.meta.objectStorageAccessKey; this.objectStorageSecretKey = this.meta.objectStorageSecretKey; this.objectStorageUseSSL = this.meta.objectStorageUseSSL; + this.objectStorageUseProxy = this.meta.objectStorageUseProxy; this.enableTwitterIntegration = this.meta.enableTwitterIntegration; this.twitterConsumerKey = this.meta.twitterConsumerKey; this.twitterConsumerSecret = this.meta.twitterConsumerSecret; @@ -411,6 +414,7 @@ export default Vue.extend({ objectStorageAccessKey: this.objectStorageAccessKey ? this.objectStorageAccessKey : null, objectStorageSecretKey: this.objectStorageSecretKey ? this.objectStorageSecretKey : null, objectStorageUseSSL: this.objectStorageUseSSL, + objectStorageUseProxy: this.objectStorageUseProxy, enableTwitterIntegration: this.enableTwitterIntegration, twitterConsumerKey: this.twitterConsumerKey, twitterConsumerSecret: this.twitterConsumerSecret, diff --git a/src/config/types.ts b/src/config/types.ts index a33901bde..4f025750b 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -35,6 +35,7 @@ export type Source = { proxy?: string; proxySmtp?: string; + proxyBypassHosts?: string[]; accesslog?: string; diff --git a/src/misc/download-url.ts b/src/misc/download-url.ts index 3f42fb3be..9c8439f2c 100644 --- a/src/misc/download-url.ts +++ b/src/misc/download-url.ts @@ -2,7 +2,7 @@ import * as fs from 'fs'; import * as stream from 'stream'; import * as util from 'util'; import fetch from 'node-fetch'; -import { httpAgent, httpsAgent } from './fetch'; +import { getAgentByUrl } from './fetch'; import { AbortController } from 'abort-controller'; import config from '../config'; import * as chalk from 'chalk'; @@ -25,7 +25,7 @@ export async function downloadUrl(url: string, path: string) { }, timeout: 10 * 1000, signal: controller.signal, - agent: u => u.protocol == 'http:' ? httpAgent : httpsAgent, + agent: getAgentByUrl, }); if (!response.ok) { diff --git a/src/misc/fetch.ts b/src/misc/fetch.ts index 887aae165..358bc2503 100644 --- a/src/misc/fetch.ts +++ b/src/misc/fetch.ts @@ -13,7 +13,7 @@ export async function getJson(url: string, accept = 'application/json, */*', tim Accept: accept }, headers || {}), timeout, - agent: u => u.protocol == 'http:' ? httpAgent : httpsAgent, + agent: getAgentByUrl, }); if (!res.ok) { @@ -27,17 +27,46 @@ export async function getJson(url: string, accept = 'application/json, */*', tim return await res.json(); } +/** + * Get http non-proxy agent + */ +const _http = new http.Agent({ + keepAlive: true, + keepAliveMsecs: 30 * 1000, +}); + +/** + * Get https non-proxy agent + */ +const _https = new https.Agent({ + keepAlive: true, + keepAliveMsecs: 30 * 1000, + lookup: cache.lookup, +}); + +/** + * Get http proxy or non-proxy agent + */ export const httpAgent = config.proxy ? new HttpProxyAgent(config.proxy) - : new http.Agent({ - keepAlive: true, - keepAliveMsecs: 30 * 1000, - }); + : _http; +/** + * Get https proxy or non-proxy agent + */ export const httpsAgent = config.proxy ? new HttpsProxyAgent(config.proxy) - : new https.Agent({ - keepAlive: true, - keepAliveMsecs: 30 * 1000, - lookup: cache.lookup, - }); + : _https; + +/** + * Get agent by URL + * @param url URL + * @param bypassProxy Allways bypass proxy + */ +export function getAgentByUrl(url: URL, bypassProxy = false) { + if (bypassProxy || (config.proxyBypassHosts || []).includes(url.hostname)) { + return url.protocol == 'http:' ? _http : _https; + } else { + return url.protocol == 'http:' ? httpAgent : httpsAgent; + } +} diff --git a/src/models/entities/meta.ts b/src/models/entities/meta.ts index ee62cb24b..bb463c52f 100644 --- a/src/models/entities/meta.ts +++ b/src/models/entities/meta.ts @@ -348,4 +348,9 @@ export class Meta { default: true, }) public objectStorageUseSSL: boolean; + + @Column('boolean', { + default: true, + }) + public objectStorageUseProxy: boolean; } diff --git a/src/remote/activitypub/request.ts b/src/remote/activitypub/request.ts index 24540827b..ab3ae025a 100644 --- a/src/remote/activitypub/request.ts +++ b/src/remote/activitypub/request.ts @@ -6,7 +6,7 @@ import config from '../../config'; import { ILocalUser } from '../../models/entities/user'; import { UserKeypairs } from '../../models'; import { ensure } from '../../prelude/ensure'; -import { httpsAgent } from '../../misc/fetch'; +import { getAgentByUrl } from '../../misc/fetch'; export default async (user: ILocalUser, url: string, object: any) => { const timeout = 10 * 1000; @@ -25,7 +25,7 @@ export default async (user: ILocalUser, url: string, object: any) => { await new Promise((resolve, reject) => { const req = https.request({ - agent: httpsAgent, + agent: getAgentByUrl(new URL(`https://example.net`)), protocol, hostname, port, diff --git a/src/server/api/endpoints/admin/update-meta.ts b/src/server/api/endpoints/admin/update-meta.ts index e9527d9ac..1bc20029e 100644 --- a/src/server/api/endpoints/admin/update-meta.ts +++ b/src/server/api/endpoints/admin/update-meta.ts @@ -394,6 +394,10 @@ export const meta = { objectStorageUseSSL: { validator: $.optional.bool }, + + objectStorageUseProxy: { + validator: $.optional.bool + } } }; @@ -632,6 +636,10 @@ export default define(meta, async (ps, me) => { set.objectStorageUseSSL = ps.objectStorageUseSSL; } + if (ps.objectStorageUseProxy !== undefined) { + set.objectStorageUseProxy = ps.objectStorageUseProxy; + } + await getConnection().transaction(async transactionalEntityManager => { const meta = await transactionalEntityManager.findOne(Meta, { order: { diff --git a/src/server/api/endpoints/meta.ts b/src/server/api/endpoints/meta.ts index 280a6fbae..179355489 100644 --- a/src/server/api/endpoints/meta.ts +++ b/src/server/api/endpoints/meta.ts @@ -190,6 +190,7 @@ export default define(meta, async (ps, me) => { response.objectStorageAccessKey = instance.objectStorageAccessKey; response.objectStorageSecretKey = instance.objectStorageSecretKey; response.objectStorageUseSSL = instance.objectStorageUseSSL; + response.objectStorageUseProxy = instance.objectStorageUseProxy; } return response; diff --git a/src/services/drive/s3.ts b/src/services/drive/s3.ts index 2cbeef106..abe3c166a 100644 --- a/src/services/drive/s3.ts +++ b/src/services/drive/s3.ts @@ -1,8 +1,12 @@ import * as S3 from 'aws-sdk/clients/s3'; import { Meta } from '../../models/entities/meta'; -import { httpsAgent, httpAgent } from '../../misc/fetch'; +import { getAgentByUrl } from '../../misc/fetch'; export function getS3(meta: Meta) { + const u = meta.objectStorageEndpoint != null + ? `${meta.objectStorageUseSSL ? 'https://' : 'http://'}${meta.objectStorageEndpoint}` + : `${meta.objectStorageUseSSL ? 'https://' : 'http://'}example.net`; + return new S3({ endpoint: meta.objectStorageEndpoint || undefined, accessKeyId: meta.objectStorageAccessKey!, @@ -11,7 +15,7 @@ export function getS3(meta: Meta) { sslEnabled: meta.objectStorageUseSSL, s3ForcePathStyle: !!meta.objectStorageEndpoint, httpOptions: { - agent: meta.objectStorageUseSSL ? httpsAgent : httpAgent + agent: getAgentByUrl(new URL(u), !meta.objectStorageUseProxy) } }); }