From 0bb0c329087149355a9968e7c142b9ecdf1023ec Mon Sep 17 00:00:00 2001 From: syuilo Date: Mon, 9 Oct 2023 20:31:39 +0900 Subject: [PATCH] =?UTF-8?q?enhance(backend):=20Redis=E3=81=B8=E3=81=AETL?= =?UTF-8?q?=E3=81=AE=E6=A7=8B=E7=AF=89=E3=82=92List=E3=81=A7=E8=A1=8C?= =?UTF-8?q?=E3=81=86=E3=82=88=E3=81=86=E3=81=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #11404 --- packages/backend/src/core/AntennaService.ts | 12 +- packages/backend/src/core/CoreModule.ts | 6 + .../backend/src/core/NoteCreateService.ts | 110 +++------------ .../backend/src/core/RedisTimelineService.ts | 80 +++++++++++ packages/backend/src/core/RoleService.ts | 9 +- .../server/api/endpoints/antennas/notes.ts | 16 +-- .../server/api/endpoints/channels/timeline.ts | 125 +++++++++--------- .../api/endpoints/notes/hybrid-timeline.ts | 28 ++-- .../api/endpoints/notes/local-timeline.ts | 21 ++- .../server/api/endpoints/notes/timeline.ts | 19 ++- .../api/endpoints/notes/user-list-timeline.ts | 19 ++- .../src/server/api/endpoints/roles/notes.ts | 14 +- .../src/server/api/endpoints/users/notes.ts | 43 ++---- 13 files changed, 239 insertions(+), 263 deletions(-) create mode 100644 packages/backend/src/core/RedisTimelineService.ts diff --git a/packages/backend/src/core/AntennaService.ts b/packages/backend/src/core/AntennaService.ts index b64120772..ca7624b1d 100644 --- a/packages/backend/src/core/AntennaService.ts +++ b/packages/backend/src/core/AntennaService.ts @@ -16,6 +16,7 @@ import type { AntennasRepository, UserListMembershipsRepository } from '@/models import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import type { GlobalEvents } from '@/core/GlobalEventService.js'; +import { RedisTimelineService } from '@/core/RedisTimelineService.js'; import type { OnApplicationShutdown } from '@nestjs/common'; @Injectable() @@ -38,6 +39,7 @@ export class AntennaService implements OnApplicationShutdown { private utilityService: UtilityService, private globalEventService: GlobalEventService, + private redisTimelineService: RedisTimelineService, ) { this.antennasFetched = false; this.antennas = []; @@ -77,9 +79,6 @@ export class AntennaService implements OnApplicationShutdown { @bindThis public async addNoteToAntennas(note: MiNote, noteUser: { id: MiUser['id']; username: string; host: string | null; }): Promise { - // リモートから遅れて届いた(もしくは後から追加された)投稿日時が古い投稿が追加されるとページネーション時に問題を引き起こすため、3分以内に投稿されたもののみを追加する - if (Date.now() - note.createdAt.getTime() > 1000 * 60 * 3) return; - const antennas = await this.getAntennas(); const antennasWithMatchResult = await Promise.all(antennas.map(antenna => this.checkHitAntenna(antenna, note, noteUser).then(hit => [antenna, hit] as const))); const matchedAntennas = antennasWithMatchResult.filter(([, hit]) => hit).map(([antenna]) => antenna); @@ -87,12 +86,7 @@ export class AntennaService implements OnApplicationShutdown { const redisPipeline = this.redisForTimelines.pipeline(); for (const antenna of matchedAntennas) { - redisPipeline.xadd( - `antennaTimeline:${antenna.id}`, - 'MAXLEN', '~', '200', - '*', - 'note', note.id); - + this.redisTimelineService.push(`antennaTimeline:${antenna.id}`, note.id, 200, redisPipeline); this.globalEventService.publishAntennaStream(antenna.id, 'note', note); } diff --git a/packages/backend/src/core/CoreModule.ts b/packages/backend/src/core/CoreModule.ts index 1984d9e6c..0dc025d99 100644 --- a/packages/backend/src/core/CoreModule.ts +++ b/packages/backend/src/core/CoreModule.ts @@ -61,6 +61,7 @@ import { FileInfoService } from './FileInfoService.js'; import { SearchService } from './SearchService.js'; import { ClipService } from './ClipService.js'; import { FeaturedService } from './FeaturedService.js'; +import { RedisTimelineService } from './RedisTimelineService.js'; import { ChartLoggerService } from './chart/ChartLoggerService.js'; import FederationChart from './chart/charts/federation.js'; import NotesChart from './chart/charts/notes.js'; @@ -189,6 +190,7 @@ const $FileInfoService: Provider = { provide: 'FileInfoService', useExisting: Fi const $SearchService: Provider = { provide: 'SearchService', useExisting: SearchService }; const $ClipService: Provider = { provide: 'ClipService', useExisting: ClipService }; const $FeaturedService: Provider = { provide: 'FeaturedService', useExisting: FeaturedService }; +const $RedisTimelineService: Provider = { provide: 'RedisTimelineService', useExisting: RedisTimelineService }; const $ChartLoggerService: Provider = { provide: 'ChartLoggerService', useExisting: ChartLoggerService }; const $FederationChart: Provider = { provide: 'FederationChart', useExisting: FederationChart }; @@ -321,6 +323,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting SearchService, ClipService, FeaturedService, + RedisTimelineService, ChartLoggerService, FederationChart, NotesChart, @@ -446,6 +449,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $SearchService, $ClipService, $FeaturedService, + $RedisTimelineService, $ChartLoggerService, $FederationChart, $NotesChart, @@ -572,6 +576,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting SearchService, ClipService, FeaturedService, + RedisTimelineService, FederationChart, NotesChart, UsersChart, @@ -696,6 +701,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $SearchService, $ClipService, $FeaturedService, + $RedisTimelineService, $FederationChart, $NotesChart, $UsersChart, diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 65beb9f97..2a7346712 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -54,6 +54,7 @@ import { RoleService } from '@/core/RoleService.js'; import { MetaService } from '@/core/MetaService.js'; import { SearchService } from '@/core/SearchService.js'; import { FeaturedService } from '@/core/FeaturedService.js'; +import { RedisTimelineService } from '@/core/RedisTimelineService.js'; type NotificationType = 'reply' | 'renote' | 'quote' | 'mention'; @@ -194,6 +195,7 @@ export class NoteCreateService implements OnApplicationShutdown { private idService: IdService, private globalEventService: GlobalEventService, private queueService: QueueService, + private redisTimelineService: RedisTimelineService, private noteReadService: NoteReadService, private notificationService: NotificationService, private relayService: RelayService, @@ -347,14 +349,6 @@ export class NoteCreateService implements OnApplicationShutdown { const note = await this.insertNote(user, data, tags, emojis, mentionedUsers); - if (data.channel) { - this.redisForTimelines.xadd( - `channelTimeline:${data.channel.id}`, - 'MAXLEN', '~', this.config.perChannelMaxNoteCacheCount.toString(), - '*', - 'note', note.id); - } - setImmediate('post created', { signal: this.#shutdownController.signal }).then( () => this.postNoteCreated(note, user, data, silent, tags!, mentionedUsers!), () => { /* aborted, ignore this */ }, @@ -822,20 +816,14 @@ export class NoteCreateService implements OnApplicationShutdown { @bindThis private async pushToTl(note: MiNote, user: { id: MiUser['id']; host: MiUser['host']; }) { - // リモートから遅れて届いた(もしくは後から追加された)投稿日時が古い投稿が追加されるとページネーション時に問題を引き起こすため、3分以内に投稿されたもののみを追加する - // TODO: https://github.com/misskey-dev/misskey/issues/11404#issuecomment-1752480890 をやる - if (note.userHost != null && (Date.now() - note.createdAt.getTime()) > 1000 * 60 * 3) return; - const meta = await this.metaService.fetch(); - const redisPipeline = this.redisForTimelines.pipeline(); + const r = this.redisForTimelines.pipeline(); if (note.channelId) { - redisPipeline.xadd( - `userTimelineWithChannel:${user.id}`, - 'MAXLEN', '~', note.userHost == null ? meta.perLocalUserUserTimelineCacheMax.toString() : meta.perRemoteUserUserTimelineCacheMax.toString(), - '*', - 'note', note.id); + this.redisTimelineService.push(`channelTimeline:${note.channelId}`, note.id, this.config.perChannelMaxNoteCacheCount, r); + + this.redisTimelineService.push(`userTimelineWithChannel:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); const channelFollowings = await this.channelFollowingsRepository.find({ where: { @@ -845,18 +833,9 @@ export class NoteCreateService implements OnApplicationShutdown { }); for (const channelFollowing of channelFollowings) { - redisPipeline.xadd( - `homeTimeline:${channelFollowing.followerId}`, - 'MAXLEN', '~', meta.perUserHomeTimelineCacheMax.toString(), - '*', - 'note', note.id); - + this.redisTimelineService.push(`homeTimeline:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r); if (note.fileIds.length > 0) { - redisPipeline.xadd( - `homeTimelineWithFiles:${channelFollowing.followerId}`, - 'MAXLEN', '~', (meta.perUserHomeTimelineCacheMax / 2).toString(), - '*', - 'note', note.id); + this.redisTimelineService.push(`homeTimelineWithFiles:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); } } } else { @@ -894,18 +873,9 @@ export class NoteCreateService implements OnApplicationShutdown { if (!following.withReplies) continue; } - redisPipeline.xadd( - `homeTimeline:${following.followerId}`, - 'MAXLEN', '~', meta.perUserHomeTimelineCacheMax.toString(), - '*', - 'note', note.id); - + this.redisTimelineService.push(`homeTimeline:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r); if (note.fileIds.length > 0) { - redisPipeline.xadd( - `homeTimelineWithFiles:${following.followerId}`, - 'MAXLEN', '~', (meta.perUserHomeTimelineCacheMax / 2).toString(), - '*', - 'note', note.id); + this.redisTimelineService.push(`homeTimelineWithFiles:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); } } @@ -921,72 +891,32 @@ export class NoteCreateService implements OnApplicationShutdown { if (!userListMembership.withReplies) continue; } - redisPipeline.xadd( - `userListTimeline:${userListMembership.userListId}`, - 'MAXLEN', '~', meta.perUserListTimelineCacheMax.toString(), - '*', - 'note', note.id); - + this.redisTimelineService.push(`userListTimeline:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax, r); if (note.fileIds.length > 0) { - redisPipeline.xadd( - `userListTimelineWithFiles:${userListMembership.userListId}`, - 'MAXLEN', '~', (meta.perUserListTimelineCacheMax / 2).toString(), - '*', - 'note', note.id); + this.redisTimelineService.push(`userListTimelineWithFiles:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax / 2, r); } } if (note.visibility !== 'specified' || !note.visibleUserIds.some(v => v === user.id)) { // 自分自身のHTL - redisPipeline.xadd( - `homeTimeline:${user.id}`, - 'MAXLEN', '~', meta.perUserHomeTimelineCacheMax.toString(), - '*', - 'note', note.id); - + this.redisTimelineService.push(`homeTimeline:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax, r); if (note.fileIds.length > 0) { - redisPipeline.xadd( - `homeTimelineWithFiles:${user.id}`, - 'MAXLEN', '~', (meta.perUserHomeTimelineCacheMax / 2).toString(), - '*', - 'note', note.id); + this.redisTimelineService.push(`homeTimelineWithFiles:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); } } // 自分自身以外への返信 if (note.replyId && note.replyUserId !== note.userId) { - redisPipeline.xadd( - `userTimelineWithReplies:${user.id}`, - 'MAXLEN', '~', note.userHost == null ? meta.perLocalUserUserTimelineCacheMax.toString() : meta.perRemoteUserUserTimelineCacheMax.toString(), - '*', - 'note', note.id); + this.redisTimelineService.push(`userTimelineWithReplies:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); } else { - redisPipeline.xadd( - `userTimeline:${user.id}`, - 'MAXLEN', '~', note.userHost == null ? meta.perLocalUserUserTimelineCacheMax.toString() : meta.perRemoteUserUserTimelineCacheMax.toString(), - '*', - 'note', note.id); - + this.redisTimelineService.push(`userTimeline:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); if (note.fileIds.length > 0) { - redisPipeline.xadd( - `userTimelineWithFiles:${user.id}`, - 'MAXLEN', '~', note.userHost == null ? (meta.perLocalUserUserTimelineCacheMax / 2).toString() : (meta.perRemoteUserUserTimelineCacheMax / 2).toString(), - '*', - 'note', note.id); + this.redisTimelineService.push(`userTimelineWithFiles:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax / 2 : meta.perRemoteUserUserTimelineCacheMax / 2, r); } if (note.visibility === 'public' && note.userHost == null) { - redisPipeline.xadd( - 'localTimeline', - 'MAXLEN', '~', '1000', - '*', - 'note', note.id); - + this.redisTimelineService.push('localTimeline', note.id, 1000, r); if (note.fileIds.length > 0) { - redisPipeline.xadd( - 'localTimelineWithFiles', - 'MAXLEN', '~', '500', - '*', - 'note', note.id); + this.redisTimelineService.push('localTimelineWithFiles', note.id, 500, r); } } } @@ -998,7 +928,7 @@ export class NoteCreateService implements OnApplicationShutdown { } } - redisPipeline.exec(); + r.exec(); } @bindThis diff --git a/packages/backend/src/core/RedisTimelineService.ts b/packages/backend/src/core/RedisTimelineService.ts new file mode 100644 index 000000000..f0ca2726d --- /dev/null +++ b/packages/backend/src/core/RedisTimelineService.ts @@ -0,0 +1,80 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import * as Redis from 'ioredis'; +import { DI } from '@/di-symbols.js'; +import { bindThis } from '@/decorators.js'; +import { IdService } from '@/core/IdService.js'; + +@Injectable() +export class RedisTimelineService { + constructor( + @Inject(DI.redisForTimelines) + private redisForTimelines: Redis.Redis, + + private idService: IdService, + ) { + } + + @bindThis + public push(tl: string, id: string, maxlen: number, pipeline: Redis.ChainableCommander) { + // リモートから遅れて届いた(もしくは後から追加された)投稿日時が古い投稿が追加されるとページネーション時に問題を引き起こすため、 + // 3分以内に投稿されたものでない場合、Redisにある最古のIDより新しい場合のみ追加する + if (this.idService.parse(id).date.getTime() > Date.now() - 1000 * 60 * 3) { + pipeline.lpush('list:' + tl, id); + if (Math.random() < 0.1) { // 10%の確率でトリム + pipeline.ltrim('list:' + tl, 0, maxlen - 1); + } + } else { + // 末尾のIDを取得 + this.redisForTimelines.lindex('list:' + tl, -1).then(lastId => { + if (lastId == null || (this.idService.parse(id).date.getTime() > this.idService.parse(lastId).date.getTime())) { + this.redisForTimelines.lpush('list:' + tl, id); + } else { + Promise.resolve(); + } + }); + } + } + + @bindThis + public get(name: string, untilId?: string | null, sinceId?: string | null) { + if (untilId && sinceId) { + return this.redisForTimelines.lrange('list:' + name, 0, -1) + .then(ids => ids.filter(id => id > untilId && id < sinceId).sort((a, b) => a > b ? -1 : 1)); + } else if (untilId) { + return this.redisForTimelines.lrange('list:' + name, 0, -1) + .then(ids => ids.filter(id => id > untilId).sort((a, b) => a > b ? -1 : 1)); + } else if (sinceId) { + return this.redisForTimelines.lrange('list:' + name, 0, -1) + .then(ids => ids.filter(id => id < sinceId).sort((a, b) => a < b ? -1 : 1)); + } else { + return this.redisForTimelines.lrange('list:' + name, 0, -1) + .then(ids => ids.sort((a, b) => a > b ? -1 : 1)); + } + } + + @bindThis + public getMulti(name: string[], untilId?: string | null, sinceId?: string | null): Promise { + const pipeline = this.redisForTimelines.pipeline(); + for (const n of name) { + pipeline.lrange('list:' + n, 0, -1); + } + return pipeline.exec().then(res => { + if (res == null) return []; + const tls = res.map(r => r[1] as string[]); + return tls.map(ids => + (untilId && sinceId) + ? ids.filter(id => id > untilId && id < sinceId).sort((a, b) => a > b ? -1 : 1) + : untilId + ? ids.filter(id => id > untilId).sort((a, b) => a > b ? -1 : 1) + : sinceId + ? ids.filter(id => id < sinceId).sort((a, b) => a < b ? -1 : 1) + : ids.sort((a, b) => a > b ? -1 : 1), + ); + }); + } +} diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index f2bd9de5e..2c3547e4a 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -20,6 +20,7 @@ import { IdService } from '@/core/IdService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; import type { Packed } from '@/misc/json-schema.js'; +import { RedisTimelineService } from '@/core/RedisTimelineService.js'; import type { OnApplicationShutdown } from '@nestjs/common'; export type RolePolicies = { @@ -102,6 +103,7 @@ export class RoleService implements OnApplicationShutdown { private globalEventService: GlobalEventService, private idService: IdService, private moderationLogService: ModerationLogService, + private redisTimelineService: RedisTimelineService, ) { //this.onMessage = this.onMessage.bind(this); @@ -472,12 +474,7 @@ export class RoleService implements OnApplicationShutdown { const redisPipeline = this.redisClient.pipeline(); for (const role of roles) { - redisPipeline.xadd( - `roleTimeline:${role.id}`, - 'MAXLEN', '~', '1000', - '*', - 'note', note.id); - + this.redisTimelineService.push(`roleTimeline:${role.id}`, note.id, 1000, redisPipeline); this.globalEventService.publishRoleTimelineStream(role.id, 'note', note); } diff --git a/packages/backend/src/server/api/endpoints/antennas/notes.ts b/packages/backend/src/server/api/endpoints/antennas/notes.ts index b563f704a..7d6841563 100644 --- a/packages/backend/src/server/api/endpoints/antennas/notes.ts +++ b/packages/backend/src/server/api/endpoints/antennas/notes.ts @@ -12,6 +12,7 @@ import { NoteReadService } from '@/core/NoteReadService.js'; import { DI } from '@/di-symbols.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { IdService } from '@/core/IdService.js'; +import { RedisTimelineService } from '@/core/RedisTimelineService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -69,8 +70,12 @@ export default class extends Endpoint { // eslint- private noteEntityService: NoteEntityService, private queryService: QueryService, private noteReadService: NoteReadService, + private redisTimelineService: RedisTimelineService, ) { super(meta, paramDef, async (ps, me) => { + const untilId = ps.untilId ?? ps.untilDate ? this.idService.genId(new Date(ps.untilDate!)) : null; + const sinceId = ps.sinceId ?? ps.sinceDate ? this.idService.genId(new Date(ps.sinceDate!)) : null; + const antenna = await this.antennasRepository.findOneBy({ id: ps.antennaId, userId: me.id, @@ -85,15 +90,8 @@ export default class extends Endpoint { // eslint- lastUsedAt: new Date(), }); - const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1 - - const noteIds = await this.redisForTimelines.xrevrange( - `antennaTimeline:${antenna.id}`, - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit, - ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)); - + let noteIds = await this.redisTimelineService.get(`antennaTimeline:${antenna.id}`, untilId, sinceId); + noteIds = noteIds.slice(0, ps.limit); if (noteIds.length === 0) { return []; } diff --git a/packages/backend/src/server/api/endpoints/channels/timeline.ts b/packages/backend/src/server/api/endpoints/channels/timeline.ts index f0b14d4fd..e063e0e2f 100644 --- a/packages/backend/src/server/api/endpoints/channels/timeline.ts +++ b/packages/backend/src/server/api/endpoints/channels/timeline.ts @@ -12,6 +12,9 @@ import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import ActiveUsersChart from '@/core/chart/charts/active-users.js'; import { DI } from '@/di-symbols.js'; import { IdService } from '@/core/IdService.js'; +import { RedisTimelineService } from '@/core/RedisTimelineService.js'; +import { isUserRelated } from '@/misc/is-user-related.js'; +import { CacheService } from '@/core/CacheService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -66,9 +69,15 @@ export default class extends Endpoint { // eslint- private idService: IdService, private noteEntityService: NoteEntityService, private queryService: QueryService, + private redisTimelineService: RedisTimelineService, + private cacheService: CacheService, private activeUsersChart: ActiveUsersChart, ) { super(meta, paramDef, async (ps, me) => { + const untilId = ps.untilId ?? ps.untilDate ? this.idService.genId(new Date(ps.untilDate!)) : null; + const sinceId = ps.sinceId ?? ps.sinceDate ? this.idService.genId(new Date(ps.sinceDate!)) : null; + const isRangeSpecified = untilId != null && sinceId != null; + const channel = await this.channelsRepository.findOneBy({ id: ps.channelId, }); @@ -77,68 +86,66 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.noSuchChannel); } - let timeline: MiNote[] = []; - - const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1 - let noteIdsRes: [string, string[]][] = []; - - if (!ps.sinceId && !ps.sinceDate) { - noteIdsRes = await this.redisForTimelines.xrevrange( - `channelTimeline:${channel.id}`, - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit); - } - - // redis から取得していないとき・取得数が足りないとき - if (noteIdsRes.length < limit) { - //#region Construct query - const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) - .andWhere('note.channelId = :channelId', { channelId: channel.id }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); - - if (me) { - this.queryService.generateMutedUserQuery(query, me); - this.queryService.generateBlockedUserQuery(query, me); - } - //#endregion - - timeline = await query.limit(ps.limit).getMany(); - } else { - const noteIds = noteIdsRes.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId); - - if (noteIds.length === 0) { - return []; - } - - //#region Construct query - const query = this.notesRepository.createQueryBuilder('note') - .where('note.id IN (:...noteIds)', { noteIds: noteIds }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); - - if (me) { - this.queryService.generateMutedUserQuery(query, me); - this.queryService.generateBlockedUserQuery(query, me); - } - //#endregion - - timeline = await query.getMany(); - timeline.sort((a, b) => a.id > b.id ? -1 : 1); - } - if (me) this.activeUsersChart.read(me); + if (isRangeSpecified || sinceId == null) { + const [ + userIdsWhoMeMuting, + ] = me ? await Promise.all([ + this.cacheService.userMutingsCache.fetch(me.id), + ]) : [new Set()]; + + let noteIds = await this.redisTimelineService.get(`channelTimeline:${channel.id}`, untilId, sinceId); + noteIds = noteIds.slice(0, ps.limit); + + if (noteIds.length > 0) { + const query = this.notesRepository.createQueryBuilder('note') + .where('note.id IN (:...noteIds)', { noteIds: noteIds }) + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser') + .leftJoinAndSelect('note.channel', 'channel'); + + let timeline = await query.getMany(); + + timeline = timeline.filter(note => { + if (me && isUserRelated(note, userIdsWhoMeMuting, true)) return false; + + return true; + }); + + // TODO: フィルタで件数が減った場合の埋め合わせ処理 + + timeline.sort((a, b) => a.id > b.id ? -1 : 1); + + if (timeline.length > 0) { + return await this.noteEntityService.packMany(timeline, me); + } + } + } + + //#region fallback to database + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) + .andWhere('note.channelId = :channelId', { channelId: channel.id }) + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser') + .leftJoinAndSelect('note.channel', 'channel'); + + if (me) { + this.queryService.generateMutedUserQuery(query, me); + this.queryService.generateBlockedUserQuery(query, me); + } + //#endregion + + const timeline = await query.limit(ps.limit).getMany(); + return await this.noteEntityService.packMany(timeline, me); + //#endregion }); } } diff --git a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts index d5b706e54..a05269794 100644 --- a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts @@ -15,6 +15,7 @@ import { RoleService } from '@/core/RoleService.js'; import { IdService } from '@/core/IdService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; import { CacheService } from '@/core/CacheService.js'; +import { RedisTimelineService } from '@/core/RedisTimelineService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -72,8 +73,12 @@ export default class extends Endpoint { // eslint- private activeUsersChart: ActiveUsersChart, private idService: IdService, private cacheService: CacheService, + private redisTimelineService: RedisTimelineService, ) { super(meta, paramDef, async (ps, me) => { + const untilId = ps.untilId ?? ps.untilDate ? this.idService.genId(new Date(ps.untilDate!)) : null; + const sinceId = ps.sinceId ?? ps.sinceDate ? this.idService.genId(new Date(ps.sinceDate!)) : null; + const policies = await this.roleService.getUserPolicies(me.id); if (!policies.ltlAvailable) { throw new ApiError(meta.errors.stlDisabled); @@ -89,27 +94,10 @@ export default class extends Endpoint { // eslint- this.cacheService.userBlockedCache.fetch(me.id), ]); - let timeline: MiNote[] = []; - - const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1 - - const redisPipeline = this.redisForTimelines.pipeline(); - redisPipeline.xrevrange( + const [htlNoteIds, ltlNoteIds] = await this.redisTimelineService.getMulti([ ps.withFiles ? `homeTimelineWithFiles:${me.id}` : `homeTimeline:${me.id}`, - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit, - ); - redisPipeline.xrevrange( ps.withFiles ? 'localTimelineWithFiles' : 'localTimeline', - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit, - ); - const [htlNoteIds, ltlNoteIds] = await redisPipeline.exec().then(res => res ? [ - (res[0][1] as string[][]).map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId), - (res[1][1] as string[][]).map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId), - ] : []); + ], untilId, sinceId); let noteIds = Array.from(new Set([...htlNoteIds, ...ltlNoteIds])); noteIds.sort((a, b) => a > b ? -1 : 1); @@ -128,7 +116,7 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('renote.user', 'renoteUser') .leftJoinAndSelect('note.channel', 'channel'); - timeline = await query.getMany(); + let timeline = await query.getMany(); timeline = timeline.filter(note => { if (note.userId === me.id) { diff --git a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts index b9305e8e2..c8ea003ba 100644 --- a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts @@ -15,6 +15,7 @@ import { RoleService } from '@/core/RoleService.js'; import { IdService } from '@/core/IdService.js'; import { CacheService } from '@/core/CacheService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; +import { RedisTimelineService } from '@/core/RedisTimelineService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -68,8 +69,12 @@ export default class extends Endpoint { // eslint- private activeUsersChart: ActiveUsersChart, private idService: IdService, private cacheService: CacheService, + private redisTimelineService: RedisTimelineService, ) { super(meta, paramDef, async (ps, me) => { + const untilId = ps.untilId ?? ps.untilDate ? this.idService.genId(new Date(ps.untilDate!)) : null; + const sinceId = ps.sinceId ?? ps.sinceDate ? this.idService.genId(new Date(ps.sinceDate!)) : null; + const policies = await this.roleService.getUserPolicies(me ? me.id : null); if (!policies.ltlAvailable) { throw new ApiError(meta.errors.ltlDisabled); @@ -85,16 +90,8 @@ export default class extends Endpoint { // eslint- this.cacheService.userBlockedCache.fetch(me.id), ]) : [new Set(), new Set(), new Set()]; - let timeline: MiNote[] = []; - - const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1 - - const noteIds = await this.redisForTimelines.xrevrange( - ps.withFiles ? 'localTimelineWithFiles' : 'localTimeline', - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit, - ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)); + let noteIds = await this.redisTimelineService.get(ps.withFiles ? 'localTimelineWithFiles' : 'localTimeline', untilId, sinceId); + noteIds = noteIds.slice(0, ps.limit); if (noteIds.length === 0) { return []; @@ -109,7 +106,7 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('renote.user', 'renoteUser') .leftJoinAndSelect('note.channel', 'channel'); - timeline = await query.getMany(); + let timeline = await query.getMany(); timeline = timeline.filter(note => { if (me && (note.userId === me.id)) { @@ -127,6 +124,8 @@ export default class extends Endpoint { // eslint- return true; }); + // TODO: フィルタした結果件数が足りなかった場合の対応 + timeline.sort((a, b) => a.id > b.id ? -1 : 1); process.nextTick(() => { diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts index cd2ec8fe2..cd45135f9 100644 --- a/packages/backend/src/server/api/endpoints/notes/timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts @@ -15,6 +15,7 @@ import { DI } from '@/di-symbols.js'; import { IdService } from '@/core/IdService.js'; import { CacheService } from '@/core/CacheService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; +import { RedisTimelineService } from '@/core/RedisTimelineService.js'; export const meta = { tags: ['notes'], @@ -62,8 +63,12 @@ export default class extends Endpoint { // eslint- private activeUsersChart: ActiveUsersChart, private idService: IdService, private cacheService: CacheService, + private redisTimelineService: RedisTimelineService, ) { super(meta, paramDef, async (ps, me) => { + const untilId = ps.untilId ?? ps.untilDate ? this.idService.genId(new Date(ps.untilDate!)) : null; + const sinceId = ps.sinceId ?? ps.sinceDate ? this.idService.genId(new Date(ps.sinceDate!)) : null; + const [ followings, userIdsWhoMeMuting, @@ -76,16 +81,8 @@ export default class extends Endpoint { // eslint- this.cacheService.userBlockedCache.fetch(me.id), ]); - let timeline: MiNote[] = []; - - const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1 - - const noteIds = await this.redisForTimelines.xrevrange( - ps.withFiles ? `homeTimelineWithFiles:${me.id}` : `homeTimeline:${me.id}`, - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit, - ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)); + let noteIds = await this.redisTimelineService.get(ps.withFiles ? `homeTimelineWithFiles:${me.id}` : `homeTimeline:${me.id}`, untilId, sinceId); + noteIds = noteIds.slice(0, ps.limit); if (noteIds.length === 0) { return []; @@ -100,7 +97,7 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('renote.user', 'renoteUser') .leftJoinAndSelect('note.channel', 'channel'); - timeline = await query.getMany(); + let timeline = await query.getMany(); timeline = timeline.filter(note => { if (note.userId === me.id) { diff --git a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts index 2eec6297c..96c8ff5b1 100644 --- a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts @@ -15,6 +15,7 @@ import { DI } from '@/di-symbols.js'; import { CacheService } from '@/core/CacheService.js'; import { IdService } from '@/core/IdService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; +import { RedisTimelineService } from '@/core/RedisTimelineService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -79,8 +80,12 @@ export default class extends Endpoint { // eslint- private activeUsersChart: ActiveUsersChart, private cacheService: CacheService, private idService: IdService, + private redisTimelineService: RedisTimelineService, ) { super(meta, paramDef, async (ps, me) => { + const untilId = ps.untilId ?? ps.untilDate ? this.idService.genId(new Date(ps.untilDate!)) : null; + const sinceId = ps.sinceId ?? ps.sinceDate ? this.idService.genId(new Date(ps.sinceDate!)) : null; + const list = await this.userListsRepository.findOneBy({ id: ps.listId, userId: me.id, @@ -100,16 +105,8 @@ export default class extends Endpoint { // eslint- this.cacheService.userBlockedCache.fetch(me.id), ]); - let timeline: MiNote[] = []; - - const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1 - - const noteIds = await this.redisForTimelines.xrevrange( - ps.withFiles ? `userListTimelineWithFiles:${list.id}` : `userListTimeline:${list.id}`, - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit, - ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)); + let noteIds = await this.redisTimelineService.get(ps.withFiles ? `userListTimelineWithFiles:${list.id}` : `userListTimeline:${list.id}`, untilId, sinceId); + noteIds = noteIds.slice(0, ps.limit); if (noteIds.length === 0) { return []; @@ -124,7 +121,7 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('renote.user', 'renoteUser') .leftJoinAndSelect('note.channel', 'channel'); - timeline = await query.getMany(); + let timeline = await query.getMany(); timeline = timeline.filter(note => { if (note.userId === me.id) { diff --git a/packages/backend/src/server/api/endpoints/roles/notes.ts b/packages/backend/src/server/api/endpoints/roles/notes.ts index 3f2b31c02..366c3a7cc 100644 --- a/packages/backend/src/server/api/endpoints/roles/notes.ts +++ b/packages/backend/src/server/api/endpoints/roles/notes.ts @@ -11,6 +11,7 @@ import { QueryService } from '@/core/QueryService.js'; import { DI } from '@/di-symbols.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { IdService } from '@/core/IdService.js'; +import { RedisTimelineService } from '@/core/RedisTimelineService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -65,8 +66,12 @@ export default class extends Endpoint { // eslint- private idService: IdService, private noteEntityService: NoteEntityService, private queryService: QueryService, + private redisTimelineService: RedisTimelineService, ) { super(meta, paramDef, async (ps, me) => { + const untilId = ps.untilId ?? ps.untilDate ? this.idService.genId(new Date(ps.untilDate!)) : null; + const sinceId = ps.sinceId ?? ps.sinceDate ? this.idService.genId(new Date(ps.sinceDate!)) : null; + const role = await this.rolesRepository.findOneBy({ id: ps.roleId, isPublic: true, @@ -78,14 +83,9 @@ export default class extends Endpoint { // eslint- if (!role.isExplorable) { return []; } - const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1 - const noteIds = await this.redisForTimelines.xrevrange( - `roleTimeline:${role.id}`, - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit, - ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)); + let noteIds = await this.redisTimelineService.get(`roleTimeline:${role.id}`, untilId, sinceId); + noteIds = noteIds.slice(0, ps.limit); if (noteIds.length === 0) { return []; diff --git a/packages/backend/src/server/api/endpoints/users/notes.ts b/packages/backend/src/server/api/endpoints/users/notes.ts index 4a3d2e5be..30cd5b791 100644 --- a/packages/backend/src/server/api/endpoints/users/notes.ts +++ b/packages/backend/src/server/api/endpoints/users/notes.ts @@ -14,6 +14,7 @@ import { CacheService } from '@/core/CacheService.js'; import { IdService } from '@/core/IdService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; import { QueryService } from '@/core/QueryService.js'; +import { RedisTimelineService } from '@/core/RedisTimelineService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -70,42 +71,24 @@ export default class extends Endpoint { // eslint- private queryService: QueryService, private cacheService: CacheService, private idService: IdService, + private redisTimelineService: RedisTimelineService, ) { super(meta, paramDef, async (ps, me) => { - const isRangeSpecified = (ps.sinceId != null || ps.sinceDate != null) && (ps.untilId != null || ps.untilDate != null); + const untilId = ps.untilId ?? ps.untilDate ? this.idService.genId(new Date(ps.untilDate!)) : null; + const sinceId = ps.sinceId ?? ps.sinceDate ? this.idService.genId(new Date(ps.sinceDate!)) : null; + const isRangeSpecified = untilId != null && sinceId != null; - if (isRangeSpecified || !(ps.sinceId != null || ps.sinceDate != null)) { + if (isRangeSpecified || sinceId == null) { const [ userIdsWhoMeMuting, ] = me ? await Promise.all([ this.cacheService.userMutingsCache.fetch(me.id), ]) : [new Set()]; - const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1 - const [noteIdsRes, repliesNoteIdsRes, channelNoteIdsRes] = await Promise.all([ - this.redisForTimelines.xrevrange( - ps.withFiles ? `userTimelineWithFiles:${ps.userId}` : `userTimeline:${ps.userId}`, - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit, - ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)), - ps.withReplies - ? this.redisForTimelines.xrevrange( - `userTimelineWithReplies:${ps.userId}`, - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit, - ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)) - : Promise.resolve([]), - ps.withChannelNotes - ? this.redisForTimelines.xrevrange( - `userTimelineWithChannel:${ps.userId}`, - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit, - ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)) - : Promise.resolve([]), + this.redisTimelineService.get(ps.withFiles ? `userTimelineWithFiles:${ps.userId}` : `userTimeline:${ps.userId}`, untilId, sinceId), + ps.withReplies ? this.redisTimelineService.get(`userTimelineWithReplies:${ps.userId}`, untilId, sinceId) : Promise.resolve([]), + ps.withChannelNotes ? this.redisTimelineService.get(`userTimelineWithChannel:${ps.userId}`, untilId, sinceId) : Promise.resolve([]), ]); let noteIds = Array.from(new Set([ @@ -145,6 +128,8 @@ export default class extends Endpoint { // eslint- return true; }); + // TODO: フィルタで件数が減った場合の埋め合わせ処理 + timeline.sort((a, b) => a.id > b.id ? -1 : 1); if (timeline.length > 0) { @@ -153,9 +138,7 @@ export default class extends Endpoint { // eslint- } } - // fallback to database - - //#region Construct query + //#region fallback to database const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) .andWhere('note.userId = :userId', { userId: ps.userId }) .innerJoinAndSelect('note.user', 'user') @@ -188,11 +171,11 @@ export default class extends Endpoint { // eslint- qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); })); } - //#endregion const timeline = await query.limit(ps.limit).getMany(); return await this.noteEntityService.packMany(timeline, me); + //#endregion }); } }