enhance(backend): improve featured system

CHANGELOG.md

@@ -34,7 +34,8 @@
 - Fix: リアクションしたユーザ一覧のUIが稀に左上に残ってしまう不具合を修正
 
 ### Server
-- Enhance: タイムライン取得時のパフォーマンスを改善
+- Enhance: タイムライン取得時のパフォーマンスを大幅に向上
+- Enhance: ハイライト取得時のパフォーマンスを大幅に向上
 
 ## 2023.9.3
 ### General

packages/backend/migration/1696569742153-clean-up.js

@@ -0,0 +1,18 @@
+/*
+ * SPDX-FileCopyrightText: syuilo and other misskey contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+export class CleanUp1696569742153 {
+    name = 'CleanUp1696569742153'
+
+    async up(queryRunner) {
+        await queryRunner.query(`DROP INDEX "public"."IDX_01f4581f114e0ebd2bbb876f0b"`);
+        await queryRunner.query(`ALTER TABLE "note" DROP COLUMN "score"`);
+    }
+
+    async down(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "note" ADD "score" integer NOT NULL DEFAULT '0'`);
+        await queryRunner.query(`CREATE INDEX "IDX_01f4581f114e0ebd2bbb876f0b" ON "note_reaction" ("createdAt") `);
+    }
+}

packages/backend/src/core/CoreModule.ts

@@ -60,6 +60,7 @@ import { UtilityService } from './UtilityService.js';
 import { FileInfoService } from './FileInfoService.js';
 import { SearchService } from './SearchService.js';
 import { ClipService } from './ClipService.js';
+import { FeaturedService } from './FeaturedService.js';
 import { ChartLoggerService } from './chart/ChartLoggerService.js';
 import FederationChart from './chart/charts/federation.js';
 import NotesChart from './chart/charts/notes.js';
@@ -187,6 +188,7 @@ const $UtilityService: Provider = { provide: 'UtilityService', useExisting: Util
 const $FileInfoService: Provider = { provide: 'FileInfoService', useExisting: FileInfoService };
 const $SearchService: Provider = { provide: 'SearchService', useExisting: SearchService };
 const $ClipService: Provider = { provide: 'ClipService', useExisting: ClipService };
+const $FeaturedService: Provider = { provide: 'FeaturedService', useExisting: FeaturedService };
 
 const $ChartLoggerService: Provider = { provide: 'ChartLoggerService', useExisting: ChartLoggerService };
 const $FederationChart: Provider = { provide: 'FederationChart', useExisting: FederationChart };
@@ -318,6 +320,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
 		FileInfoService,
 		SearchService,
 		ClipService,
+		FeaturedService,
 		ChartLoggerService,
 		FederationChart,
 		NotesChart,
@@ -442,6 +445,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
 		$FileInfoService,
 		$SearchService,
 		$ClipService,
+		$FeaturedService,
 		$ChartLoggerService,
 		$FederationChart,
 		$NotesChart,
@@ -567,6 +571,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
 		FileInfoService,
 		SearchService,
 		ClipService,
+		FeaturedService,
 		FederationChart,
 		NotesChart,
 		UsersChart,
@@ -690,6 +695,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
 		$FileInfoService,
 		$SearchService,
 		$ClipService,
+		$FeaturedService,
 		$FederationChart,
 		$NotesChart,
 		$UsersChart,

packages/backend/src/core/FeaturedService.ts

@@ -0,0 +1,126 @@
+/*
+ * SPDX-FileCopyrightText: syuilo and other misskey contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+import { Inject, Injectable } from '@nestjs/common';
+import * as Redis from 'ioredis';
+import type { MiNote } from '@/models/_.js';
+import { DI } from '@/di-symbols.js';
+import { bindThis } from '@/decorators.js';
+
+@Injectable()
+export class FeaturedService {
+	constructor(
+		@Inject(DI.redis)
+		private redisClient: Redis.Redis,
+	) {
+	}
+
+	@bindThis
+	private getCurrentPerUserFriendRankingWindow(): number {
+		const passed = new Date().getTime() - new Date(new Date().getFullYear(), 0, 1).getTime();
+		return Math.floor(passed / (1000 * 60 * 60 * 24 * 7)); // 1週間ごと
+	}
+
+	@bindThis
+	private getCurrentGlobalNotesRankingWindow(): number {
+		const passed = new Date().getTime() - new Date(new Date().getFullYear(), 0, 1).getTime();
+		return Math.floor(passed / (1000 * 60 * 60 * 24 * 3)); // 3日ごと
+	}
+
+	@bindThis
+	public async updateGlobalNotesRanking(noteId: MiNote['id'], score = 1): Promise<void> {
+		// TODO: フォロワー数の多い人が常にランキング上位になるのを防ぎたい
+		const currentWindow = this.getCurrentGlobalNotesRankingWindow();
+		const redisTransaction = this.redisClient.multi();
+		redisTransaction.zincrby(
+			`featuredGlobalNotesRanking:${currentWindow}`,
+			score.toString(),
+			noteId);
+		redisTransaction.expire(
+			`featuredGlobalNotesRanking:${currentWindow}`,
+			60 * 60 * 24 * 9, // 9日間保持
+			'NX'); // "NX -- Set expiry only when the key has no expiry" = 有効期限がないときだけ設定
+		await redisTransaction.exec();
+	}
+
+	@bindThis
+	public async updateInChannelNotesRanking(noteId: MiNote['id'], channelId: MiNote['channelId'], score = 1): Promise<void> {
+		const currentWindow = this.getCurrentGlobalNotesRankingWindow();
+		const redisTransaction = this.redisClient.multi();
+		redisTransaction.zincrby(
+			`featuredInChannelNotesRanking:${channelId}:${currentWindow}`,
+			score.toString(),
+			noteId);
+		redisTransaction.expire(
+			`featuredInChannelNotesRanking:${channelId}:${currentWindow}`,
+			60 * 60 * 24 * 9, // 9日間保持
+			'NX'); // "NX -- Set expiry only when the key has no expiry" = 有効期限がないときだけ設定
+		await redisTransaction.exec();
+	}
+
+	@bindThis
+	public async getGlobalNotesRanking(limit: number): Promise<MiNote['id'][]> {
+		const currentWindow = this.getCurrentGlobalNotesRankingWindow();
+		const previousWindow = currentWindow - 1;
+
+		const [currentRankingResult, previousRankingResult] = await Promise.all([
+			this.redisClient.zrange(
+				`featuredGlobalNotesRanking:${currentWindow}`, 0, limit, 'REV', 'WITHSCORES'),
+			this.redisClient.zrange(
+				`featuredGlobalNotesRanking:${previousWindow}`, 0, limit, 'REV', 'WITHSCORES'),
+		]);
+
+		const ranking = new Map<MiNote['id'], number>();
+		for (let i = 0; i < currentRankingResult.length; i += 2) {
+			const noteId = currentRankingResult[i];
+			const score = parseInt(currentRankingResult[i + 1], 10);
+			ranking.set(noteId, score);
+		}
+		for (let i = 0; i < previousRankingResult.length; i += 2) {
+			const noteId = previousRankingResult[i];
+			const score = parseInt(previousRankingResult[i + 1], 10);
+			const exist = ranking.get(noteId);
+			if (exist != null) {
+				ranking.set(noteId, (exist + score) / 2);
+			} else {
+				ranking.set(noteId, score);
+			}
+		}
+
+		return Array.from(ranking.keys());
+	}
+
+	@bindThis
+	public async getInChannelNotesRanking(channelId: MiNote['channelId'], limit: number): Promise<MiNote['id'][]> {
+		const currentWindow = this.getCurrentGlobalNotesRankingWindow();
+		const previousWindow = currentWindow - 1;
+
+		const [currentRankingResult, previousRankingResult] = await Promise.all([
+			this.redisClient.zrange(
+				`featuredInChannelNotesRanking:${channelId}:${currentWindow}`, 0, limit, 'REV', 'WITHSCORES'),
+			this.redisClient.zrange(
+				`featuredInChannelNotesRanking:${channelId}:${previousWindow}`, 0, limit, 'REV', 'WITHSCORES'),
+		]);
+
+		const ranking = new Map<MiNote['id'], number>();
+		for (let i = 0; i < currentRankingResult.length; i += 2) {
+			const noteId = currentRankingResult[i];
+			const score = parseInt(currentRankingResult[i + 1], 10);
+			ranking.set(noteId, score);
+		}
+		for (let i = 0; i < previousRankingResult.length; i += 2) {
+			const noteId = previousRankingResult[i];
+			const score = parseInt(previousRankingResult[i + 1], 10);
+			const exist = ranking.get(noteId);
+			if (exist != null) {
+				ranking.set(noteId, (exist + score) / 2);
+			} else {
+				ranking.set(noteId, score);
+			}
+		}
+
+		return Array.from(ranking.keys());
+	}
+}

packages/backend/src/core/NoteCreateService.ts

@@ -53,6 +53,7 @@ import { DB_MAX_NOTE_TEXT_LENGTH } from '@/const.js';
 import { RoleService } from '@/core/RoleService.js';
 import { MetaService } from '@/core/MetaService.js';
 import { SearchService } from '@/core/SearchService.js';
+import { FeaturedService } from '@/core/FeaturedService.js';
 
 type NotificationType = 'reply' | 'renote' | 'quote' | 'mention';
 
@@ -200,6 +201,7 @@ export class NoteCreateService implements OnApplicationShutdown {
 		private hashtagService: HashtagService,
 		private antennaService: AntennaService,
 		private webhookService: WebhookService,
+		private featuredService: FeaturedService,
 		private remoteUserResolveService: RemoteUserResolveService,
 		private apDeliverManagerService: ApDeliverManagerService,
 		private apRendererService: ApRendererService,
@@ -721,10 +723,18 @@ export class NoteCreateService implements OnApplicationShutdown {
 		this.notesRepository.createQueryBuilder().update()
 			.set({
 				renoteCount: () => '"renoteCount" + 1',
-				score: () => '"score" + 1',
 			})
 			.where('id = :id', { id: renote.id })
 			.execute();
+
+		// 30%の確率でハイライト用ランキング更新
+		if (Math.random() < 0.3) {
+			if (renote.channelId != null) {
+				this.featuredService.updateInChannelNotesRanking(renote.id, renote.channelId, 1);
+			} else if (renote.visibility === 'public' && renote.userHost == null) {
+				this.featuredService.updateGlobalNotesRanking(renote.id, 1);
+			}
+		}
 	}
 
 	@bindThis

packages/backend/src/core/NoteDeleteService.ts

@@ -67,7 +67,6 @@ export class NoteDeleteService {
 		// この投稿を除く指定したユーザーによる指定したノートのリノートが存在しないとき
 		if (note.renoteId && (await this.noteEntityService.countSameRenotes(user.id, note.renoteId, note.id)) === 0) {
 			this.notesRepository.decrement({ id: note.renoteId }, 'renoteCount', 1);
-			if (!user.isBot) this.notesRepository.decrement({ id: note.renoteId }, 'score', 1);
 		}
 
 		if (note.replyId) {

packages/backend/src/core/ReactionService.ts

@@ -4,6 +4,7 @@
  */
 
 import { Inject, Injectable } from '@nestjs/common';
+import * as Redis from 'ioredis';
 import { DI } from '@/di-symbols.js';
 import type { EmojisRepository, NoteReactionsRepository, UsersRepository, NotesRepository } from '@/models/_.js';
 import { IdentifiableError } from '@/misc/identifiable-error.js';
@@ -26,6 +27,7 @@ import { UtilityService } from '@/core/UtilityService.js';
 import { UserBlockingService } from '@/core/UserBlockingService.js';
 import { CustomEmojiService } from '@/core/CustomEmojiService.js';
 import { RoleService } from '@/core/RoleService.js';
+import { FeaturedService } from '@/core/FeaturedService.js';
 
 const FALLBACK = '❤';
 
@@ -66,6 +68,9 @@ const decodeCustomEmojiRegexp = /^:([\w+-]+)(?:@([\w.-]+))?:$/;
 @Injectable()
 export class ReactionService {
 	constructor(
+		@Inject(DI.redis)
+		private redisClient: Redis.Redis,
+
 		@Inject(DI.usersRepository)
 		private usersRepository: UsersRepository,
 
@@ -86,6 +91,7 @@ export class ReactionService {
 		private noteEntityService: NoteEntityService,
 		private userBlockingService: UserBlockingService,
 		private idService: IdService,
+		private featuredService: FeaturedService,
 		private globalEventService: GlobalEventService,
 		private apRendererService: ApRendererService,
 		private apDeliverManagerService: ApDeliverManagerService,
@@ -182,11 +188,19 @@ export class ReactionService {
 		await this.notesRepository.createQueryBuilder().update()
 			.set({
 				reactions: () => sql,
-				... (!user.isBot ? { score: () => '"score" + 1' } : {}),
 			})
 			.where('id = :id', { id: note.id })
 			.execute();
 
+		// 30%の確率でハイライト用ランキング更新
+		if (Math.random() < 0.3) {
+			if (note.channelId != null) {
+				this.featuredService.updateInChannelNotesRanking(note.id, note.channelId, 1);
+			} else if (note.visibility === 'public' && note.userHost == null) {
+				this.featuredService.updateGlobalNotesRanking(note.id, 1);
+			}
+		}
+
 		const meta = await this.metaService.fetch();
 
 		if (meta.enableChartsForRemoteUser || (user.host == null)) {
@@ -275,8 +289,6 @@ export class ReactionService {
 			.where('id = :id', { id: note.id })
 			.execute();
 
-		if (!user.isBot) this.notesRepository.decrement({ id: note.id }, 'score', 1);
-
 		this.globalEventService.publishNoteStream(note.id, 'unreacted', {
 			reaction: this.decodeReaction(exist.reaction).reaction,
 			userId: user.id,

packages/backend/src/models/Note.ts

@@ -138,11 +138,6 @@ export class MiNote {
 	})
 	public url: string | null;
 
-	@Column('integer', {
-		default: 0, select: false,
-	})
-	public score: number;
-
 	@Index()
 	@Column({
 		...id(),

packages/backend/src/models/NoteReaction.ts

@@ -14,7 +14,6 @@ export class MiNoteReaction {
 	@PrimaryColumn(id())
 	public id: string;
 
-	@Index()
 	@Column('timestamp with time zone', {
 		comment: 'The created date of the NoteReaction.',
 	})

packages/backend/src/server/api/endpoints/notes/featured.ts

@@ -6,9 +6,9 @@
 import { Inject, Injectable } from '@nestjs/common';
 import type { NotesRepository } from '@/models/_.js';
 import { Endpoint } from '@/server/api/endpoint-base.js';
-import { QueryService } from '@/core/QueryService.js';
 import { NoteEntityService } from '@/core/entities/NoteEntityService.js';
 import { DI } from '@/di-symbols.js';
+import { FeaturedService } from '@/core/FeaturedService.js';
 
 export const meta = {
 	tags: ['notes'],
@@ -40,41 +40,50 @@ export const paramDef = {
 
 @Injectable()
 export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export
+	private globalNotesRankingCache: string[] = [];
+	private globalNotesRankingCacheLastFetchedAt = 0;
+
 	constructor(
 		@Inject(DI.notesRepository)
 		private notesRepository: NotesRepository,
 
 		private noteEntityService: NoteEntityService,
-		private queryService: QueryService,
+		private featuredService: FeaturedService,
 	) {
 		super(meta, paramDef, async (ps, me) => {
-			const day = 1000 * 60 * 60 * 24 * 3; // 3日前まで
+			let noteIds: string[];
+			if (ps.channelId) {
+				noteIds = await this.featuredService.getInChannelNotesRanking(ps.channelId, 50);
+			} else {
+				if (this.globalNotesRankingCacheLastFetchedAt !== 0 && (Date.now() - this.globalNotesRankingCacheLastFetchedAt < 1000 * 60 * 30)) {
+					noteIds = this.globalNotesRankingCache;
+				} else {
+					noteIds = await this.featuredService.getGlobalNotesRanking(100);
+					this.globalNotesRankingCache = noteIds;
+					this.globalNotesRankingCacheLastFetchedAt = Date.now();
+				}
+			}
+
+			if (noteIds.length === 0) {
+				return [];
+			}
+
+			noteIds.sort((a, b) => a > b ? -1 : 1);
+			noteIds.slice(ps.offset, ps.offset + ps.limit);
 
 			const query = this.notesRepository.createQueryBuilder('note')
-				.addSelect('note.score')
-				.where('note.userHost IS NULL')
-				.andWhere('note.score > 0')
-				.andWhere('note.createdAt > :date', { date: new Date(Date.now() - day) })
-				.andWhere('note.visibility = \'public\'')
+				.where('note.id IN (:...noteIds)', { noteIds: noteIds })
 				.innerJoinAndSelect('note.user', 'user')
 				.leftJoinAndSelect('note.reply', 'reply')
 				.leftJoinAndSelect('note.renote', 'renote')
 				.leftJoinAndSelect('reply.user', 'replyUser')
-				.leftJoinAndSelect('renote.user', 'renoteUser');
+				.leftJoinAndSelect('renote.user', 'renoteUser')
+				.leftJoinAndSelect('note.channel', 'channel');
 
-			if (ps.channelId) query.andWhere('note.channelId = :channelId', { channelId: ps.channelId });
+			const notes = await query.getMany();
+			notes.sort((a, b) => a.id > b.id ? -1 : 1);
 
-			if (me) this.queryService.generateMutedUserQuery(query, me);
-			if (me) this.queryService.generateBlockedUserQuery(query, me);
-
-			let notes = await query
-				.orderBy('note.score', 'DESC')
-				.limit(100)
-				.getMany();
-
-			notes.sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime());
-
-			notes = notes.slice(ps.offset, ps.offset + ps.limit);
+			// TODO: ミュート等考慮
 
 			return await this.noteEntityService.packMany(notes, me);
 		});