enhance(backend): improve hashtags/trend performance

This commit is contained in:
syuilo 2023-10-07 12:05:17 +09:00
parent 3a7558f36c
commit 71edc65d0d
4 changed files with 121 additions and 102 deletions

View file

@ -41,6 +41,7 @@
### Server ### Server
- Enhance: タイムライン取得時のパフォーマンスを大幅に向上 - Enhance: タイムライン取得時のパフォーマンスを大幅に向上
- Enhance: ハイライト取得時のパフォーマンスを大幅に向上 - Enhance: ハイライト取得時のパフォーマンスを大幅に向上
- Enhance: トレンドハッシュタグ取得時のパフォーマンスを大幅に向上
- Enhance: 不要なPostgreSQLのインデックスを削除しパフォーマンスを向上 - Enhance: 不要なPostgreSQLのインデックスを削除しパフォーマンスを向上
## 2023.9.3 ## 2023.9.3

View file

@ -11,6 +11,7 @@ import { bindThis } from '@/decorators.js';
const GLOBAL_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 3; // 3日ごと const GLOBAL_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 3; // 3日ごと
const PER_USER_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 7; // 1週間ごと const PER_USER_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 7; // 1週間ごと
const HASHTAG_RANKING_WINDOW = 1000 * 60 * 60; // 1時間ごと
@Injectable() @Injectable()
export class FeaturedService { export class FeaturedService {
@ -88,6 +89,11 @@ export class FeaturedService {
return this.updateRankingOf(`featuredPerUserNotesRanking:${userId}`, PER_USER_NOTES_RANKING_WINDOW, noteId, score); return this.updateRankingOf(`featuredPerUserNotesRanking:${userId}`, PER_USER_NOTES_RANKING_WINDOW, noteId, score);
} }
@bindThis
public updateHashtagsRanking(hashtag: string, score = 1): Promise<void> {
return this.updateRankingOf('featuredHashtagsRanking', HASHTAG_RANKING_WINDOW, hashtag, score);
}
@bindThis @bindThis
public getGlobalNotesRanking(limit: number): Promise<MiNote['id'][]> { public getGlobalNotesRanking(limit: number): Promise<MiNote['id'][]> {
return this.getRankingOf('featuredGlobalNotesRanking', GLOBAL_NOTES_RANKING_WINDOW, limit); return this.getRankingOf('featuredGlobalNotesRanking', GLOBAL_NOTES_RANKING_WINDOW, limit);
@ -102,4 +108,9 @@ export class FeaturedService {
public getPerUserNotesRanking(userId: MiUser['id'], limit: number): Promise<MiNote['id'][]> { public getPerUserNotesRanking(userId: MiUser['id'], limit: number): Promise<MiNote['id'][]> {
return this.getRankingOf(`featuredPerUserNotesRanking:${userId}`, PER_USER_NOTES_RANKING_WINDOW, limit); return this.getRankingOf(`featuredPerUserNotesRanking:${userId}`, PER_USER_NOTES_RANKING_WINDOW, limit);
} }
@bindThis
public getHashtagsRanking(limit: number): Promise<string[]> {
return this.getRankingOf('featuredHashtagsRanking', HASHTAG_RANKING_WINDOW, limit);
}
} }

View file

@ -4,6 +4,8 @@
*/ */
import { Inject, Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import * as Redis from 'ioredis';
import { getLineAndCharacterOfPosition } from 'typescript';
import { DI } from '@/di-symbols.js'; import { DI } from '@/di-symbols.js';
import type { MiUser } from '@/models/User.js'; import type { MiUser } from '@/models/User.js';
import { normalizeForSearch } from '@/misc/normalize-for-search.js'; import { normalizeForSearch } from '@/misc/normalize-for-search.js';
@ -12,14 +14,19 @@ import type { MiHashtag } from '@/models/Hashtag.js';
import type { HashtagsRepository } from '@/models/_.js'; import type { HashtagsRepository } from '@/models/_.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { FeaturedService } from '@/core/FeaturedService.js';
@Injectable() @Injectable()
export class HashtagService { export class HashtagService {
constructor( constructor(
@Inject(DI.redis)
private redisClient: Redis.Redis, // TODO: 専用のRedisサーバーを設定できるようにする
@Inject(DI.hashtagsRepository) @Inject(DI.hashtagsRepository)
private hashtagsRepository: HashtagsRepository, private hashtagsRepository: HashtagsRepository,
private userEntityService: UserEntityService, private userEntityService: UserEntityService,
private featuredService: FeaturedService,
private idService: IdService, private idService: IdService,
) { ) {
} }
@ -46,6 +53,9 @@ export class HashtagService {
public async updateHashtag(user: { id: MiUser['id']; host: MiUser['host']; }, tag: string, isUserAttached = false, inc = true) { public async updateHashtag(user: { id: MiUser['id']; host: MiUser['host']; }, tag: string, isUserAttached = false, inc = true) {
tag = normalizeForSearch(tag); tag = normalizeForSearch(tag);
// TODO: サンプリング
this.updateHashtagsRanking(tag, user.id);
const index = await this.hashtagsRepository.findOneBy({ name: tag }); const index = await this.hashtagsRepository.findOneBy({ name: tag });
if (index == null && !inc) return; if (index == null && !inc) return;
@ -85,7 +95,7 @@ export class HashtagService {
} }
} }
} else { } else {
// 自分が初めてこのタグを使ったなら // 自分が初めてこのタグを使ったなら
if (!index.mentionedUserIds.some(id => id === user.id)) { if (!index.mentionedUserIds.some(id => id === user.id)) {
set.mentionedUserIds = () => `array_append("mentionedUserIds", '${user.id}')`; set.mentionedUserIds = () => `array_append("mentionedUserIds", '${user.id}')`;
set.mentionedUsersCount = () => '"mentionedUsersCount" + 1'; set.mentionedUsersCount = () => '"mentionedUsersCount" + 1';
@ -144,4 +154,93 @@ export class HashtagService {
} }
} }
} }
@bindThis
public async updateHashtagsRanking(hashtag: string, userId: MiUser['id']): Promise<void> {
// TODO: instance.hiddenTagsの考慮
// YYYYMMDDHHmm (10分間隔)
const now = new Date();
now.setMinutes(Math.floor(now.getMinutes() / 10) * 10, 0, 0);
const window = `${now.getUTCFullYear()}${(now.getUTCMonth() + 1).toString().padStart(2, '0')}${now.getUTCDate().toString().padStart(2, '0')}${now.getUTCHours().toString().padStart(2, '0')}${now.getUTCMinutes().toString().padStart(2, '0')}`;
const exist = await this.redisClient.sismember(`hashtagUsers:${hashtag}`, userId);
if (exist === 1) return;
this.featuredService.updateHashtagsRanking(hashtag, 1);
const redisPipeline = this.redisClient.pipeline();
// TODO: これらの Set は Bloom Filter を使うようにしても良さそう
// チャート用
redisPipeline.sadd(`hashtagUsers:${hashtag}:${window}`, userId);
redisPipeline.expire(`hashtagUsers:${hashtag}:${window}`,
60 * 60 * 24 * 3, // 3日間
'NX', // "NX -- Set expiry only when the key has no expiry" = 有効期限がないときだけ設定
);
// ユニークカウント用
redisPipeline.sadd(`hashtagUsers:${hashtag}`, userId);
redisPipeline.expire(`hashtagUsers:${hashtag}`,
60 * 60, // 1時間
'NX', // "NX -- Set expiry only when the key has no expiry" = 有効期限がないときだけ設定
);
redisPipeline.exec();
}
@bindThis
public async getChart(hashtag: string, range: number): Promise<number[]> {
const now = new Date();
now.setMinutes(Math.floor(now.getMinutes() / 10) * 10, 0, 0);
const redisPipeline = this.redisClient.pipeline();
for (let i = 0; i < range; i++) {
const window = `${now.getUTCFullYear()}${(now.getUTCMonth() + 1).toString().padStart(2, '0')}${now.getUTCDate().toString().padStart(2, '0')}${now.getUTCHours().toString().padStart(2, '0')}${now.getUTCMinutes().toString().padStart(2, '0')}`;
redisPipeline.scard(`hashtagUsers:${hashtag}:${window}`);
now.setMinutes(now.getMinutes() - (i * 10), 0, 0);
}
const result = await redisPipeline.exec();
if (result == null) return [];
return result.map(x => x[1]) as number[];
}
@bindThis
public async getCharts(hashtags: string[], range: number): Promise<Record<string, number[]>> {
const now = new Date();
now.setMinutes(Math.floor(now.getMinutes() / 10) * 10, 0, 0);
const redisPipeline = this.redisClient.pipeline();
for (let i = 0; i < range; i++) {
const window = `${now.getUTCFullYear()}${(now.getUTCMonth() + 1).toString().padStart(2, '0')}${now.getUTCDate().toString().padStart(2, '0')}${now.getUTCHours().toString().padStart(2, '0')}${now.getUTCMinutes().toString().padStart(2, '0')}`;
for (const hashtag of hashtags) {
redisPipeline.scard(`hashtagUsers:${hashtag}:${window}`);
}
now.setMinutes(now.getMinutes() - (i * 10), 0, 0);
}
const result = await redisPipeline.exec();
if (result == null) return {};
// key is hashtag
const charts = {} as Record<string, number[]>;
for (const hashtag of hashtags) {
charts[hashtag] = [];
}
for (let i = 0; i < range; i++) {
for (let j = 0; j < hashtags.length; j++) {
charts[hashtags[j]].push(result[(i * hashtags.length) + j][1] as number);
}
}
return charts;
}
} }

View file

@ -3,29 +3,13 @@
* SPDX-License-Identifier: AGPL-3.0-only * SPDX-License-Identifier: AGPL-3.0-only
*/ */
import { Brackets } from 'typeorm';
import { Inject, Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import { Endpoint } from '@/server/api/endpoint-base.js'; import { Endpoint } from '@/server/api/endpoint-base.js';
import type { NotesRepository } from '@/models/_.js';
import type { MiNote } from '@/models/Note.js';
import { safeForSql } from '@/misc/safe-for-sql.js';
import { normalizeForSearch } from '@/misc/normalize-for-search.js'; import { normalizeForSearch } from '@/misc/normalize-for-search.js';
import { MetaService } from '@/core/MetaService.js'; import { MetaService } from '@/core/MetaService.js';
import { DI } from '@/di-symbols.js'; import { DI } from '@/di-symbols.js';
import { FeaturedService } from '@/core/FeaturedService.js';
/* import { HashtagService } from '@/core/HashtagService.js';
a分間のユニーク投稿数が今からa分前b分前の間のユニーク投稿数のn倍以上5
稿稿稿稿1
..PostgreSQLでどうするのか分からないので単にAの内に投稿されたユニーク投稿数が多いハッシュタグ
*/
const rangeA = 1000 * 60 * 60; // 60分
//const rangeB = 1000 * 60 * 120; // 2時間
//const coefficient = 1.25; // 「n倍」の部分
//const requiredUsers = 3; // 最低何人がそのタグを投稿している必要があるか
const max = 5;
export const meta = { export const meta = {
tags: ['hashtags'], tags: ['hashtags'],
@ -71,98 +55,22 @@ export const paramDef = {
@Injectable() @Injectable()
export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export
constructor( constructor(
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
private metaService: MetaService, private metaService: MetaService,
private featuredService: FeaturedService,
private hashtagService: HashtagService,
) { ) {
super(meta, paramDef, async () => { super(meta, paramDef, async () => {
const instance = await this.metaService.fetch(true); const instance = await this.metaService.fetch(true);
const hiddenTags = instance.hiddenTags.map(t => normalizeForSearch(t)); const hiddenTags = instance.hiddenTags.map(t => normalizeForSearch(t));
const now = new Date(); // 5分単位で丸めた現在日時 const ranking = await this.featuredService.getHashtagsRanking(10);
now.setMinutes(Math.round(now.getMinutes() / 5) * 5, 0, 0);
const tagNotes = await this.notesRepository.createQueryBuilder('note') const charts = ranking.length === 0 ? {} : await this.hashtagService.getCharts(ranking, 20);
.where('note.createdAt > :date', { date: new Date(now.getTime() - rangeA) })
.andWhere(new Brackets(qb => { qb
.where('note.visibility = \'public\'')
.orWhere('note.visibility = \'home\'');
}))
.andWhere('note.tags != \'{}\'')
.select(['note.tags', 'note.userId'])
.cache(60000) // 1 min
.getMany();
if (tagNotes.length === 0) { const stats = ranking.map((tag, i) => ({
return [];
}
const tags: {
name: string;
users: MiNote['userId'][];
}[] = [];
for (const note of tagNotes) {
for (const tag of note.tags) {
if (hiddenTags.includes(tag)) continue;
const x = tags.find(x => x.name === tag);
if (x) {
if (!x.users.includes(note.userId)) {
x.users.push(note.userId);
}
} else {
tags.push({
name: tag,
users: [note.userId],
});
}
}
}
// タグを人気順に並べ替え
const hots = tags
.sort((a, b) => b.users.length - a.users.length)
.map(tag => tag.name)
.slice(0, max);
//#region 2(または3)で話題と判定されたタグそれぞれについて過去の投稿数グラフを取得する
const countPromises: Promise<number[]>[] = [];
const range = 20;
// 10分
const interval = 1000 * 60 * 10;
for (let i = 0; i < range; i++) {
countPromises.push(Promise.all(hots.map(tag => this.notesRepository.createQueryBuilder('note')
.select('count(distinct note.userId)')
.where(`'{"${safeForSql(tag) ? tag : 'aichan_kawaii'}"}' <@ note.tags`)
.andWhere('note.createdAt < :lt', { lt: new Date(now.getTime() - (interval * i)) })
.andWhere('note.createdAt > :gt', { gt: new Date(now.getTime() - (interval * (i + 1))) })
.cache(60000) // 1 min
.getRawOne()
.then(x => parseInt(x.count, 10)),
)));
}
const countsLog = await Promise.all(countPromises);
//#endregion
const totalCounts = await Promise.all(hots.map(tag => this.notesRepository.createQueryBuilder('note')
.select('count(distinct note.userId)')
.where(`'{"${safeForSql(tag) ? tag : 'aichan_kawaii'}"}' <@ note.tags`)
.andWhere('note.createdAt > :gt', { gt: new Date(now.getTime() - rangeA) })
.cache(60000 * 60) // 60 min
.getRawOne()
.then(x => parseInt(x.count, 10)),
));
const stats = hots.map((tag, i) => ({
tag, tag,
chart: countsLog.map(counts => counts[i]), chart: charts[tag],
usersCount: totalCounts[i], usersCount: Math.max(...charts[tag]),
})); }));
return stats; return stats;