mirror of
https://git.joinsharkey.org/Sharkey/Sharkey.git
synced 2024-11-09 19:43:09 +02:00
add: Importing of Posts
- Supports Instagram, Mastodon/Pleroma/Akkoma, Twitter and *key
This commit is contained in:
Mar0xy
Marie
parent
4f0e0f067e
commit
83f328de8a
18 changed files with 971 additions and 6 deletions
|
|
@ -1594,6 +1594,7 @@ _role:
|
|||
gtlAvailable: "Can view the global timeline"
|
||||
ltlAvailable: "Can view the local timeline"
|
||||
canPublicNote: "Can send public notes"
|
||||
canImportNotes: "Can import notes"
|
||||
canInvite: "Can create instance invite codes"
|
||||
inviteLimit: "Invite limit"
|
||||
inviteLimitCycle: "Invite limit cooldown"
|
||||
|
|
|
|||
1
locales/index.d.ts
vendored
1
locales/index.d.ts
vendored
|
|
@ -1696,6 +1696,7 @@ export interface Locale {
|
|||
"gtlAvailable": string;
|
||||
"ltlAvailable": string;
|
||||
"canPublicNote": string;
|
||||
"canImportNotes": string;
|
||||
"canInvite": string;
|
||||
"inviteLimit": string;
|
||||
"inviteLimitCycle": string;
|
||||
|
|
|
|||
|
|
@ -1605,6 +1605,7 @@ _role:
|
|||
gtlAvailable: "グローバルタイムラインの閲覧"
|
||||
ltlAvailable: "ローカルタイムラインの閲覧"
|
||||
canPublicNote: "パブリック投稿の許可"
|
||||
canImportNotes: "ノートのインポートが可能"
|
||||
canInvite: "サーバー招待コードの発行"
|
||||
inviteLimit: "招待コードの作成可能数"
|
||||
inviteLimitCycle: "招待コードの発行間隔"
|
||||
|
|
|
|||
|
|
@ -378,6 +378,167 @@ export class NoteCreateService implements OnApplicationShutdown {
|
|||
return note;
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public async import(user: {
|
||||
id: MiUser['id'];
|
||||
username: MiUser['username'];
|
||||
host: MiUser['host'];
|
||||
isBot: MiUser['isBot'];
|
||||
isIndexable: MiUser['isIndexable'];
|
||||
}, data: Option, silent = false): Promise<MiNote> {
|
||||
// チャンネル外にリプライしたら対象のスコープに合わせる
|
||||
// (クライアントサイドでやっても良い処理だと思うけどとりあえずサーバーサイドで)
|
||||
if (data.reply && data.channel && data.reply.channelId !== data.channel.id) {
|
||||
if (data.reply.channelId) {
|
||||
data.channel = await this.channelsRepository.findOneBy({ id: data.reply.channelId });
|
||||
} else {
|
||||
data.channel = null;
|
||||
}
|
||||
}
|
||||
|
||||
// チャンネル内にリプライしたら対象のスコープに合わせる
|
||||
// (クライアントサイドでやっても良い処理だと思うけどとりあえずサーバーサイドで)
|
||||
if (data.reply && (data.channel == null) && data.reply.channelId) {
|
||||
data.channel = await this.channelsRepository.findOneBy({ id: data.reply.channelId });
|
||||
}
|
||||
|
||||
if (data.createdAt == null) data.createdAt = new Date();
|
||||
if (data.visibility == null) data.visibility = 'public';
|
||||
if (data.localOnly == null) data.localOnly = false;
|
||||
if (data.channel != null) data.visibility = 'public';
|
||||
if (data.channel != null) data.visibleUsers = [];
|
||||
if (data.channel != null) data.localOnly = true;
|
||||
|
||||
const meta = await this.metaService.fetch();
|
||||
|
||||
if (data.visibility === 'public' && data.channel == null) {
|
||||
const sensitiveWords = meta.sensitiveWords;
|
||||
if (this.isSensitive(data, sensitiveWords)) {
|
||||
data.visibility = 'home';
|
||||
} else if ((await this.roleService.getUserPolicies(user.id)).canPublicNote === false) {
|
||||
data.visibility = 'home';
|
||||
}
|
||||
}
|
||||
|
||||
const inSilencedInstance = this.utilityService.isSilencedHost(meta.silencedHosts, user.host);
|
||||
|
||||
if (data.visibility === 'public' && inSilencedInstance && user.host !== null) {
|
||||
data.visibility = 'home';
|
||||
}
|
||||
|
||||
if (data.renote) {
|
||||
switch (data.renote.visibility) {
|
||||
case 'public':
|
||||
// public noteは無条件にrenote可能
|
||||
break;
|
||||
case 'home':
|
||||
// home noteはhome以下にrenote可能
|
||||
if (data.visibility === 'public') {
|
||||
data.visibility = 'home';
|
||||
}
|
||||
break;
|
||||
case 'followers':
|
||||
// 他人のfollowers noteはreject
|
||||
if (data.renote.userId !== user.id) {
|
||||
throw new Error('Renote target is not public or home');
|
||||
}
|
||||
|
||||
// Renote対象がfollowersならfollowersにする
|
||||
data.visibility = 'followers';
|
||||
break;
|
||||
case 'specified':
|
||||
// specified / direct noteはreject
|
||||
throw new Error('Renote target is not public or home');
|
||||
}
|
||||
}
|
||||
|
||||
// Check blocking
|
||||
if (data.renote && data.text == null && data.poll == null && (data.files == null || data.files.length === 0)) {
|
||||
if (data.renote.userHost === null) {
|
||||
if (data.renote.userId !== user.id) {
|
||||
const blocked = await this.userBlockingService.checkBlocked(data.renote.userId, user.id);
|
||||
if (blocked) {
|
||||
throw new Error('blocked');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 返信対象がpublicではないならhomeにする
|
||||
if (data.reply && data.reply.visibility !== 'public' && data.visibility === 'public') {
|
||||
data.visibility = 'home';
|
||||
}
|
||||
|
||||
// ローカルのみをRenoteしたらローカルのみにする
|
||||
if (data.renote && data.renote.localOnly && data.channel == null) {
|
||||
data.localOnly = true;
|
||||
}
|
||||
|
||||
// ローカルのみにリプライしたらローカルのみにする
|
||||
if (data.reply && data.reply.localOnly && data.channel == null) {
|
||||
data.localOnly = true;
|
||||
}
|
||||
|
||||
if (data.text) {
|
||||
if (data.text.length > DB_MAX_NOTE_TEXT_LENGTH) {
|
||||
data.text = data.text.slice(0, DB_MAX_NOTE_TEXT_LENGTH);
|
||||
}
|
||||
data.text = data.text.trim();
|
||||
} else {
|
||||
data.text = null;
|
||||
}
|
||||
|
||||
let tags = data.apHashtags;
|
||||
let emojis = data.apEmojis;
|
||||
let mentionedUsers = data.apMentions;
|
||||
|
||||
// Parse MFM if needed
|
||||
if (!tags || !emojis || !mentionedUsers) {
|
||||
const tokens = (data.text ? mfm.parse(data.text)! : []);
|
||||
const cwTokens = data.cw ? mfm.parse(data.cw)! : [];
|
||||
const choiceTokens = data.poll && data.poll.choices
|
||||
? concat(data.poll.choices.map(choice => mfm.parse(choice)!))
|
||||
: [];
|
||||
|
||||
const combinedTokens = tokens.concat(cwTokens).concat(choiceTokens);
|
||||
|
||||
tags = data.apHashtags ?? extractHashtags(combinedTokens);
|
||||
|
||||
emojis = data.apEmojis ?? extractCustomEmojisFromMfm(combinedTokens);
|
||||
|
||||
mentionedUsers = data.apMentions ?? await this.extractMentionedUsers(user, combinedTokens);
|
||||
}
|
||||
|
||||
tags = tags.filter(tag => Array.from(tag).length <= 128).splice(0, 32);
|
||||
|
||||
if (data.reply && (user.id !== data.reply.userId) && !mentionedUsers.some(u => u.id === data.reply!.userId)) {
|
||||
mentionedUsers.push(await this.usersRepository.findOneByOrFail({ id: data.reply!.userId }));
|
||||
}
|
||||
|
||||
if (data.visibility === 'specified') {
|
||||
if (data.visibleUsers == null) throw new Error('invalid param');
|
||||
|
||||
for (const u of data.visibleUsers) {
|
||||
if (!mentionedUsers.some(x => x.id === u.id)) {
|
||||
mentionedUsers.push(u);
|
||||
}
|
||||
}
|
||||
|
||||
if (data.reply && !data.visibleUsers.some(x => x.id === data.reply!.userId)) {
|
||||
data.visibleUsers.push(await this.usersRepository.findOneByOrFail({ id: data.reply!.userId }));
|
||||
}
|
||||
}
|
||||
|
||||
const note = await this.insertNote(user, data, tags, emojis, mentionedUsers);
|
||||
|
||||
setImmediate('post created', { signal: this.#shutdownController.signal }).then(
|
||||
() => this.postNoteImported(note, user, data, silent, tags!, mentionedUsers!),
|
||||
() => { /* aborted, ignore this */ },
|
||||
);
|
||||
|
||||
return note;
|
||||
}
|
||||
|
||||
@bindThis
|
||||
private async insertNote(user: { id: MiUser['id']; host: MiUser['host']; }, data: Option, tags: string[], emojis: string[], mentionedUsers: MinimumUser[]) {
|
||||
const insert = new MiNote({
|
||||
|
|
@ -715,6 +876,113 @@ export class NoteCreateService implements OnApplicationShutdown {
|
|||
if (user.isIndexable) this.index(note);
|
||||
}
|
||||
|
||||
@bindThis
|
||||
private async postNoteImported(note: MiNote, user: {
|
||||
id: MiUser['id'];
|
||||
username: MiUser['username'];
|
||||
host: MiUser['host'];
|
||||
isBot: MiUser['isBot'];
|
||||
isIndexable: MiUser['isIndexable'];
|
||||
}, data: Option, silent: boolean, tags: string[], mentionedUsers: MinimumUser[]) {
|
||||
const meta = await this.metaService.fetch();
|
||||
|
||||
this.notesChart.update(note, true);
|
||||
if (meta.enableChartsForRemoteUser || (user.host == null)) {
|
||||
this.perUserNotesChart.update(user, note, true);
|
||||
}
|
||||
|
||||
// Register host
|
||||
if (this.userEntityService.isRemoteUser(user)) {
|
||||
this.federatedInstanceService.fetch(user.host).then(async i => {
|
||||
if (note.renote && note.text) {
|
||||
this.instancesRepository.increment({ id: i.id }, 'notesCount', 1);
|
||||
} else if (!note.renote) {
|
||||
this.instancesRepository.increment({ id: i.id }, 'notesCount', 1);
|
||||
}
|
||||
if ((await this.metaService.fetch()).enableChartsForFederatedInstances) {
|
||||
this.instanceChart.updateNote(i.host, note, true);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (data.renote && data.text) {
|
||||
// Increment notes count (user)
|
||||
this.incNotesCountOfUser(user);
|
||||
} else if (!data.renote) {
|
||||
// Increment notes count (user)
|
||||
this.incNotesCountOfUser(user);
|
||||
}
|
||||
|
||||
this.pushToTl(note, user);
|
||||
|
||||
this.antennaService.addNoteToAntennas(note, user);
|
||||
|
||||
if (data.reply) {
|
||||
this.saveReply(data.reply, note);
|
||||
}
|
||||
|
||||
if (data.reply == null) {
|
||||
// TODO: キャッシュ
|
||||
this.followingsRepository.findBy({
|
||||
followeeId: user.id,
|
||||
notify: 'normal',
|
||||
}).then(followings => {
|
||||
for (const following of followings) {
|
||||
// TODO: ワードミュート考慮
|
||||
this.notificationService.createNotification(following.followerId, 'note', {
|
||||
noteId: note.id,
|
||||
}, user.id);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (data.renote && data.text == null && data.renote.userId !== user.id && !user.isBot) {
|
||||
this.incRenoteCount(data.renote);
|
||||
}
|
||||
|
||||
if (data.poll && data.poll.expiresAt) {
|
||||
const delay = data.poll.expiresAt.getTime() - Date.now();
|
||||
this.queueService.endedPollNotificationQueue.add(note.id, {
|
||||
noteId: note.id,
|
||||
}, {
|
||||
delay,
|
||||
removeOnComplete: true,
|
||||
});
|
||||
}
|
||||
|
||||
if (!silent) {
|
||||
if (this.userEntityService.isLocalUser(user)) this.activeUsersChart.write(user);
|
||||
|
||||
// Pack the note
|
||||
const noteObj = await this.noteEntityService.pack(note, null, { skipHide: true, withReactionAndUserPairCache: true });
|
||||
|
||||
this.globalEventService.publishNotesStream(noteObj);
|
||||
|
||||
this.roleService.addNoteToRoleTimeline(noteObj);
|
||||
}
|
||||
|
||||
if (data.channel) {
|
||||
this.channelsRepository.increment({ id: data.channel.id }, 'notesCount', 1);
|
||||
this.channelsRepository.update(data.channel.id, {
|
||||
lastNotedAt: new Date(),
|
||||
});
|
||||
|
||||
this.notesRepository.countBy({
|
||||
userId: user.id,
|
||||
channelId: data.channel.id,
|
||||
}).then(count => {
|
||||
// この処理が行われるのはノート作成後なので、ノートが一つしかなかったら最初の投稿だと判断できる
|
||||
// TODO: とはいえノートを削除して何回も投稿すればその分だけインクリメントされる雑さもあるのでどうにかしたい
|
||||
if (count === 1) {
|
||||
this.channelsRepository.increment({ id: data.channel!.id }, 'usersCount', 1);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Register to search database
|
||||
if (user.isIndexable) this.index(note);
|
||||
}
|
||||
|
||||
@bindThis
|
||||
private isSensitive(note: Option, sensitiveWord: string[]): boolean {
|
||||
if (sensitiveWord.length > 0) {
|
||||
|
|
|
|||
|
|
@ -258,6 +258,48 @@ export class QueueService {
|
|||
});
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public createImportNotesJob(user: ThinUser, fileId: MiDriveFile['id'], type: string | null | undefined) {
|
||||
return this.dbQueue.add('importNotes', {
|
||||
user: { id: user.id },
|
||||
fileId: fileId,
|
||||
type: type,
|
||||
}, {
|
||||
removeOnComplete: true,
|
||||
removeOnFail: true,
|
||||
});
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public createImportTweetsToDbJob(user: ThinUser, targets: string[]) {
|
||||
const jobs = targets.map(rel => this.generateToDbJobData('importTweetsToDb', { user, target: rel }));
|
||||
return this.dbQueue.addBulk(jobs);
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public createImportMastoToDbJob(user: ThinUser, targets: string[]) {
|
||||
const jobs = targets.map(rel => this.generateToDbJobData('importMastoToDb', { user, target: rel }));
|
||||
return this.dbQueue.addBulk(jobs);
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public createImportPleroToDbJob(user: ThinUser, targets: string[]) {
|
||||
const jobs = targets.map(rel => this.generateToDbJobData('importPleroToDb', { user, target: rel }));
|
||||
return this.dbQueue.addBulk(jobs);
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public createImportKeyNotesToDbJob(user: ThinUser, targets: string[]) {
|
||||
const jobs = targets.map(rel => this.generateToDbJobData('importKeyNotesToDb', { user, target: rel }));
|
||||
return this.dbQueue.addBulk(jobs);
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public createImportIGToDbJob(user: ThinUser, targets: string[]) {
|
||||
const jobs = targets.map(rel => this.generateToDbJobData('importIGToDb', { user, target: rel }));
|
||||
return this.dbQueue.addBulk(jobs);
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public createImportFollowingToDbJob(user: ThinUser, targets: string[], withReplies?: boolean) {
|
||||
const jobs = targets.map(rel => this.generateToDbJobData('importFollowingToDb', { user, target: rel, withReplies }));
|
||||
|
|
@ -293,7 +335,7 @@ export class QueueService {
|
|||
}
|
||||
|
||||
@bindThis
|
||||
private generateToDbJobData<T extends 'importFollowingToDb' | 'importBlockingToDb', D extends DbJobData<T>>(name: T, data: D): {
|
||||
private generateToDbJobData<T extends 'importFollowingToDb' | 'importBlockingToDb' | 'importTweetsToDb' | 'importIGToDb' | 'importMastoToDb' | 'importPleroToDb' | 'importKeyNotesToDb', D extends DbJobData<T>>(name: T, data: D): {
|
||||
name: string,
|
||||
data: D,
|
||||
opts: Bull.JobsOptions,
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ export type RolePolicies = {
|
|||
userListLimit: number;
|
||||
userEachUserListsLimit: number;
|
||||
rateLimitFactor: number;
|
||||
canImportNotes: boolean;
|
||||
};
|
||||
|
||||
export const DEFAULT_POLICIES: RolePolicies = {
|
||||
|
|
@ -73,6 +74,7 @@ export const DEFAULT_POLICIES: RolePolicies = {
|
|||
userListLimit: 10,
|
||||
userEachUserListsLimit: 50,
|
||||
rateLimitFactor: 1,
|
||||
canImportNotes: true,
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
|
|
@ -323,6 +325,7 @@ export class RoleService implements OnApplicationShutdown {
|
|||
userListLimit: calc('userListLimit', vs => Math.max(...vs)),
|
||||
userEachUserListsLimit: calc('userEachUserListsLimit', vs => Math.max(...vs)),
|
||||
rateLimitFactor: calc('rateLimitFactor', vs => Math.max(...vs)),
|
||||
canImportNotes: calc('canImportNotes', vs => vs.some(v => v === true)),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import { ExportUserListsProcessorService } from './processors/ExportUserListsPro
|
|||
import { ExportAntennasProcessorService } from './processors/ExportAntennasProcessorService.js';
|
||||
import { ImportBlockingProcessorService } from './processors/ImportBlockingProcessorService.js';
|
||||
import { ImportCustomEmojisProcessorService } from './processors/ImportCustomEmojisProcessorService.js';
|
||||
import { ImportNotesProcessorService } from './processors/ImportNotesProcessorService.js';
|
||||
import { ImportFollowingProcessorService } from './processors/ImportFollowingProcessorService.js';
|
||||
import { ImportMutingProcessorService } from './processors/ImportMutingProcessorService.js';
|
||||
import { ImportUserListsProcessorService } from './processors/ImportUserListsProcessorService.js';
|
||||
|
|
@ -61,6 +62,7 @@ import { RelationshipProcessorService } from './processors/RelationshipProcessor
|
|||
ExportBlockingProcessorService,
|
||||
ExportUserListsProcessorService,
|
||||
ExportAntennasProcessorService,
|
||||
ImportNotesProcessorService,
|
||||
ImportFollowingProcessorService,
|
||||
ImportMutingProcessorService,
|
||||
ImportBlockingProcessorService,
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ import { CleanProcessorService } from './processors/CleanProcessorService.js';
|
|||
import { AggregateRetentionProcessorService } from './processors/AggregateRetentionProcessorService.js';
|
||||
import { QueueLoggerService } from './QueueLoggerService.js';
|
||||
import { QUEUE, baseQueueOptions } from './const.js';
|
||||
import { ImportNotesProcessorService } from './processors/ImportNotesProcessorService.js';
|
||||
|
||||
// ref. https://github.com/misskey-dev/misskey/pull/7635#issue-971097019
|
||||
function httpRelatedBackoff(attemptsMade: number) {
|
||||
|
|
@ -100,6 +101,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
|||
private exportUserListsProcessorService: ExportUserListsProcessorService,
|
||||
private exportAntennasProcessorService: ExportAntennasProcessorService,
|
||||
private importFollowingProcessorService: ImportFollowingProcessorService,
|
||||
private importNotesProcessorService: ImportNotesProcessorService,
|
||||
private importMutingProcessorService: ImportMutingProcessorService,
|
||||
private importBlockingProcessorService: ImportBlockingProcessorService,
|
||||
private importUserListsProcessorService: ImportUserListsProcessorService,
|
||||
|
|
@ -174,6 +176,12 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
|||
case 'exportUserLists': return this.exportUserListsProcessorService.process(job);
|
||||
case 'exportAntennas': return this.exportAntennasProcessorService.process(job);
|
||||
case 'importFollowing': return this.importFollowingProcessorService.process(job);
|
||||
case 'importNotes': return this.importNotesProcessorService.process(job);
|
||||
case 'importTweetsToDb': return this.importNotesProcessorService.processTwitterDb(job);
|
||||
case 'importIGToDb': return this.importNotesProcessorService.processIGDb(job);
|
||||
case 'importMastoToDb': return this.importNotesProcessorService.processMastoToDb(job);
|
||||
case 'importPleroToDb': return this.importNotesProcessorService.processPleroToDb(job);
|
||||
case 'importKeyNotesToDb': return this.importNotesProcessorService.processKeyNotesToDb(job);
|
||||
case 'importFollowingToDb': return this.importFollowingProcessorService.processDb(job);
|
||||
case 'importMuting': return this.importMutingProcessorService.process(job);
|
||||
case 'importBlocking': return this.importBlockingProcessorService.process(job);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,490 @@
|
|||
import * as fs from 'node:fs';
|
||||
import * as vm from 'node:vm';
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import { IsNull } from 'typeorm';
|
||||
import { ZipReader } from 'slacc';
|
||||
import { DI } from '@/di-symbols.js';
|
||||
import type { UsersRepository, DriveFilesRepository, MiDriveFile, MiNote } from '@/models/_.js';
|
||||
import type Logger from '@/logger.js';
|
||||
import { DownloadService } from '@/core/DownloadService.js';
|
||||
import { UtilityService } from '@/core/UtilityService.js';
|
||||
import { bindThis } from '@/decorators.js';
|
||||
import { QueueService } from '@/core/QueueService.js';
|
||||
import { createTemp, createTempDir } from '@/misc/create-temp.js';
|
||||
import { NoteCreateService } from '@/core/NoteCreateService.js';
|
||||
import { DriveService } from '@/core/DriveService.js';
|
||||
import { MfmService } from '@/core/MfmService.js';
|
||||
import { ApNoteService } from '@/core/activitypub/models/ApNoteService.js';
|
||||
import { extractApHashtagObjects } from '@/core/activitypub/models/tag.js';
|
||||
import { QueueLoggerService } from '../QueueLoggerService.js';
|
||||
import type * as Bull from 'bullmq';
|
||||
import type { DbNoteImportToDbJobData, DbNoteImportJobData } from '../types.js';
|
||||
|
||||
@Injectable()
|
||||
export class ImportNotesProcessorService {
|
||||
private logger: Logger;
|
||||
|
||||
constructor(
|
||||
@Inject(DI.usersRepository)
|
||||
private usersRepository: UsersRepository,
|
||||
|
||||
@Inject(DI.driveFilesRepository)
|
||||
private driveFilesRepository: DriveFilesRepository,
|
||||
|
||||
private queueService: QueueService,
|
||||
private utilityService: UtilityService,
|
||||
private noteCreateService: NoteCreateService,
|
||||
private mfmService: MfmService,
|
||||
private apNoteService: ApNoteService,
|
||||
private driveService: DriveService,
|
||||
private downloadService: DownloadService,
|
||||
private queueLoggerService: QueueLoggerService,
|
||||
) {
|
||||
this.logger = this.queueLoggerService.logger.createSubLogger('import-notes');
|
||||
}
|
||||
@bindThis
|
||||
private async _keepTweet(tweet: any) {
|
||||
if (!tweet.created_at.endsWith(new Date().getFullYear())) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return !tweet.full_text.startsWith('@');
|
||||
}
|
||||
|
||||
@bindThis
|
||||
private async uploadFiles(dir: any, user: any) {
|
||||
const fileList = fs.readdirSync(dir);
|
||||
for (const file of fileList) {
|
||||
const name = `${dir}/${file}`;
|
||||
if (fs.statSync(name).isDirectory()) {
|
||||
await this.uploadFiles(name, user);
|
||||
} else {
|
||||
const exists = await this.driveFilesRepository.findOneBy({ name: file, userId: user.id });
|
||||
|
||||
if (file.endsWith('.srt')) return;
|
||||
|
||||
if (!exists) {
|
||||
await this.driveService.addFile({
|
||||
user: user,
|
||||
path: name,
|
||||
name: file,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@bindThis
|
||||
private isIterable(obj: any) {
|
||||
if (obj == null) {
|
||||
return false;
|
||||
}
|
||||
return typeof obj[Symbol.iterator] === 'function';
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public async process(job: Bull.Job<DbNoteImportJobData>): Promise<void> {
|
||||
this.logger.info(`Importing following of ${job.data.user.id} ...`);
|
||||
|
||||
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
|
||||
if (user == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
const file = await this.driveFilesRepository.findOneBy({
|
||||
id: job.data.fileId,
|
||||
});
|
||||
if (file == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (job.data.type === 'Twitter' || file.name.startsWith('twitter') && file.name.endsWith('.zip')) {
|
||||
const [path, cleanup] = await createTempDir();
|
||||
|
||||
this.logger.info(`Temp dir is ${path}`);
|
||||
|
||||
const destPath = path + '/twitter.zip';
|
||||
|
||||
try {
|
||||
fs.writeFileSync(destPath, '', 'binary');
|
||||
await this.downloadService.downloadUrl(file.url, destPath);
|
||||
} catch (e) { // TODO: 何度か再試行
|
||||
if (e instanceof Error || typeof e === 'string') {
|
||||
this.logger.error(e);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
|
||||
const outputPath = path + '/twitter';
|
||||
try {
|
||||
this.logger.succ(`Unzipping to ${outputPath}`);
|
||||
ZipReader.withDestinationPath(outputPath).viaBuffer(await fs.promises.readFile(destPath));
|
||||
const fakeWindow: any = {
|
||||
window: {
|
||||
YTD: {
|
||||
tweets: {
|
||||
part0: {},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
const script = new vm.Script(fs.readFileSync(outputPath + '/data/tweets.js', 'utf-8'));
|
||||
const context = vm.createContext(fakeWindow);
|
||||
script.runInContext(context);
|
||||
const tweets = Object.keys(fakeWindow.window.YTD.tweets.part0).reduce((m, key, i, obj) => {
|
||||
return m.concat(fakeWindow.window.YTD.tweets.part0[key].tweet);
|
||||
}, []).filter(this._keepTweet);
|
||||
this.queueService.createImportTweetsToDbJob({ id: user.id }, tweets);
|
||||
} finally {
|
||||
cleanup();
|
||||
}
|
||||
} else if (file.name.endsWith('.zip')) {
|
||||
const [path, cleanup] = await createTempDir();
|
||||
|
||||
this.logger.info(`Temp dir is ${path}`);
|
||||
|
||||
const destPath = path + '/unknown.zip';
|
||||
|
||||
try {
|
||||
fs.writeFileSync(destPath, '', 'binary');
|
||||
await this.downloadService.downloadUrl(file.url, destPath);
|
||||
} catch (e) { // TODO: 何度か再試行
|
||||
if (e instanceof Error || typeof e === 'string') {
|
||||
this.logger.error(e);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
|
||||
const outputPath = path + '/unknown';
|
||||
try {
|
||||
this.logger.succ(`Unzipping to ${outputPath}`);
|
||||
ZipReader.withDestinationPath(outputPath).viaBuffer(await fs.promises.readFile(destPath));
|
||||
const isInstagram = job.data.type === 'Instagram' || fs.existsSync(outputPath + '/instagram_live') || fs.existsSync(outputPath + '/instagram_ads_and_businesses');
|
||||
const isOutbox = job.data.type === 'Mastodon' || fs.existsSync(outputPath + '/outbox.json');
|
||||
if (isInstagram) {
|
||||
const postsJson = fs.readFileSync(outputPath + '/content/posts_1.json', 'utf-8');
|
||||
const posts = JSON.parse(postsJson);
|
||||
await this.uploadFiles(outputPath + '/media/posts', user);
|
||||
this.queueService.createImportIGToDbJob({ id: user.id }, posts);
|
||||
} else if (isOutbox) {
|
||||
const actorJson = fs.readFileSync(outputPath + '/actor.json', 'utf-8');
|
||||
const actor = JSON.parse(actorJson);
|
||||
const isPleroma = actor['@context'].some((v: any) => typeof v === 'string' && v.match(/litepub(.*)/));
|
||||
if (isPleroma) {
|
||||
const outboxJson = fs.readFileSync(outputPath + '/outbox.json', 'utf-8');
|
||||
const outbox = JSON.parse(outboxJson);
|
||||
this.queueService.createImportPleroToDbJob({ id: user.id }, outbox.orderedItems.filter((x: any) => x.type === 'Create' && x.object.type === 'Note'));
|
||||
} else {
|
||||
const outboxJson = fs.readFileSync(outputPath + '/outbox.json', 'utf-8');
|
||||
const outbox = JSON.parse(outboxJson);
|
||||
if (fs.existsSync(outputPath + '/media_attachments/files')) await this.uploadFiles(outputPath + '/media_attachments/files', user);
|
||||
this.queueService.createImportMastoToDbJob({ id: user.id }, outbox.orderedItems.filter((x: any) => x.type === 'Create' && x.object.type === 'Note'));
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
cleanup();
|
||||
}
|
||||
} else if (job.data.type === 'Misskey' || file.name.startsWith('notes-') && file.name.endsWith('.json')) {
|
||||
const [path, cleanup] = await createTemp();
|
||||
|
||||
this.logger.info(`Temp dir is ${path}`);
|
||||
|
||||
try {
|
||||
fs.writeFileSync(path, '', 'utf-8');
|
||||
await this.downloadService.downloadUrl(file.url, path);
|
||||
} catch (e) { // TODO: 何度か再試行
|
||||
if (e instanceof Error || typeof e === 'string') {
|
||||
this.logger.error(e);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
|
||||
const notesJson = fs.readFileSync(path, 'utf-8');
|
||||
const notes = JSON.parse(notesJson);
|
||||
this.queueService.createImportKeyNotesToDbJob({ id: user.id }, notes);
|
||||
cleanup();
|
||||
}
|
||||
|
||||
this.logger.succ('Import jobs created');
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public async processKeyNotesToDb(job: Bull.Job<DbNoteImportToDbJobData>): Promise<void> {
|
||||
const note = job.data.target;
|
||||
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
|
||||
if (user == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
const files: MiDriveFile[] = [];
|
||||
const date = new Date(note.createdAt);
|
||||
|
||||
if (note.files && this.isIterable(note.files)) {
|
||||
for await (const file of note.files) {
|
||||
const [filePath, cleanup] = await createTemp();
|
||||
const slashdex = file.url.lastIndexOf('/');
|
||||
const name = file.url.substring(slashdex + 1);
|
||||
|
||||
const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id });
|
||||
|
||||
if (!exists) {
|
||||
try {
|
||||
await this.downloadService.downloadUrl(file.url, filePath);
|
||||
} catch (e) { // TODO: 何度か再試行
|
||||
this.logger.error(e instanceof Error ? e : new Error(e as string));
|
||||
}
|
||||
const driveFile = await this.driveService.addFile({
|
||||
user: user,
|
||||
path: filePath,
|
||||
name: name,
|
||||
});
|
||||
files.push(driveFile);
|
||||
} else {
|
||||
files.push(exists);
|
||||
}
|
||||
|
||||
cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
await this.noteCreateService.import(user, { createdAt: date, text: note.text, apMentions: new Array(0), visibility: note.visibility, localOnly: note.localOnly, files: files, cw: note.cw });
|
||||
if (note.childNotes) this.queueService.createImportKeyNotesToDbJob(user, note.childNotes);
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public async processMastoToDb(job: Bull.Job<DbNoteImportToDbJobData>): Promise<void> {
|
||||
const toot = job.data.target;
|
||||
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
|
||||
if (user == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
const date = new Date(toot.object.published);
|
||||
let text = undefined;
|
||||
const files: MiDriveFile[] = [];
|
||||
let reply: MiNote | null = null;
|
||||
|
||||
if (toot.object.inReplyTo != null) {
|
||||
try {
|
||||
reply = await this.apNoteService.resolveNote(toot.object.inReplyTo);
|
||||
} catch (error) {
|
||||
reply = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (toot.directMessage) return;
|
||||
|
||||
const hashtags = extractApHashtagObjects(toot.object.tag).map((x) => x.name).filter((x): x is string => x != null);
|
||||
|
||||
try {
|
||||
text = await this.mfmService.fromHtml(toot.object.content, hashtags);
|
||||
} catch (error) {
|
||||
text = undefined;
|
||||
}
|
||||
|
||||
if (toot.object.attachment && this.isIterable(toot.object.attachment)) {
|
||||
for await (const file of toot.object.attachment) {
|
||||
const slashdex = file.url.lastIndexOf('/');
|
||||
const name = file.url.substring(slashdex + 1);
|
||||
const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id });
|
||||
if (exists) {
|
||||
files.push(exists);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await this.noteCreateService.import(user, { createdAt: date, text: text, files: files, apMentions: new Array(0), cw: toot.object.sensitive ? toot.object.summary : null, reply: reply });
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public async processPleroToDb(job: Bull.Job<DbNoteImportToDbJobData>): Promise<void> {
|
||||
const post = job.data.target;
|
||||
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
|
||||
if (user == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
const date = new Date(post.object.published);
|
||||
let text = undefined;
|
||||
const files: MiDriveFile[] = [];
|
||||
let reply: MiNote | null = null;
|
||||
|
||||
if (post.object.inReplyTo != null) {
|
||||
try {
|
||||
reply = await this.apNoteService.resolveNote(post.object.inReplyTo);
|
||||
} catch (error) {
|
||||
reply = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (post.directMessage) return;
|
||||
|
||||
const hashtags = extractApHashtagObjects(post.object.tag).map((x) => x.name).filter((x): x is string => x != null);
|
||||
|
||||
try {
|
||||
text = await this.mfmService.fromHtml(post.object.content, hashtags);
|
||||
} catch (error) {
|
||||
text = undefined;
|
||||
}
|
||||
|
||||
if (post.object.attachment && this.isIterable(post.object.attachment)) {
|
||||
for await (const file of post.object.attachment) {
|
||||
const slashdex = file.url.lastIndexOf('/');
|
||||
const name = file.url.substring(slashdex + 1);
|
||||
const [filePath, cleanup] = await createTemp();
|
||||
|
||||
const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id });
|
||||
|
||||
if (!exists) {
|
||||
try {
|
||||
await this.downloadService.downloadUrl(file.url, filePath);
|
||||
} catch (e) { // TODO: 何度か再試行
|
||||
this.logger.error(e instanceof Error ? e : new Error(e as string));
|
||||
}
|
||||
const driveFile = await this.driveService.addFile({
|
||||
user: user,
|
||||
path: filePath,
|
||||
name: name,
|
||||
});
|
||||
files.push(driveFile);
|
||||
} else {
|
||||
files.push(exists);
|
||||
}
|
||||
|
||||
cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
await this.noteCreateService.import(user, { createdAt: date, text: text, files: files, apMentions: new Array(0), cw: post.object.sensitive ? post.object.summary : null, reply: reply });
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public async processIGDb(job: Bull.Job<DbNoteImportToDbJobData>): Promise<void> {
|
||||
const post = job.data.target;
|
||||
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
|
||||
if (user == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
let date;
|
||||
let title;
|
||||
const files: MiDriveFile[] = [];
|
||||
|
||||
if (post.media && this.isIterable(post.media) && post.media.length > 1) {
|
||||
date = new Date(post.creation_timestamp * 1000);
|
||||
title = post.title;
|
||||
for await (const file of post.media) {
|
||||
const slashdex = file.uri.lastIndexOf('/');
|
||||
const name = file.uri.substring(slashdex + 1);
|
||||
const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: `${name}.jpg`, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: `${name}.mp4`, userId: user.id });
|
||||
if (exists) {
|
||||
files.push(exists);
|
||||
}
|
||||
}
|
||||
} else if (post.media && this.isIterable(post.media) && !(post.media.length > 1)) {
|
||||
date = new Date(post.media[0].creation_timestamp * 1000);
|
||||
title = post.media[0].title;
|
||||
const slashdex = post.media[0].uri.lastIndexOf('/');
|
||||
const name = post.media[0].uri.substring(slashdex + 1);
|
||||
const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: `${name}.jpg`, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: `${name}.mp4`, userId: user.id });
|
||||
if (exists) {
|
||||
files.push(exists);
|
||||
}
|
||||
}
|
||||
|
||||
await this.noteCreateService.import(user, { createdAt: date, text: title, files: files });
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public async processTwitterDb(job: Bull.Job<DbNoteImportToDbJobData>): Promise<void> {
|
||||
const tweet = job.data.target;
|
||||
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
|
||||
if (user == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (tweet.in_reply_to_status_id_str) return;
|
||||
|
||||
async function replaceTwitterUrls(full_text: string, urls: any) {
|
||||
let full_textedit = full_text;
|
||||
urls.forEach((url: any) => {
|
||||
full_textedit = full_textedit.replaceAll(url.url, url.expanded_url);
|
||||
});
|
||||
return full_textedit;
|
||||
}
|
||||
|
||||
async function replaceTwitterMentions(full_text: string, mentions: any) {
|
||||
let full_textedit = full_text;
|
||||
mentions.forEach((mention: any) => {
|
||||
full_textedit = full_textedit.replaceAll(`@${mention.screen_name}`, `[@${mention.screen_name}](https://nitter.net/${mention.screen_name})`);
|
||||
});
|
||||
return full_textedit;
|
||||
}
|
||||
|
||||
try {
|
||||
const date = new Date(tweet.created_at);
|
||||
const textReplaceURLs = tweet.entities.urls && tweet.entities.urls.length > 0 ? await replaceTwitterUrls(tweet.full_text, tweet.entities.urls) : tweet.full_text;
|
||||
const text = tweet.entities.user_mentions && tweet.entities.user_mentions.length > 0 ? await replaceTwitterMentions(textReplaceURLs, tweet.entities.user_mentions) : textReplaceURLs;
|
||||
const files: MiDriveFile[] = [];
|
||||
|
||||
if (tweet.extended_entities && this.isIterable(tweet.extended_entities.media)) {
|
||||
for await (const file of tweet.extended_entities.media) {
|
||||
if (file.video_info) {
|
||||
const [filePath, cleanup] = await createTemp();
|
||||
const slashdex = file.video_info.variants[0].url.lastIndexOf('/');
|
||||
const name = file.video_info.variants[0].url.substring(slashdex + 1);
|
||||
|
||||
const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id });
|
||||
|
||||
const videos = file.video_info.variants.filter((x: any) => x.content_type === 'video/mp4');
|
||||
|
||||
if (!exists) {
|
||||
try {
|
||||
await this.downloadService.downloadUrl(videos[0].url, filePath);
|
||||
} catch (e) { // TODO: 何度か再試行
|
||||
this.logger.error(e instanceof Error ? e : new Error(e as string));
|
||||
}
|
||||
const driveFile = await this.driveService.addFile({
|
||||
user: user,
|
||||
path: filePath,
|
||||
name: name,
|
||||
});
|
||||
files.push(driveFile);
|
||||
} else {
|
||||
files.push(exists);
|
||||
}
|
||||
|
||||
cleanup();
|
||||
} else if (file.media_url_https) {
|
||||
const [filePath, cleanup] = await createTemp();
|
||||
const slashdex = file.media_url_https.lastIndexOf('/');
|
||||
const name = file.media_url_https.substring(slashdex + 1);
|
||||
|
||||
const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id });
|
||||
|
||||
if (!exists) {
|
||||
try {
|
||||
await this.downloadService.downloadUrl(file.media_url_https, filePath);
|
||||
} catch (e) { // TODO: 何度か再試行
|
||||
this.logger.error(e instanceof Error ? e : new Error(e as string));
|
||||
}
|
||||
|
||||
const driveFile = await this.driveService.addFile({
|
||||
user: user,
|
||||
path: filePath,
|
||||
name: name,
|
||||
});
|
||||
files.push(driveFile);
|
||||
} else {
|
||||
files.push(exists);
|
||||
}
|
||||
cleanup();
|
||||
}
|
||||
}
|
||||
}
|
||||
await this.noteCreateService.import(user, { createdAt: date, text: text, files: files });
|
||||
} catch (e) {
|
||||
this.logger.warn(`Error: ${e}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -49,6 +49,12 @@ export type DbJobMap = {
|
|||
exportBlocking: DbJobDataWithUser;
|
||||
exportUserLists: DbJobDataWithUser;
|
||||
importAntennas: DBAntennaImportJobData;
|
||||
importNotes: DbUserImportJobData;
|
||||
importTweetsToDb: DbNoteImportToDbJobData;
|
||||
importIGToDb: DbNoteImportToDbJobData;
|
||||
importMastoToDb: DbNoteImportToDbJobData;
|
||||
importPleroToDb: DbNoteImportToDbJobData;
|
||||
importKeyNotesToDb: DbNoteImportToDbJobData;
|
||||
importFollowing: DbUserImportJobData;
|
||||
importFollowingToDb: DbUserImportToDbJobData;
|
||||
importMuting: DbUserImportJobData;
|
||||
|
|
@ -84,6 +90,12 @@ export type DbUserImportJobData = {
|
|||
withReplies?: boolean;
|
||||
};
|
||||
|
||||
export type DbNoteImportJobData = {
|
||||
user: ThinUser;
|
||||
fileId: MiDriveFile['id'];
|
||||
type?: string;
|
||||
};
|
||||
|
||||
export type DBAntennaImportJobData = {
|
||||
user: ThinUser,
|
||||
antenna: Antenna
|
||||
|
|
@ -95,6 +107,11 @@ export type DbUserImportToDbJobData = {
|
|||
withReplies?: boolean;
|
||||
};
|
||||
|
||||
export type DbNoteImportToDbJobData = {
|
||||
user: ThinUser;
|
||||
target: any;
|
||||
};
|
||||
|
||||
export type ObjectStorageJobData = ObjectStorageFileJobData | Record<string, unknown>;
|
||||
|
||||
export type ObjectStorageFileJobData = {
|
||||
|
|
|
|||
|
|
@ -219,6 +219,7 @@ import * as ep___i_gallery_likes from './endpoints/i/gallery/likes.js';
|
|||
import * as ep___i_gallery_posts from './endpoints/i/gallery/posts.js';
|
||||
import * as ep___i_importBlocking from './endpoints/i/import-blocking.js';
|
||||
import * as ep___i_importFollowing from './endpoints/i/import-following.js';
|
||||
import * as ep___i_importNotes from './endpoints/i/import-notes.js';
|
||||
import * as ep___i_importMuting from './endpoints/i/import-muting.js';
|
||||
import * as ep___i_importUserLists from './endpoints/i/import-user-lists.js';
|
||||
import * as ep___i_importAntennas from './endpoints/i/import-antennas.js';
|
||||
|
|
@ -587,6 +588,7 @@ const $i_gallery_likes: Provider = { provide: 'ep:i/gallery/likes', useClass: ep
|
|||
const $i_gallery_posts: Provider = { provide: 'ep:i/gallery/posts', useClass: ep___i_gallery_posts.default };
|
||||
const $i_importBlocking: Provider = { provide: 'ep:i/import-blocking', useClass: ep___i_importBlocking.default };
|
||||
const $i_importFollowing: Provider = { provide: 'ep:i/import-following', useClass: ep___i_importFollowing.default };
|
||||
const $i_importNotes: Provider = { provide: 'ep:i/import-notes', useClass: ep___i_importNotes.default };
|
||||
const $i_importMuting: Provider = { provide: 'ep:i/import-muting', useClass: ep___i_importMuting.default };
|
||||
const $i_importUserLists: Provider = { provide: 'ep:i/import-user-lists', useClass: ep___i_importUserLists.default };
|
||||
const $i_importAntennas: Provider = { provide: 'ep:i/import-antennas', useClass: ep___i_importAntennas.default };
|
||||
|
|
@ -959,6 +961,7 @@ const $sponsors: Provider = { provide: 'ep:sponsors', useClass: ep___sponsors.de
|
|||
$i_gallery_posts,
|
||||
$i_importBlocking,
|
||||
$i_importFollowing,
|
||||
$i_importNotes,
|
||||
$i_importMuting,
|
||||
$i_importUserLists,
|
||||
$i_importAntennas,
|
||||
|
|
@ -1325,6 +1328,7 @@ const $sponsors: Provider = { provide: 'ep:sponsors', useClass: ep___sponsors.de
|
|||
$i_gallery_posts,
|
||||
$i_importBlocking,
|
||||
$i_importFollowing,
|
||||
$i_importNotes,
|
||||
$i_importMuting,
|
||||
$i_importUserLists,
|
||||
$i_importAntennas,
|
||||
|
|
|
|||
|
|
@ -219,6 +219,7 @@ import * as ep___i_gallery_likes from './endpoints/i/gallery/likes.js';
|
|||
import * as ep___i_gallery_posts from './endpoints/i/gallery/posts.js';
|
||||
import * as ep___i_importBlocking from './endpoints/i/import-blocking.js';
|
||||
import * as ep___i_importFollowing from './endpoints/i/import-following.js';
|
||||
import * as ep___i_importNotes from './endpoints/i/import-notes.js';
|
||||
import * as ep___i_importMuting from './endpoints/i/import-muting.js';
|
||||
import * as ep___i_importUserLists from './endpoints/i/import-user-lists.js';
|
||||
import * as ep___i_importAntennas from './endpoints/i/import-antennas.js';
|
||||
|
|
@ -585,6 +586,7 @@ const eps = [
|
|||
['i/gallery/posts', ep___i_gallery_posts],
|
||||
['i/import-blocking', ep___i_importBlocking],
|
||||
['i/import-following', ep___i_importFollowing],
|
||||
['i/import-notes', ep___i_importNotes],
|
||||
['i/import-muting', ep___i_importMuting],
|
||||
['i/import-user-lists', ep___i_importUserLists],
|
||||
['i/import-antennas', ep___i_importAntennas],
|
||||
|
|
|
|||
72
packages/backend/src/server/api/endpoints/i/import-notes.ts
Normal file
72
packages/backend/src/server/api/endpoints/i/import-notes.ts
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import ms from 'ms';
|
||||
import { Endpoint } from '@/server/api/endpoint-base.js';
|
||||
import { QueueService } from '@/core/QueueService.js';
|
||||
import type { DriveFilesRepository } from '@/models/_.js';
|
||||
import { DI } from '@/di-symbols.js';
|
||||
import { RoleService } from '@/core/RoleService.js';
|
||||
import { ApiError } from '../../error.js';
|
||||
|
||||
export const meta = {
|
||||
secure: true,
|
||||
requireCredential: true,
|
||||
prohibitMoved: true,
|
||||
limit: {
|
||||
duration: ms('1hour'),
|
||||
max: 1,
|
||||
},
|
||||
|
||||
errors: {
|
||||
noSuchFile: {
|
||||
message: 'No such file.',
|
||||
code: 'NO_SUCH_FILE',
|
||||
id: 'b98644cf-a5ac-4277-a502-0b8054a709a3',
|
||||
},
|
||||
|
||||
emptyFile: {
|
||||
message: 'That file is empty.',
|
||||
code: 'EMPTY_FILE',
|
||||
id: '31a1b42c-06f7-42ae-8a38-a661c5c9f691',
|
||||
},
|
||||
|
||||
notPermitted: {
|
||||
message: 'You are not allowed to import notes.',
|
||||
code: 'NO_PERMISSION',
|
||||
id: '31a1b42c-06f7-42ae-8a38-a661c5c9f692',
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
export const paramDef = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
fileId: { type: 'string', format: 'misskey:id' },
|
||||
type: { type: 'string', nullable: true },
|
||||
},
|
||||
required: ['fileId'],
|
||||
} as const;
|
||||
|
||||
@Injectable()
|
||||
export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export
|
||||
constructor(
|
||||
@Inject(DI.driveFilesRepository)
|
||||
private driveFilesRepository: DriveFilesRepository,
|
||||
|
||||
private queueService: QueueService,
|
||||
private roleService: RoleService,
|
||||
) {
|
||||
super(meta, paramDef, async (ps, me) => {
|
||||
const file = await this.driveFilesRepository.findOneBy({ id: ps.fileId });
|
||||
|
||||
if (file == null) throw new ApiError(meta.errors.noSuchFile);
|
||||
|
||||
if (file.size === 0) throw new ApiError(meta.errors.emptyFile);
|
||||
|
||||
if ((await this.roleService.getUserPolicies(me.id)).canImportNotes === false) {
|
||||
throw new ApiError(meta.errors.notPermitted);
|
||||
}
|
||||
|
||||
this.queueService.createImportNotesJob(me, file.id, ps.type);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -112,6 +112,7 @@ export const ROLE_POLICIES = [
|
|||
'gtlAvailable',
|
||||
'ltlAvailable',
|
||||
'canPublicNote',
|
||||
'canImportNotes',
|
||||
'canInvite',
|
||||
'inviteLimit',
|
||||
'inviteLimitCycle',
|
||||
|
|
|
|||
|
|
@ -160,6 +160,26 @@ SPDX-License-Identifier: AGPL-3.0-only
|
|||
</div>
|
||||
</MkFolder>
|
||||
|
||||
<MkFolder v-if="matchQuery([i18n.ts._role._options.canImportNotes, 'canImportNotes'])">
|
||||
<template #label>{{ i18n.ts._role._options.canImportNotes }}</template>
|
||||
<template #suffix>
|
||||
<span v-if="role.policies.canImportNotes.useDefault" :class="$style.useDefaultLabel">{{ i18n.ts._role.useBaseValue }}</span>
|
||||
<span v-else>{{ role.policies.canImportNotes.value ? i18n.ts.yes : i18n.ts.no }}</span>
|
||||
<span :class="$style.priorityIndicator"><i :class="getPriorityIcon(role.policies.canImportNotes)"></i></span>
|
||||
</template>
|
||||
<div class="_gaps">
|
||||
<MkSwitch v-model="role.policies.canImportNotes.useDefault" :readonly="readonly">
|
||||
<template #label>{{ i18n.ts._role.useBaseValue }}</template>
|
||||
</MkSwitch>
|
||||
<MkSwitch v-model="role.policies.canImportNotes.value" :disabled="role.policies.canImportNotes.useDefault" :readonly="readonly">
|
||||
<template #label>{{ i18n.ts.enable }}</template>
|
||||
</MkSwitch>
|
||||
<MkRange v-model="role.policies.canImportNotes.priority" :min="0" :max="2" :step="1" easing :textConverter="(v) => v === 0 ? i18n.ts._role._priority.low : v === 1 ? i18n.ts._role._priority.middle : v === 2 ? i18n.ts._role._priority.high : ''">
|
||||
<template #label>{{ i18n.ts._role.priority }}</template>
|
||||
</MkRange>
|
||||
</div>
|
||||
</MkFolder>
|
||||
|
||||
<MkFolder v-if="matchQuery([i18n.ts._role._options.canInvite, 'canInvite'])">
|
||||
<template #label>{{ i18n.ts._role._options.canInvite }}</template>
|
||||
<template #suffix>
|
||||
|
|
|
|||
|
|
@ -48,6 +48,14 @@ SPDX-License-Identifier: AGPL-3.0-only
|
|||
</MkSwitch>
|
||||
</MkFolder>
|
||||
|
||||
<MkFolder v-if="matchQuery([i18n.ts._role._options.canImportNotes, 'canImportNotes'])">
|
||||
<template #label>{{ i18n.ts._role._options.canImportNotes }}</template>
|
||||
<template #suffix>{{ policies.canImportNotes ? i18n.ts.yes : i18n.ts.no }}</template>
|
||||
<MkSwitch v-model="policies.canImportNotes">
|
||||
<template #label>{{ i18n.ts.enable }}</template>
|
||||
</MkSwitch>
|
||||
</MkFolder>
|
||||
|
||||
<MkFolder v-if="matchQuery([i18n.ts._role._options.canInvite, 'canInvite'])">
|
||||
<template #label>{{ i18n.ts._role._options.canInvite }}</template>
|
||||
<template #suffix>{{ policies.canInvite ? i18n.ts.yes : i18n.ts.no }}</template>
|
||||
|
|
|
|||
|
|
@ -7,11 +7,25 @@ SPDX-License-Identifier: AGPL-3.0-only
|
|||
<div class="_gaps_m">
|
||||
<FormSection first>
|
||||
<template #label><i class="ph-pencil ph-bold ph-lg"></i> {{ i18n.ts._exportOrImport.allNotes }}</template>
|
||||
<div class="_gaps_s">
|
||||
<MkFolder>
|
||||
<template #label>{{ i18n.ts.export }}</template>
|
||||
<template #icon><i class="ph-download ph-bold ph-lg"></i></template>
|
||||
<MkButton primary :class="$style.button" inline @click="exportNotes()"><i class="ph-download ph-bold ph-lg"></i> {{ i18n.ts.export }}</MkButton>
|
||||
</MkFolder>
|
||||
<MkFolder v-if="$i && $i.policies.canImportNotes">
|
||||
<template #label>{{ i18n.ts.import }}</template>
|
||||
<template #icon><i class="ph-upload ph-bold ph-lg"></i></template>
|
||||
<MkRadios v-model="noteType" style="padding-bottom: 8px;" small>
|
||||
<template #label>Origin</template>
|
||||
<option value="Misskey">Misskey/Firefish</option>
|
||||
<option value="Mastodon">Mastodon/Pleroma/Akkoma</option>
|
||||
<option value="Twitter">Twitter</option>
|
||||
<option value="Instagram">Instagram</option>
|
||||
</MkRadios>
|
||||
<MkButton primary :class="$style.button" inline @click="importNotes($event)"><i class="ph-upload ph-bold ph-lg"></i> {{ i18n.ts.import }}</MkButton>
|
||||
</MkFolder>
|
||||
</div>
|
||||
</FormSection>
|
||||
<FormSection>
|
||||
<template #label><i class="ph-star ph-bold ph-lg"></i> {{ i18n.ts._exportOrImport.favoritedNotes }}</template>
|
||||
|
|
@ -116,6 +130,7 @@ import MkButton from '@/components/MkButton.vue';
|
|||
import FormSection from '@/components/form/section.vue';
|
||||
import MkFolder from '@/components/MkFolder.vue';
|
||||
import MkSwitch from '@/components/MkSwitch.vue';
|
||||
import MkRadios from '@/components/MkRadios.vue';
|
||||
import * as os from '@/os.js';
|
||||
import { selectFile } from '@/scripts/select-file.js';
|
||||
import { i18n } from '@/i18n.js';
|
||||
|
|
@ -125,6 +140,7 @@ import { defaultStore } from "@/store.js";
|
|||
|
||||
const excludeMutingUsers = ref(false);
|
||||
const excludeInactiveUsers = ref(false);
|
||||
const noteType = ref(null);
|
||||
const withReplies = ref(defaultStore.state.defaultWithReplies);
|
||||
|
||||
const onExportSuccess = () => {
|
||||
|
|
@ -188,6 +204,14 @@ const importFollowing = async (ev) => {
|
|||
}).then(onImportSuccess).catch(onError);
|
||||
};
|
||||
|
||||
const importNotes = async (ev) => {
|
||||
const file = await selectFile(ev.currentTarget ?? ev.target);
|
||||
os.api('i/import-notes', {
|
||||
fileId: file.id,
|
||||
type: noteType.value,
|
||||
}).then(onImportSuccess).catch(onError);
|
||||
};
|
||||
|
||||
const importUserLists = async (ev) => {
|
||||
const file = await selectFile(ev.currentTarget ?? ev.target);
|
||||
os.api('i/import-user-lists', { fileId: file.id }).then(onImportSuccess).catch(onError);
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ namespace MisskeyEntity {
|
|||
gtlAvailable: boolean
|
||||
ltlAvailable: boolean
|
||||
canPublicNote: boolean
|
||||
canImportNotes: boolean
|
||||
canInvite: boolean
|
||||
canManageCustomEmojis: boolean
|
||||
canHideAds: boolean
|
||||
|
|
|
|||
Loading…
Reference in a new issue