Sharkey/src/migrate.ts

468 lines
14 KiB
TypeScript
Raw Normal View History

2019-04-09 06:31:45 +03:00
process.env.NODE_ENV = 'production';
2019-04-08 19:03:58 +03:00
import monk from 'monk';
import * as mongo from 'mongodb';
import * as fs from 'fs';
import * as uuid from 'uuid';
2019-04-09 06:31:45 +03:00
import chalk from 'chalk';
2019-04-08 14:33:42 +03:00
import config from './config';
import { initDb } from './db/postgre';
import { User } from './models/entities/user';
import { getRepository } from 'typeorm';
import generateUserToken from './server/api/common/generate-native-user-token';
2019-04-08 17:24:44 +03:00
import { DriveFile } from './models/entities/drive-file';
2019-04-08 19:31:54 +03:00
import { DriveFolder } from './models/entities/drive-folder';
2019-04-08 19:03:58 +03:00
import { InternalStorage } from './services/drive/internal-storage';
import { createTemp } from './misc/create-temp';
2019-04-09 06:31:45 +03:00
import { Note } from './models/entities/note';
2019-04-09 12:35:07 +03:00
import { Following } from './models/entities/following';
import { genId } from './misc/gen-id';
import { Poll } from './models/entities/poll';
import { PollVote } from './models/entities/poll-vote';
2019-04-09 12:41:53 +03:00
import { NoteFavorite } from './models/entities/note-favorite';
2019-04-09 12:45:21 +03:00
import { NoteReaction } from './models/entities/note-reaction';
2019-04-09 13:02:09 +03:00
import { UserPublickey } from './models/entities/user-publickey';
import { UserKeypair } from './models/entities/user-keypair';
import { extractPublic } from './crypto_key';
2019-04-09 13:20:10 +03:00
import { Emoji } from './models/entities/emoji';
2019-04-08 14:33:42 +03:00
const u = (config as any).mongodb.user ? encodeURIComponent((config as any).mongodb.user) : null;
const p = (config as any).mongodb.pass ? encodeURIComponent((config as any).mongodb.pass) : null;
const uri = `mongodb://${u && p ? `${u}:${p}@` : ''}${(config as any).mongodb.host}:${(config as any).mongodb.port}/${(config as any).mongodb.db}`;
2019-04-08 19:03:58 +03:00
const db = monk(uri);
let mdb: mongo.Db;
const nativeDbConn = async (): Promise<mongo.Db> => {
if (mdb) return mdb;
const db = await ((): Promise<mongo.Db> => new Promise((resolve, reject) => {
mongo.MongoClient.connect(uri, { useNewUrlParser: true }, (e: Error, client: any) => {
if (e) return reject(e);
2019-04-08 19:31:54 +03:00
resolve(client.db((config as any).mongodb.db));
2019-04-08 19:03:58 +03:00
});
}))();
mdb = db;
return db;
};
2019-04-08 14:33:42 +03:00
const _User = db.get<any>('users');
2019-04-08 17:24:44 +03:00
const _DriveFile = db.get<any>('driveFiles.files');
2019-04-08 19:31:54 +03:00
const _DriveFolder = db.get<any>('driveFolders');
2019-04-09 06:31:45 +03:00
const _Note = db.get<any>('notes');
2019-04-09 12:35:07 +03:00
const _Following = db.get<any>('following');
const _PollVote = db.get<any>('pollVotes');
2019-04-09 12:41:53 +03:00
const _Favorite = db.get<any>('favorites');
2019-04-09 12:45:21 +03:00
const _NoteReaction = db.get<any>('noteReactions');
2019-04-09 13:20:10 +03:00
const _Emoji = db.get<any>('emoji');
2019-04-08 19:03:58 +03:00
const getDriveFileBucket = async (): Promise<mongo.GridFSBucket> => {
const db = await nativeDbConn();
const bucket = new mongo.GridFSBucket(db, {
bucketName: 'driveFiles'
});
return bucket;
};
2019-04-08 14:33:42 +03:00
async function main() {
await initDb();
const Users = getRepository(User);
2019-04-08 17:24:44 +03:00
const DriveFiles = getRepository(DriveFile);
2019-04-08 19:31:54 +03:00
const DriveFolders = getRepository(DriveFolder);
2019-04-09 06:31:45 +03:00
const Notes = getRepository(Note);
2019-04-09 12:35:07 +03:00
const Followings = getRepository(Following);
const Polls = getRepository(Poll);
const PollVotes = getRepository(PollVote);
2019-04-09 12:41:53 +03:00
const NoteFavorites = getRepository(NoteFavorite);
2019-04-09 12:45:21 +03:00
const NoteReactions = getRepository(NoteReaction);
2019-04-09 13:02:09 +03:00
const UserPublickeys = getRepository(UserPublickey);
const UserKeypairs = getRepository(UserKeypair);
2019-04-09 13:20:10 +03:00
const Emojis = getRepository(Emoji);
2019-04-09 12:35:07 +03:00
async function migrateUser(user: any) {
2019-04-09 13:02:09 +03:00
await Users.save({
2019-04-09 12:35:07 +03:00
id: user._id.toHexString(),
createdAt: user.createdAt || new Date(),
username: user.username,
usernameLower: user.username.toLowerCase(),
host: user.host,
token: generateUserToken(),
password: user.password,
isAdmin: user.isAdmin,
autoAcceptFollowed: true,
autoWatch: false,
name: user.name,
location: user.profile ? user.profile.location : null,
birthday: user.profile ? user.profile.birthday : null,
followersCount: user.followersCount,
followingCount: user.followingCount,
notesCount: user.notesCount,
description: user.description,
isBot: user.isBot,
isCat: user.isCat,
isVerified: user.isVerified,
inbox: user.inbox,
sharedInbox: user.sharedInbox,
uri: user.uri,
});
2019-04-09 13:02:09 +03:00
if (user.publicKey) {
await UserPublickeys.save({
id: genId(),
userId: user._id.toHexString(),
keyId: user.publicKey.id,
keyPem: user.publicKey.publicKeyPem
});
}
if (user.keypair) {
await UserKeypairs.save({
id: genId(),
userId: user._id.toHexString(),
publicKey: extractPublic(user.keypair),
privateKey: user.keypair,
});
}
2019-04-09 12:35:07 +03:00
}
async function migrateFollowing(following: any) {
await Followings.save({
id: following._id.toHexString(),
createdAt: following.createdAt || new Date(),
followerId: following.followerId.toHexString(),
followeeId: following.followeeId.toHexString(),
// 非正規化
followerHost: following._follower ? following._follower.host : null,
followerInbox: following._follower ? following._follower.inbox : null,
followerSharedInbox: following._follower ? following._follower.sharedInbox : null,
followeeHost: following._followee ? following._followee.host : null,
followeeInbox: following._followee ? following._followee.inbox : null,
2019-04-09 12:37:14 +03:00
followeeSharedInbox: following._followee ? following._followee.sharedInbo : null
2019-04-09 12:35:07 +03:00
});
}
async function migrateDriveFolder(folder: any) {
await DriveFolders.save({
id: folder._id.toHexString(),
createdAt: folder.createdAt || new Date(),
name: folder.name,
parentId: folder.parentId ? folder.parentId.toHexString() : null,
});
}
2019-04-08 19:31:54 +03:00
async function migrateDriveFile(file: any) {
const user = await _User.findOne({
_id: file.metadata.userId
});
if (file.metadata.storage && file.metadata.storage.key) { // when object storage
await DriveFiles.save({
id: file._id.toHexString(),
userId: user._id.toHexString(),
userHost: user.host,
createdAt: file.uploadDate || new Date(),
md5: file.md5,
name: file.filename,
type: file.contentType,
properties: file.metadata.properties,
size: file.length,
url: file.metadata.url,
uri: file.metadata.uri,
accessKey: file.metadata.storage.key,
2019-04-09 12:35:07 +03:00
folderId: file.metadata.folderId ? file.metadata.folderId.toHexString() : null,
2019-04-08 19:31:54 +03:00
storedInternal: false,
isRemote: false
});
} else if (!file.metadata.isRemote) {
const [temp, clean] = await createTemp();
await new Promise(async (res, rej) => {
const bucket = await getDriveFileBucket();
const readable = bucket.openDownloadStream(file._id);
const dest = fs.createWriteStream(temp);
2019-04-08 19:42:07 +03:00
readable.pipe(dest);
2019-04-08 19:31:54 +03:00
readable.on('end', () => {
dest.end();
res();
});
});
const key = uuid.v4();
const url = InternalStorage.saveFromPath(key, temp);
await DriveFiles.save({
id: file._id.toHexString(),
userId: user._id.toHexString(),
userHost: user.host,
createdAt: file.uploadDate || new Date(),
md5: file.md5,
name: file.filename,
type: file.contentType,
properties: file.metadata.properties,
size: file.length,
url: url,
uri: file.metadata.uri,
accessKey: key,
folderId: file.metadata.folderId,
storedInternal: true,
isRemote: false
});
clean();
} else {
await DriveFiles.save({
id: file._id.toHexString(),
userId: user._id.toHexString(),
userHost: user.host,
createdAt: file.uploadDate || new Date(),
md5: file.md5,
name: file.filename,
type: file.contentType,
properties: file.metadata.properties,
size: file.length,
url: file.metadata.url,
uri: file.metadata.uri,
accessKey: null,
folderId: file.metadata.folderId,
storedInternal: false,
isRemote: true
});
}
}
2019-04-08 14:33:42 +03:00
2019-04-09 06:31:45 +03:00
async function migrateNote(note: any) {
2019-04-09 13:02:09 +03:00
await Notes.save({
2019-04-09 06:31:45 +03:00
id: note._id.toHexString(),
createdAt: note.createdAt || new Date(),
text: note.text,
cw: note.cw || null,
tags: note.tags || [],
userId: note.userId.toHexString(),
viaMobile: note.viaMobile || false,
geo: note.geo,
appId: null,
visibility: note.visibility || 'public',
visibleUserIds: note.visibleUserIds ? note.visibleUserIds.map((id: any) => id.toHexString()) : [],
replyId: note.replyId ? note.replyId.toHexString() : null,
renoteId: note.renoteId ? note.renoteId.toHexString() : null,
userHost: null,
fileIds: note.fileIds ? note.fileIds.map((id: any) => id.toHexString()) : [],
2019-04-09 12:35:07 +03:00
localOnly: note.localOnly || false,
hasPoll: note.poll != null
});
if (note.poll) {
await Polls.save({
id: genId(),
noteId: note._id.toHexString(),
choices: note.poll.choices.map((x: any) => x.text),
expiresAt: note.poll.expiresAt,
multiple: note.poll.multiple,
votes: note.poll.choices.map((x: any) => x.votes),
noteVisibility: note.visibility,
userId: note.userId.toHexString(),
userHost: null
});
}
}
async function migratePollVote(vote: any) {
await PollVotes.save({
id: vote._id.toHexString(),
createdAt: vote.createdAt,
2019-04-09 12:41:53 +03:00
noteId: vote.noteId.toHexString(),
userId: vote.userId.toHexString(),
2019-04-09 12:35:07 +03:00
choice: vote.choice
2019-04-09 06:31:45 +03:00
});
}
2019-04-09 12:41:53 +03:00
async function migrateNoteFavorite(favorite: any) {
await NoteFavorites.save({
id: favorite._id.toHexString(),
createdAt: favorite.createdAt,
noteId: favorite.noteId.toHexString(),
userId: favorite.userId.toHexString(),
});
}
2019-04-09 12:45:21 +03:00
async function migrateNoteReaction(reaction: any) {
await NoteReactions.save({
id: reaction._id.toHexString(),
createdAt: reaction.createdAt,
noteId: reaction.noteId.toHexString(),
userId: reaction.userId.toHexString(),
reaction: reaction.reaction
});
}
2019-04-09 13:14:18 +03:00
async function reMigrateUser(user: any) {
const u = await _User.findOne({
_id: new mongo.ObjectId(user.id)
});
const avatar = await DriveFiles.findOne(u.avatarId.toHexString());
const banner = await DriveFiles.findOne(u.bannerId.toHexString());
await Users.update(user.id, {
avatarId: avatar.id,
bannerId: banner.id,
avatarUrl: avatar.url,
bannerUrl: banner.url
});
}
2019-04-09 13:20:10 +03:00
async function migrateEmoji(emoji: any) {
await NoteReactions.save({
id: emoji._id.toHexString(),
updatedAt: emoji.createdAt,
aliases: emoji.aliases,
url: emoji.url,
uri: emoji.uri,
host: emoji.host,
name: emoji.name
});
}
2019-04-08 14:33:42 +03:00
const allUsersCount = await _User.count();
for (let i = 0; i < allUsersCount; i++) {
const user = await _User.findOne({}, {
skip: i
});
2019-04-09 12:35:07 +03:00
try {
await migrateUser(user);
console.log(`USER (${i + 1}/${allUsersCount}) ${user._id} ${chalk.green('DONE')}`);
} catch (e) {
console.log(`USER (${i + 1}/${allUsersCount}) ${user._id} ${chalk.red('ERR')}`);
console.error(e);
}
}
const allFollowingsCount = await _Following.count();
for (let i = 0; i < allFollowingsCount; i++) {
const following = await _Following.findOne({}, {
skip: i
2019-04-08 17:24:44 +03:00
});
2019-04-09 12:35:07 +03:00
try {
await migrateFollowing(following);
console.log(`FOLLOWING (${i + 1}/${allFollowingsCount}) ${following._id} ${chalk.green('DONE')}`);
} catch (e) {
console.log(`FOLLOWING (${i + 1}/${allFollowingsCount}) ${following._id} ${chalk.red('ERR')}`);
console.error(e);
}
2019-04-08 17:24:44 +03:00
}
2019-04-08 19:31:54 +03:00
const allDriveFoldersCount = await _DriveFolder.count();
for (let i = 0; i < allDriveFoldersCount; i++) {
const folder = await _DriveFolder.findOne({}, {
skip: i
});
2019-04-09 12:35:07 +03:00
try {
await migrateDriveFolder(folder);
2019-04-09 12:45:21 +03:00
console.log(`FOLDER (${i + 1}/${allDriveFoldersCount}) ${folder._id} ${chalk.green('DONE')}`);
2019-04-09 12:35:07 +03:00
} catch (e) {
2019-04-09 12:45:21 +03:00
console.log(`FOLDER (${i + 1}/${allDriveFoldersCount}) ${folder._id} ${chalk.red('ERR')}`);
2019-04-09 12:35:07 +03:00
console.error(e);
}
2019-04-08 19:31:54 +03:00
}
2019-04-08 17:24:44 +03:00
const allDriveFilesCount = await _DriveFile.count();
for (let i = 0; i < allDriveFilesCount; i++) {
const file = await _DriveFile.findOne({}, {
skip: i
});
2019-04-08 19:31:54 +03:00
try {
await migrateDriveFile(file);
2019-04-09 12:45:21 +03:00
console.log(`FILE (${i + 1}/${allDriveFilesCount}) ${file._id} ${chalk.green('DONE')}`);
2019-04-09 06:31:45 +03:00
} catch (e) {
2019-04-09 12:45:21 +03:00
console.log(`FILE (${i + 1}/${allDriveFilesCount}) ${file._id} ${chalk.red('ERR')}`);
2019-04-09 06:31:45 +03:00
console.error(e);
}
}
const allNotesCount = await _Note.count({
'_user.host': null
});
for (let i = 0; i < allNotesCount; i++) {
const note = await _Note.findOne({
'_user.host': null
}, {
skip: i
});
try {
await migrateNote(note);
console.log(`NOTE (${i + 1}/${allNotesCount}) ${note._id} ${chalk.green('DONE')}`);
2019-04-08 19:31:54 +03:00
} catch (e) {
2019-04-09 06:31:45 +03:00
console.log(`NOTE (${i + 1}/${allNotesCount}) ${note._id} ${chalk.red('ERR')}`);
console.error(e);
2019-04-08 19:03:58 +03:00
}
2019-04-08 14:33:42 +03:00
}
2019-04-09 12:35:07 +03:00
const allPollVotesCount = await _PollVote.count();
for (let i = 0; i < allPollVotesCount; i++) {
const vote = await _PollVote.findOne({}, {
skip: i
});
try {
await migratePollVote(vote);
2019-04-09 12:45:21 +03:00
console.log(`VOTE (${i + 1}/${allPollVotesCount}) ${vote._id} ${chalk.green('DONE')}`);
2019-04-09 12:35:07 +03:00
} catch (e) {
2019-04-09 12:45:21 +03:00
console.log(`VOTE (${i + 1}/${allPollVotesCount}) ${vote._id} ${chalk.red('ERR')}`);
2019-04-09 12:35:07 +03:00
console.error(e);
}
}
2019-04-09 12:41:53 +03:00
const allNoteFavoritesCount = await _Favorite.count();
for (let i = 0; i < allNoteFavoritesCount; i++) {
const favorite = await _Favorite.findOne({}, {
skip: i
});
try {
await migrateNoteFavorite(favorite);
console.log(`FAVORITE (${i + 1}/${allNoteFavoritesCount}) ${favorite._id} ${chalk.green('DONE')}`);
} catch (e) {
console.log(`FAVORITE (${i + 1}/${allNoteFavoritesCount}) ${favorite._id} ${chalk.red('ERR')}`);
console.error(e);
}
}
2019-04-09 12:45:21 +03:00
const allNoteReactionsCount = await _NoteReaction.count();
for (let i = 0; i < allNoteReactionsCount; i++) {
const reaction = await _NoteReaction.findOne({}, {
skip: i
});
try {
await migrateNoteReaction(reaction);
console.log(`REACTION (${i + 1}/${allNoteReactionsCount}) ${reaction._id} ${chalk.green('DONE')}`);
} catch (e) {
console.log(`REACTION (${i + 1}/${allNoteReactionsCount}) ${reaction._id} ${chalk.red('ERR')}`);
console.error(e);
}
}
2019-04-09 13:14:18 +03:00
const allActualUsersCount = await Users.count();
for (let i = 0; i < allActualUsersCount; i++) {
const [user] = await Users.find({
take: 1,
skip: i
});
try {
await reMigrateUser(user);
console.log(`RE:USER (${i + 1}/${allActualUsersCount}) ${user.id} ${chalk.green('DONE')}`);
} catch (e) {
console.log(`RE:USER (${i + 1}/${allActualUsersCount}) ${user.id} ${chalk.red('ERR')}`);
console.error(e);
}
}
2019-04-09 13:20:10 +03:00
const allEmojisCount = await _Emoji.count();
for (let i = 0; i < allEmojisCount; i++) {
const emoji = await _Emoji.findOne({}, {
skip: i
});
try {
await migrateEmoji(emoji);
console.log(`EMOJI (${i + 1}/${allEmojisCount}) ${emoji._id} ${chalk.green('DONE')}`);
} catch (e) {
console.log(`EMOJI (${i + 1}/${allEmojisCount}) ${emoji._id} ${chalk.red('ERR')}`);
console.error(e);
}
}
2019-04-08 14:33:42 +03:00
}
main();