Sharkey/src/services/drive/add-file.ts

544 lines
14 KiB
TypeScript
Raw Normal View History

import { Buffer } from 'buffer';
import * as fs from 'fs';
2016-12-29 00:49:51 +02:00
import * as mongodb from 'mongodb';
import * as crypto from 'crypto';
2018-07-23 19:58:11 +03:00
import * as Minio from 'minio';
import * as uuid from 'uuid';
2018-07-24 18:29:18 +03:00
import * as sharp from 'sharp';
import * as fileType from 'file-type';
import DriveFile, { IMetadata, getDriveFileBucket, IDriveFile } from '../../models/drive-file';
2018-04-04 21:21:11 +03:00
import DriveFolder from '../../models/drive-folder';
import { pack } from '../../models/drive-file';
2019-02-05 07:14:23 +02:00
import { publishMainStream, publishDriveStream } from '../stream';
import { isLocalUser, IUser, IRemoteUser, isRemoteUser } from '../../models/user';
import delFile from './delete-file';
2018-07-21 13:17:15 +03:00
import config from '../../config';
import { getDriveFileWebpublicBucket } from '../../models/drive-file-webpublic';
2018-08-16 00:27:35 +03:00
import { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail';
import driveChart from '../../services/chart/drive';
import perUserDriveChart from '../../services/chart/per-user-drive';
import instanceChart from '../../services/chart/instance';
import fetchMeta from '../../misc/fetch-meta';
import { GenerateVideoThumbnail } from './generate-video-thumbnail';
2019-02-02 21:18:27 +02:00
import { driveLogger } from './logger';
import { IImage, ConvertToJpeg, ConvertToWebp, ConvertToPng } from './image-processor';
import Instance from '../../models/instance';
import checkSvg from '../../misc/check-svg';
2017-02-06 14:11:09 +02:00
2019-02-03 09:45:13 +02:00
const logger = driveLogger.createSubLogger('register', 'yellow');
/***
* Save file
* @param path Path for original
* @param name Name for original
* @param type Content-Type for original
* @param hash Hash for original
* @param size Size for original
* @param metadata
*/
async function save(path: string, name: string, type: string, hash: string, size: number, metadata: IMetadata): Promise<IDriveFile> {
// thunbnail, webpublic を必要なら生成
const alts = await generateAlts(path, type, !metadata.uri);
2018-08-16 00:27:35 +03:00
2018-07-25 02:01:12 +03:00
if (config.drive && config.drive.storage == 'minio') {
//#region ObjectStorage params
let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) || ['']);
if (ext === '') {
if (type === 'image/jpeg') ext = '.jpg';
if (type === 'image/png') ext = '.png';
if (type === 'image/webp') ext = '.webp';
}
const baseUrl = config.drive.baseUrl
|| `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`;
// for original
const key = `${config.drive.prefix}/${uuid.v4()}${ext}`;
const url = `${ baseUrl }/${ key }`;
2018-07-26 21:21:50 +03:00
// for alts
let webpublicKey = null as string;
let webpublicUrl = null as string;
let thumbnailKey = null as string;
let thumbnailUrl = null as string;
//#endregion
//#region Uploads
2019-02-03 09:45:13 +02:00
logger.info(`uploading original: ${key}`);
const uploads = [
upload(key, fs.createReadStream(path), type)
];
2018-07-26 21:21:50 +03:00
if (alts.webpublic) {
webpublicKey = `${config.drive.prefix}/${uuid.v4()}.${alts.webpublic.ext}`;
webpublicUrl = `${ baseUrl }/${ webpublicKey }`;
2019-02-03 09:45:13 +02:00
logger.info(`uploading webpublic: ${webpublicKey}`);
uploads.push(upload(webpublicKey, alts.webpublic.data, alts.webpublic.type));
}
2018-07-23 19:58:11 +03:00
if (alts.thumbnail) {
thumbnailKey = `${config.drive.prefix}/${uuid.v4()}.${alts.thumbnail.ext}`;
thumbnailUrl = `${ baseUrl }/${ thumbnailKey }`;
2019-02-03 09:45:13 +02:00
logger.info(`uploading thumbnail: ${thumbnailKey}`);
uploads.push(upload(thumbnailKey, alts.thumbnail.data, alts.thumbnail.type));
2018-08-16 00:27:35 +03:00
}
await Promise.all(uploads);
//#endregion
//#region DB
2018-07-25 02:01:12 +03:00
Object.assign(metadata, {
withoutChunks: true,
storage: 'minio',
storageProps: {
key,
webpublicKey,
thumbnailKey,
2018-07-25 02:01:12 +03:00
},
url,
webpublicUrl,
thumbnailUrl,
} as IMetadata);
2018-07-23 19:58:11 +03:00
2018-07-25 02:01:12 +03:00
const file = await DriveFile.insert({
length: size,
uploadDate: new Date(),
md5: hash,
filename: name,
metadata: metadata,
contentType: type
});
//#endregion
2018-07-23 19:58:11 +03:00
2018-07-25 02:01:12 +03:00
return file;
} else { // use MongoDB GridFS
// #region store original
const originalDst = await getDriveFileBucket();
// web用(Exif削除済み)がある場合はオリジナルにアクセス制限
if (alts.webpublic) metadata.accessKey = uuid.v4();
2018-08-16 00:27:35 +03:00
const originalFile = await storeOriginal(originalDst, name, path, type, metadata);
2018-08-16 00:27:35 +03:00
2019-02-03 09:45:13 +02:00
logger.info(`original stored to ${originalFile._id}`);
// #endregion store original
// #region store webpublic
if (alts.webpublic) {
const webDst = await getDriveFileWebpublicBucket();
const webFile = await storeAlts(webDst, name, alts.webpublic.data, alts.webpublic.type, originalFile._id);
2019-02-03 09:45:13 +02:00
logger.info(`web stored ${webFile._id}`);
}
// #endregion store webpublic
if (alts.thumbnail) {
const thumDst = await getDriveFileThumbnailBucket();
const thumFile = await storeAlts(thumDst, name, alts.thumbnail.data, alts.thumbnail.type, originalFile._id);
logger.info(`web stored ${thumFile._id}`);
}
2018-08-16 00:27:35 +03:00
return originalFile;
}
}
2018-08-16 00:27:35 +03:00
/**
* Generate webpublic, thumbnail, etc
* @param path Path for original
* @param type Content-Type for original
* @param generateWeb Generate webpublic or not
*/
export async function generateAlts(path: string, type: string, generateWeb: boolean) {
// #region webpublic
let webpublic: IImage;
if (generateWeb) {
logger.info(`creating web image`);
if (['image/jpeg'].includes(type)) {
webpublic = await ConvertToJpeg(path, 2048, 2048);
} else if (['image/webp'].includes(type)) {
webpublic = await ConvertToWebp(path, 2048, 2048);
} else if (['image/png'].includes(type)) {
webpublic = await ConvertToPng(path, 2048, 2048);
} else {
logger.info(`web image not created (not an image)`);
2018-08-16 00:27:35 +03:00
}
} else {
logger.info(`web image not created (from remote)`);
}
// #endregion webpublic
2018-08-16 00:27:35 +03:00
// #region thumbnail
let thumbnail: IImage;
if (['image/jpeg', 'image/webp'].includes(type)) {
thumbnail = await ConvertToJpeg(path, 498, 280);
} else if (['image/png'].includes(type)) {
thumbnail = await ConvertToPng(path, 498, 280);
} else if (type.startsWith('video/')) {
try {
thumbnail = await GenerateVideoThumbnail(path);
} catch (e) {
logger.error(`GenerateVideoThumbnail failed: ${e}`);
}
2018-07-23 19:58:11 +03:00
}
// #endregion thumbnail
return {
webpublic,
thumbnail,
};
2018-07-23 19:58:11 +03:00
}
/**
* Upload to ObjectStorage
*/
async function upload(key: string, stream: fs.ReadStream | Buffer, type: string) {
const minio = new Minio.Client(config.drive.config);
await minio.putObject(config.drive.bucket, key, stream, null, {
'Content-Type': type,
'Cache-Control': 'max-age=31536000, immutable'
});
}
/**
* GridFSBucketにオリジナルを格納する
*/
export async function storeOriginal(bucket: mongodb.GridFSBucket, name: string, path: string, contentType: string, metadata: any) {
return new Promise<IDriveFile>((resolve, reject) => {
const writeStream = bucket.openUploadStream(name, {
contentType,
metadata
});
writeStream.once('finish', resolve);
writeStream.on('error', reject);
fs.createReadStream(path).pipe(writeStream);
});
}
/**
* GridFSBucketにオリジナル以外を格納する
*/
export async function storeAlts(bucket: mongodb.GridFSBucket, name: string, data: Buffer, contentType: string, originalId: mongodb.ObjectID) {
return new Promise<IDriveFile>((resolve, reject) => {
const writeStream = bucket.openUploadStream(name, {
contentType,
metadata: {
originalId
}
});
writeStream.once('finish', resolve);
writeStream.on('error', reject);
writeStream.end(data);
});
}
2018-05-25 12:41:49 +03:00
async function deleteOldFile(user: IRemoteUser) {
const oldFile = await DriveFile.findOne({
_id: {
$nin: [user.avatarId, user.bannerId]
2018-08-25 04:28:09 +03:00
},
'metadata.userId': user._id
2018-05-25 12:41:49 +03:00
}, {
sort: {
_id: 1
}
});
if (oldFile) {
delFile(oldFile, true);
2018-05-25 12:41:49 +03:00
}
}
/**
* Add file to drive
*
* @param user User who wish to add file
* @param path File path
* @param name Name
* @param comment Comment
* @param folderId Folder ID
* @param force If set to true, forcibly upload the file even if there is a file with the same hash.
* @param isLink Do not save file to local
* @param url URL of source (URLからアップロードされた場合(/)URL)
* @param uri URL of source (URLからアップロードされた場合の元URL)
* @param sensitive Mark file as sensitive
2018-05-25 12:41:49 +03:00
* @return Created drive file
*/
export default async function(
2018-05-26 03:57:31 +03:00
user: IUser,
path: string,
2016-12-29 00:49:51 +02:00
name: string = null,
comment: string = null,
folderId: mongodb.ObjectID = null,
2018-04-03 17:45:13 +03:00
force: boolean = false,
2018-07-23 23:04:43 +03:00
isLink: boolean = false,
url: string = null,
2018-07-19 20:40:37 +03:00
uri: string = null,
sensitive: boolean = null
2018-05-25 12:41:49 +03:00
): Promise<IDriveFile> {
// Calc md5 hash
const calcHash = new Promise<string>((res, rej) => {
const readable = fs.createReadStream(path);
const hash = crypto.createHash('md5');
const chunks: Buffer[] = [];
2018-05-25 12:41:49 +03:00
readable
.on('error', rej)
.pipe(hash)
.on('error', rej)
.on('data', chunk => chunks.push(chunk))
.on('end', () => {
const buffer = Buffer.concat(chunks);
res(buffer.toString('hex'));
});
2018-05-25 12:41:49 +03:00
});
// Detect content type
const detectMime = new Promise<[string, string]>((res, rej) => {
const readable = fs.createReadStream(path);
readable
.on('error', rej)
.once('data', (buffer: Buffer) => {
readable.destroy();
const type = fileType(buffer);
if (type) {
2019-03-05 00:56:23 +02:00
if (type.mime == 'application/xml' && checkSvg(path)) {
res(['image/svg+xml', 'svg']);
} else {
res([type.mime, type.ext]);
}
} else if (checkSvg(path)) {
2019-01-28 10:49:20 +02:00
res(['image/svg+xml', 'svg']);
2018-05-25 12:41:49 +03:00
} else {
// 種類が同定できなかったら application/octet-stream にする
res(['application/octet-stream', null]);
}
})
.on('end', () => {
// maybe 0 bytes
res(['application/octet-stream', null]);
2018-05-25 12:41:49 +03:00
});
});
// Get file size
const getFileSize = new Promise<number>((res, rej) => {
fs.stat(path, (err, stats) => {
if (err) return rej(err);
res(stats.size);
});
});
const [hash, [mime, ext], size] = await Promise.all([calcHash, detectMime, getFileSize]);
2019-02-03 09:45:13 +02:00
logger.info(`hash: ${hash}, mime: ${mime}, ext: ${ext}, size: ${size}`);
2016-12-29 00:49:51 +02:00
// detect name
2018-05-25 12:41:49 +03:00
const detectedName = name || (ext ? `untitled.${ext}` : 'untitled');
if (!force) {
// Check if there is a file with the same hash
const much = await DriveFile.findOne({
md5: hash,
2018-05-13 10:52:47 +03:00
'metadata.userId': user._id,
2018-05-14 06:47:33 +03:00
'metadata.deletedAt': { $exists: false }
});
2018-06-08 16:03:14 +03:00
if (much) {
2019-02-03 09:45:13 +02:00
logger.info(`file with same hash is found: ${much._id}`);
return much;
}
}
2018-05-25 12:41:49 +03:00
//#region Check drive usage
2018-07-23 23:04:43 +03:00
if (!isLink) {
const usage = await DriveFile
.aggregate([{
$match: {
'metadata.userId': user._id,
'metadata.deletedAt': { $exists: false }
}
}, {
$project: {
length: true
}
}, {
$group: {
_id: null,
usage: { $sum: '$length' }
}
}])
.then((aggregates: any[]) => {
if (aggregates.length > 0) {
return aggregates[0].usage;
}
return 0;
});
2019-03-03 00:47:58 +02:00
logger.debug(`drive usage is ${usage}`);
const instance = await fetchMeta();
const driveCapacity = 1024 * 1024 * (isLocalUser(user) ? instance.localDriveCapacityMb : instance.remoteDriveCapacityMb);
2018-07-21 13:17:15 +03:00
// If usage limit exceeded
2018-07-21 13:17:15 +03:00
if (usage + size > driveCapacity) {
if (isLocalUser(user)) {
throw 'no-free-space';
} else {
// (アバターまたはバナーを含まず)最も古いファイルを削除する
deleteOldFile(user);
}
2018-05-25 12:41:49 +03:00
}
}
//#endregion
const fetchFolder = async () => {
if (!folderId) {
return null;
}
const driveFolder = await DriveFolder.findOne({
_id: folderId,
userId: user._id
});
if (driveFolder == null) throw 'folder-not-found';
return driveFolder;
};
const properties: {[key: string]: any} = {};
2018-05-25 12:41:49 +03:00
let propPromises: Promise<void>[] = [];
2018-07-24 18:29:18 +03:00
const isImage = ['image/jpeg', 'image/gif', 'image/png', 'image/webp'].includes(mime);
2018-05-25 12:41:49 +03:00
if (isImage) {
2018-07-24 18:29:18 +03:00
const img = sharp(path);
2018-05-25 12:41:49 +03:00
// Calc width and height
const calcWh = async () => {
2019-02-03 09:45:13 +02:00
logger.debug('calculating image width and height...');
2017-12-11 06:33:33 +02:00
2017-12-08 12:42:02 +02:00
// Calculate width and height
2018-07-24 18:29:18 +03:00
const meta = await img.metadata();
2019-02-03 09:45:13 +02:00
logger.debug(`image width and height is calculated: ${meta.width}, ${meta.height}`);
2018-07-24 18:29:18 +03:00
properties['width'] = meta.width;
properties['height'] = meta.height;
2018-05-25 12:41:49 +03:00
};
2017-12-11 06:33:33 +02:00
2018-05-25 12:41:49 +03:00
// Calc average color
const calcAvg = async () => {
2019-02-03 09:45:13 +02:00
logger.debug('calculating average color...');
2017-12-11 06:33:33 +02:00
2018-07-26 11:10:43 +03:00
try {
const info = await (img as any).stats();
2017-12-11 06:33:33 +02:00
2018-07-26 11:10:43 +03:00
const r = Math.round(info.channels[0].mean);
const g = Math.round(info.channels[1].mean);
const b = Math.round(info.channels[2].mean);
2017-12-11 06:33:33 +02:00
2019-02-03 09:45:13 +02:00
logger.debug(`average color is calculated: ${r}, ${g}, ${b}`);
2017-12-11 06:33:33 +02:00
2018-07-26 11:10:43 +03:00
const value = info.isOpaque ? [r, g, b] : [r, g, b, 255];
2018-07-26 11:10:43 +03:00
properties['avgColor'] = value;
} catch (e) { }
2018-05-25 12:41:49 +03:00
};
2017-12-08 12:42:02 +02:00
2018-05-25 12:41:49 +03:00
propPromises = [calcWh(), calcAvg()];
2017-12-08 12:42:02 +02:00
}
2018-05-27 17:19:57 +03:00
const [folder] = await Promise.all([fetchFolder(), Promise.all(propPromises)]);
2018-05-25 12:41:49 +03:00
2018-04-03 17:45:13 +03:00
const metadata = {
2018-03-29 08:48:47 +03:00
userId: user._id,
_user: {
host: user.host
},
2018-03-29 08:48:47 +03:00
folderId: folder !== null ? folder._id : null,
comment: comment,
properties: properties,
2018-07-23 23:04:43 +03:00
withoutChunks: isLink,
2018-07-24 18:29:18 +03:00
isRemote: isLink,
isSensitive: isLocalUser(user) && user.settings.alwaysMarkNsfw ? true :
(sensitive !== null && sensitive !== undefined)
? sensitive
: false
2018-04-03 17:45:13 +03:00
} as IMetadata;
if (url !== null) {
2018-07-23 23:04:43 +03:00
metadata.src = url;
if (isLink) {
metadata.url = url;
}
}
2018-04-03 17:45:13 +03:00
if (uri !== null) {
metadata.uri = uri;
}
2018-08-10 08:33:34 +03:00
let driveFile: IDriveFile;
if (isLink) {
try {
driveFile = await DriveFile.insert({
length: 0,
uploadDate: new Date(),
md5: hash,
filename: detectedName,
metadata: metadata,
contentType: mime
});
} catch (e) {
// duplicate key error (when already registered)
if (e.code === 11000) {
2019-02-03 09:45:13 +02:00
logger.info(`already registered ${metadata.uri}`);
2018-08-10 08:33:34 +03:00
driveFile = await DriveFile.findOne({
'metadata.uri': metadata.uri,
'metadata.userId': user._id
});
} else {
2019-02-03 09:45:13 +02:00
logger.error(e);
2018-08-10 08:33:34 +03:00
throw e;
}
}
} else {
2018-08-16 00:27:35 +03:00
driveFile = await (save(path, detectedName, mime, hash, size, metadata));
2018-08-10 08:33:34 +03:00
}
2018-05-25 12:41:49 +03:00
2019-02-03 09:45:13 +02:00
logger.succ(`drive file has been created ${driveFile._id}`);
2018-05-25 12:41:49 +03:00
pack(driveFile).then(packedFile => {
// Publish driveFileCreated event
publishMainStream(user._id, 'driveFileCreated', packedFile);
2018-10-23 01:01:43 +03:00
publishDriveStream(user._id, 'fileCreated', packedFile);
2018-05-25 12:41:49 +03:00
});
2018-08-18 17:56:44 +03:00
// 統計を更新
2018-10-22 23:36:35 +03:00
driveChart.update(driveFile, true);
perUserDriveChart.update(driveFile, true);
if (isRemoteUser(driveFile.metadata._user)) {
instanceChart.updateDrive(driveFile, true);
Instance.update({ host: driveFile.metadata._user.host }, {
$inc: {
driveUsage: driveFile.length,
driveFiles: 1
}
});
}
2018-05-25 12:41:49 +03:00
return driveFile;
}