Introduce processor

gulpfile.ts

@@ -22,7 +22,7 @@ import * as replace from 'gulp-replace';
 import * as htmlmin from 'gulp-htmlmin';
 const uglifyes = require('uglify-es');
 
-import { fa } from './src/common/build/fa';
+import { fa } from './src/build/fa';
 import version from './src/version';
 import config from './src/conf';
 
@@ -39,7 +39,7 @@ if (isDebug) {
 
 const constants = require('./src/const.json');
 
-require('./src/web/docs/gulpfile.ts');
+require('./src/server/web/docs/gulpfile.ts');
 
 gulp.task('build', [
 	'build:js',
@@ -52,7 +52,7 @@ gulp.task('build', [
 gulp.task('rebuild', ['clean', 'build']);
 
 gulp.task('build:js', () =>
-	gulp.src(['./src/**/*.js', '!./src/web/**/*.js'])
+	gulp.src(['./src/**/*.js', '!./src/server/web/**/*.js'])
 		.pipe(gulp.dest('./built/'))
 );
 
@@ -71,7 +71,7 @@ gulp.task('build:copy', () =>
 	gulp.src([
 		'./build/Release/crypto_key.node',
 		'./src/**/assets/**/*',
-		'!./src/web/app/**/assets/**/*'
+		'!./src/server/web/app/**/assets/**/*'
 	]).pipe(gulp.dest('./built/'))
 );
 
@@ -121,7 +121,7 @@ gulp.task('build:client', [
 ]);
 
 gulp.task('build:client:script', () =>
-	gulp.src(['./src/web/app/boot.js', './src/web/app/safe.js'])
+	gulp.src(['./src/server/web/app/boot.js', './src/server/web/app/safe.js'])
 		.pipe(replace('VERSION', JSON.stringify(version)))
 		.pipe(replace('API', JSON.stringify(config.api_url)))
 		.pipe(replace('ENV', JSON.stringify(env)))
@@ -129,15 +129,15 @@ gulp.task('build:client:script', () =>
 		.pipe(isProduction ? uglify({
 			toplevel: true
 		} as any) : gutil.noop())
-		.pipe(gulp.dest('./built/web/assets/')) as any
+		.pipe(gulp.dest('./built/server/web/assets/')) as any
 );
 
 gulp.task('build:client:styles', () =>
-	gulp.src('./src/web/app/init.css')
+	gulp.src('./src/server/web/app/init.css')
 		.pipe(isProduction
 			? (cssnano as any)()
 			: gutil.noop())
-		.pipe(gulp.dest('./built/web/assets/'))
+		.pipe(gulp.dest('./built/server/web/assets/'))
 );
 
 gulp.task('copy:client', [
@@ -145,14 +145,14 @@ gulp.task('copy:client', [
 ], () =>
 		gulp.src([
 			'./assets/**/*',
-			'./src/web/assets/**/*',
-			'./src/web/app/*/assets/**/*'
+			'./src/server/web/assets/**/*',
+			'./src/server/web/app/*/assets/**/*'
 		])
 			.pipe(isProduction ? (imagemin as any)() : gutil.noop())
 			.pipe(rename(path => {
 				path.dirname = path.dirname.replace('assets', '.');
 			}))
-			.pipe(gulp.dest('./built/web/assets/'))
+			.pipe(gulp.dest('./built/server/web/assets/'))
 );
 
 gulp.task('build:client:pug', [
@@ -160,13 +160,13 @@ gulp.task('build:client:pug', [
 	'build:client:script',
 	'build:client:styles'
 ], () =>
-		gulp.src('./src/web/app/base.pug')
+		gulp.src('./src/server/web/app/base.pug')
 			.pipe(pug({
 				locals: {
 					themeColor: constants.themeColor,
 					facss: fa.dom.css(),
 					//hljscss: fs.readFileSync('./node_modules/highlight.js/styles/default.css', 'utf8')
-					hljscss: fs.readFileSync('./src/web/assets/code-highlight.css', 'utf8')
+					hljscss: fs.readFileSync('./src/server/web/assets/code-highlight.css', 'utf8')
 				}
 			}))
 			.pipe(htmlmin({
@@ -201,5 +201,5 @@ gulp.task('build:client:pug', [
 				// CSSも圧縮する
 				minifyCSS: true
 			}))
-			.pipe(gulp.dest('./built/web/app/'))
+			.pipe(gulp.dest('./built/server/web/app/'))
 );

package.json

@@ -135,6 +135,7 @@
 		"is-url": "1.2.3",
 		"js-yaml": "3.11.0",
 		"jsdom": "^11.6.2",
+		"kue": "^0.11.6",
 		"license-checker": "18.0.0",
 		"loader-utils": "1.1.0",
 		"mecab-async": "0.1.2",
@@ -149,6 +150,7 @@
 		"nan": "^2.10.0",
 		"node-sass": "4.7.2",
 		"node-sass-json-importer": "3.1.5",
+		"nopt": "^4.0.1",
 		"nprogress": "0.2.0",
 		"object-assign-deep": "0.3.1",
 		"on-build-webpack": "0.1.0",

src/build/i18n.ts

@@ -2,7 +2,7 @@
  * Replace i18n texts
  */
 
-import locale from '../../../locales';
+import locale from '../../locales';
 
 export default class Replacer {
 	private lang: string;

src/build/license.ts

@@ -1,6 +1,6 @@
 import * as fs from 'fs';
 
-const license = fs.readFileSync(__dirname + '/../../../LICENSE', 'utf-8');
+const license = fs.readFileSync(__dirname + '/../../LICENSE', 'utf-8');
 
 const licenseHtml = license
 	.replace(/\r\n/g, '\n')

src/index.ts

@@ -24,6 +24,8 @@ import stats from './utils/stats';
 import { Config, path as configPath } from './config';
 import loadConfig from './config';
 
+import parseOpt from './parse-opt';
+
 const clusterLog = debug('misskey:cluster');
 const ev = new Xev();
 
@@ -36,20 +38,22 @@ main();
  * Init process
  */
 function main() {
+	const opt = parseOpt(process.argv, 2);
+
 	if (cluster.isMaster) {
-		masterMain();
+		masterMain(opt);
 
 		ev.mount();
 		stats();
 	} else {
-		workerMain();
+		workerMain(opt);
 	}
 }
 
 /**
  * Init master process
  */
-async function masterMain() {
+async function masterMain(opt) {
 	let config: Config;
 
 	try {
@@ -69,19 +73,35 @@ async function masterMain() {
 	}
 
 	spawnWorkers(() => {
-		Logger.info(chalk.bold.green(
-			`Now listening on port ${chalk.underline(config.port.toString())}`));
+		if (!opt['only-processor']) {
+			Logger.info(chalk.bold.green(
+				`Now listening on port ${chalk.underline(config.port.toString())}`));
 
-		Logger.info(chalk.bold.green(config.url));
+			Logger.info(chalk.bold.green(config.url));
+		}
+
+		if (!opt['only-server']) {
+			Logger.info(chalk.bold.green('Now processing jobs'));
+		}
 	});
 }
 
 /**
  * Init worker process
  */
-function workerMain() {
-	// start server
-	require('./server');
+async function workerMain(opt) {
+	if (!opt['only-processor']) {
+		// start server
+		await require('./server').default();
+	}
+
+	if (!opt['only-server']) {
+		// start processor
+		require('./processor').default();
+	}
+
+	// Send a 'ready' message to parent process
+	process.send('ready');
 }
 
 /**

src/parse-opt.ts

@@ -0,0 +1,17 @@
+const nopt = require('nopt');
+
+export default (vector, index) => {
+	const parsed = nopt({
+		'only-processor': Boolean,
+		'only-server': Boolean
+	}, {
+		p: ['--only-processor'],
+		s: ['--only-server']
+	}, vector, index);
+
+	if (parsed['only-processor'] && parsed['only-server']) {
+		throw 'only-processor option and only-server option cannot be set at the same time';
+	}
+
+	return parsed;
+};

src/processor/index.ts

@@ -0,0 +1,4 @@
+import queue from '../queue';
+import reportGitHubFailure from './report-github-failure';
+
+export default () => queue.process('gitHubFailureReport', reportGitHubFailure);

src/processor/report-github-failure.ts

@@ -0,0 +1,29 @@
+import * as request from 'request';
+import User from '../server/api/models/user';
+const createPost = require('../server/api/endpoints/posts/create');
+
+export default ({ data }, done) => {
+	const asyncBot = User.findOne({ _id: data.userId });
+
+	// Fetch parent status
+	request({
+		url: `${data.parentUrl}/statuses`,
+		headers: {
+			'User-Agent': 'misskey'
+		}
+	}, async (err, res, body) => {
+		if (err) {
+			console.error(err);
+			return;
+		}
+		const parentStatuses = JSON.parse(body);
+		const parentState = parentStatuses[0].state;
+		const stillFailed = parentState == 'failure' || parentState == 'error';
+		const text = stillFailed ?
+			`**⚠️BUILD STILL FAILED⚠️**: ?[${data.message}](${data.htmlUrl})` :
+			`**🚨BUILD FAILED🚨**: →→→?[${data.message}](${data.htmlUrl})←←←`;
+
+		createPost({ text }, await asyncBot);
+		done();
+	});
+};

src/queue.ts

@@ -0,0 +1,10 @@
+import { createQueue } from 'kue';
+import config from './conf';
+
+export default createQueue({
+	redis: {
+		port: config.redis.port,
+		host: config.redis.host,
+		auth: config.redis.pass
+	}
+});

src/server/api/bot/interfaces/line.ts

@@ -3,9 +3,9 @@ import * as express from 'express';
 import * as request from 'request';
 import * as crypto from 'crypto';
 import User from '../../models/user';
-import config from '../../../conf';
+import config from '../../../../conf';
 import BotCore from '../core';
-import _redis from '../../../db/redis';
+import _redis from '../../../../db/redis';
 import prominence = require('prominence');
 import getAcct from '../../../common/user/get-acct';
 import parseAcct from '../../../common/user/parse-acct';

src/server/api/common/drive/add-file.ts

@@ -15,7 +15,7 @@ import DriveFolder from '../../models/drive-folder';
 import { pack } from '../../models/drive-file';
 import event, { publishDriveStream } from '../../event';
 import getAcct from '../../../common/user/get-acct';
-import config from '../../../conf';
+import config from '../../../../conf';
 
 const gm = _gm.subClass({
 	imageMagick: true

src/server/api/common/push-sw.ts

@@ -1,7 +1,7 @@
 const push = require('web-push');
 import * as mongo from 'mongodb';
 import Subscription from '../models/sw-subscription';
-import config from '../../conf';
+import config from '../../../conf';
 
 if (config.sw) {
 	// アプリケーションの連絡先と、サーバーサイドの鍵ペアの情報を登録

src/server/api/common/signin.ts

@@ -1,4 +1,4 @@
-import config from '../../conf';
+import config from '../../../conf';
 
 export default function(res, user, redirect: boolean) {
 	const expires = 1000 * 60 * 60 * 24 * 365; // One Year

src/server/api/endpoints/auth/session/generate.ts

@@ -5,7 +5,7 @@ import * as uuid from 'uuid';
 import $ from 'cafy';
 import App from '../../../models/app';
 import AuthSess from '../../../models/auth-session';
-import config from '../../../../conf';
+import config from '../../../../../conf';
 
 /**
  * @swagger

src/server/api/endpoints/i/2fa/register.ts

@@ -6,7 +6,7 @@ import * as bcrypt from 'bcryptjs';
 import * as speakeasy from 'speakeasy';
 import * as QRCode from 'qrcode';
 import User from '../../../models/user';
-import config from '../../../../conf';
+import config from '../../../../../conf';
 
 module.exports = async (params, user) => new Promise(async (res, rej) => {
 	// Get 'password' parameter

src/server/api/endpoints/i/update.ts

@@ -4,7 +4,7 @@
 import $ from 'cafy';
 import User, { isValidName, isValidDescription, isValidLocation, isValidBirthday, pack } from '../../models/user';
 import event from '../../event';
-import config from '../../../conf';
+import config from '../../../../conf';
 
 /**
  * Update myself

src/server/api/endpoints/messaging/messages/create.ts

@@ -11,7 +11,7 @@ import DriveFile from '../../../models/drive-file';
 import { pack } from '../../../models/messaging-message';
 import publishUserStream from '../../../event';
 import { publishMessagingStream, publishMessagingIndexStream, pushSw } from '../../../event';
-import config from '../../../../conf';
+import config from '../../../../../conf';
 
 /**
  * Create a message

src/server/api/endpoints/meta.ts

@@ -2,8 +2,8 @@
  * Module dependencies
  */
 import * as os from 'os';
-import version from '../../version';
-import config from '../../conf';
+import version from '../../../version';
+import config from '../../../conf';
 import Meta from '../models/meta';
 
 /**