Merge pull request #1879 from syuilo/hybrid-tl

Hybrid tl

locales/ja.yml

@@ -93,6 +93,7 @@ common:
     widgets: "ウィジェット"
     home: "ホーム"
     local: "ローカル"
+    hybrid: "ハイブリッド"
     global: "グローバル"
     notifications: "通知"
     list: "リスト"
@@ -642,6 +643,7 @@ desktop/views/components/taskmanager.vue:
 desktop/views/components/timeline.vue:
   home: "ホーム"
   local: "ローカル"
+  hybrid: "ハイブリッド"
   global: "グローバル"
   list: "リスト"
 
@@ -965,6 +967,7 @@ mobile/views/pages/following.vue:
 mobile/views/pages/home.vue:
   home: "ホーム"
   local: "ローカル"
+  hybrid: "ハイブリッド"
   global: "グローバル"
 
 mobile/views/pages/messaging.vue:

src/client/app/common/scripts/streaming/hybrid-timeline.ts

@@ -0,0 +1,34 @@
+import Stream from './stream';
+import StreamManager from './stream-manager';
+import MiOS from '../../../mios';
+
+/**
+ * Hybrid timeline stream connection
+ */
+export class HybridTimelineStream extends Stream {
+	constructor(os: MiOS, me) {
+		super(os, 'hybrid-timeline', {
+			i: me.token
+		});
+	}
+}
+
+export class HybridTimelineStreamManager extends StreamManager<HybridTimelineStream> {
+	private me;
+	private os: MiOS;
+
+	constructor(os: MiOS, me) {
+		super();
+
+		this.me = me;
+		this.os = os;
+	}
+
+	public getConnection() {
+		if (this.connection == null) {
+			this.connection = new HybridTimelineStream(this.os, this.me);
+		}
+
+		return this.connection;
+	}
+}

src/client/app/desktop/views/components/timeline.core.vue

@@ -43,19 +43,21 @@ export default Vue.extend({
 		},
 
 		stream(): any {
-			return this.src == 'home'
+			switch (this.src) {
-				? (this as any).os.stream
+				case 'home': return (this as any).os.stream;
-				: this.src == 'local'
+				case 'local': return (this as any).os.streams.localTimelineStream;
-					? (this as any).os.streams.localTimelineStream
+				case 'hybrid': return (this as any).os.streams.hybridTimelineStream;
-					: (this as any).os.streams.globalTimelineStream;
+				case 'global': return (this as any).os.streams.globalTimelineStream;
+			}
 		},
 
 		endpoint(): string {
-			return this.src == 'home'
+			switch (this.src) {
-				? 'notes/timeline'
+				case 'home': return 'notes/timeline';
-				: this.src == 'local'
+				case 'local': return 'notes/local-timeline';
-					? 'notes/local-timeline'
+				case 'hybrid': return 'notes/hybrid-timeline';
-					: 'notes/global-timeline';
+				case 'global': return 'notes/global-timeline';
+			}
 		},
 
 		canFetchMore(): boolean {

src/client/app/desktop/views/components/timeline.vue

@@ -3,12 +3,14 @@
 	<header>
 		<span :data-active="src == 'home'" @click="src = 'home'">%fa:home% %i18n:@home%</span>
 		<span :data-active="src == 'local'" @click="src = 'local'">%fa:R comments% %i18n:@local%</span>
+		<span :data-active="src == 'hybrid'" @click="src = 'hybrid'">%fa:share-alt% %i18n:@hybrid%</span>
 		<span :data-active="src == 'global'" @click="src = 'global'">%fa:globe% %i18n:@global%</span>
 		<span :data-active="src == 'list'" @click="src = 'list'" v-if="list">%fa:list% {{ list.title }}</span>
 		<button @click="chooseList" title="%i18n:@list%">%fa:list%</button>
 	</header>
 	<x-core v-if="src == 'home'" ref="tl" key="home" src="home"/>
 	<x-core v-if="src == 'local'" ref="tl" key="local" src="local"/>
+	<x-core v-if="src == 'hybrid'" ref="tl" key="hybrid" src="hybrid"/>
 	<x-core v-if="src == 'global'" ref="tl" key="global" src="global"/>
 	<mk-user-list-timeline v-if="src == 'list'" ref="tl" :key="list.id" :list="list"/>
 </div>

src/client/app/desktop/views/pages/deck/deck.tl-column.vue

@@ -3,6 +3,7 @@
 	<span slot="header">
 		<template v-if="column.type == 'home'">%fa:home%</template>
 		<template v-if="column.type == 'local'">%fa:R comments%</template>
+		<template v-if="column.type == 'hybrid'">%fa:share-alt%</template>
 		<template v-if="column.type == 'global'">%fa:globe%</template>
 		<template v-if="column.type == 'list'">%fa:list%</template>
 		<span>{{ name }}</span>
@@ -61,6 +62,7 @@ export default Vue.extend({
 			switch (this.column.type) {
 				case 'home': return '%i18n:common.deck.home%';
 				case 'local': return '%i18n:common.deck.local%';
+				case 'hybrid': return '%i18n:common.deck.hybrid%';
 				case 'global': return '%i18n:common.deck.global%';
 				case 'list': return this.column.list.title;
 			}

src/client/app/desktop/views/pages/deck/deck.tl.vue

@@ -41,27 +41,29 @@ export default Vue.extend({
 		};
 	},
 
-	watch: {
-		mediaOnly() {
-			this.fetch();
-		}
-	},
-
 	computed: {
 		stream(): any {
-			return this.src == 'home'
+			switch (this.src) {
-				? (this as any).os.stream
+				case 'home': return (this as any).os.stream;
-				: this.src == 'local'
+				case 'local': return (this as any).os.streams.localTimelineStream;
-					? (this as any).os.streams.localTimelineStream
+				case 'hybrid': return (this as any).os.streams.hybridTimelineStream;
-					: (this as any).os.streams.globalTimelineStream;
+				case 'global': return (this as any).os.streams.globalTimelineStream;
+			}
 		},
 
 		endpoint(): string {
-			return this.src == 'home'
+			switch (this.src) {
-				? 'notes/timeline'
+				case 'home': return 'notes/timeline';
-				: this.src == 'local'
+				case 'local': return 'notes/local-timeline';
-					? 'notes/local-timeline'
+				case 'hybrid': return 'notes/hybrid-timeline';
-					: 'notes/global-timeline';
+				case 'global': return 'notes/global-timeline';
+			}
+		},
+	},
+
+	watch: {
+		mediaOnly() {
+			this.fetch();
 		}
 	},
 

src/client/app/desktop/views/pages/deck/deck.vue

@@ -119,6 +119,15 @@ export default Vue.extend({
 							type: 'local'
 						});
 					}
+				}, {
+					icon: '%fa:share-alt%',
+					text: '%i18n:common.deck.hybrid%',
+					action: () => {
+						this.$store.dispatch('settings/addDeckColumn', {
+							id: uuid(),
+							type: 'hybrid'
+						});
+					}
 				}, {
 					icon: '%fa:globe%',
 					text: '%i18n:common.deck.global%',

src/client/app/mios.ts

@@ -15,6 +15,7 @@ import { ReversiStreamManager } from './common/scripts/streaming/games/reversi/r
 
 import Err from './common/views/components/connect-failed.vue';
 import { LocalTimelineStreamManager } from './common/scripts/streaming/local-timeline';
+import { HybridTimelineStreamManager } from './common/scripts/streaming/hybrid-timeline';
 import { GlobalTimelineStreamManager } from './common/scripts/streaming/global-timeline';
 
 //#region api requests
@@ -103,6 +104,7 @@ export default class MiOS extends EventEmitter {
 	 */
 	public streams: {
 		localTimelineStream: LocalTimelineStreamManager;
+		hybridTimelineStream: HybridTimelineStreamManager;
 		globalTimelineStream: GlobalTimelineStreamManager;
 		driveStream: DriveStreamManager;
 		serverStatsStream: ServerStatsStreamManager;
@@ -111,6 +113,7 @@ export default class MiOS extends EventEmitter {
 		reversiStream: ReversiStreamManager;
 	} = {
 		localTimelineStream: null,
+		hybridTimelineStream: null,
 		globalTimelineStream: null,
 		driveStream: null,
 		serverStatsStream: null,
@@ -230,6 +233,7 @@ export default class MiOS extends EventEmitter {
 
 			// Init other stream manager
 			this.streams.localTimelineStream = new LocalTimelineStreamManager(this, this.store.state.i);
+			this.streams.hybridTimelineStream = new HybridTimelineStreamManager(this, this.store.state.i);
 			this.streams.globalTimelineStream = new GlobalTimelineStreamManager(this, this.store.state.i);
 			this.streams.driveStream = new DriveStreamManager(this, this.store.state.i);
 			this.streams.messagingIndexStream = new MessagingIndexStreamManager(this, this.store.state.i);

src/client/app/mobile/views/pages/home.timeline.vue

@@ -42,19 +42,21 @@ export default Vue.extend({
 		},
 
 		stream(): any {
-			return this.src == 'home'
+			switch (this.src) {
-				? (this as any).os.stream
+				case 'home': return (this as any).os.stream;
-				: this.src == 'local'
+				case 'local': return (this as any).os.streams.localTimelineStream;
-					? (this as any).os.streams.localTimelineStream
+				case 'hybrid': return (this as any).os.streams.hybridTimelineStream;
-					: (this as any).os.streams.globalTimelineStream;
+				case 'global': return (this as any).os.streams.globalTimelineStream;
+			}
 		},
 
 		endpoint(): string {
-			return this.src == 'home'
+			switch (this.src) {
-				? 'notes/timeline'
+				case 'home': return 'notes/timeline';
-				: this.src == 'local'
+				case 'local': return 'notes/local-timeline';
-					? 'notes/local-timeline'
+				case 'hybrid': return 'notes/hybrid-timeline';
-					: 'notes/global-timeline';
+				case 'global': return 'notes/global-timeline';
+			}
 		},
 
 		canFetchMore(): boolean {

src/client/app/mobile/views/pages/home.vue

@@ -4,6 +4,7 @@
 		<span>
 			<span v-if="src == 'home'">%fa:home%%i18n:@home%</span>
 			<span v-if="src == 'local'">%fa:R comments%%i18n:@local%</span>
+			<span v-if="src == 'hybrid'">%fa:share-alt%%i18n:@hybrid%</span>
 			<span v-if="src == 'global'">%fa:globe%%i18n:@global%</span>
 			<span v-if="src == 'list'">%fa:list%{{ list.title }}</span>
 		</span>
@@ -24,6 +25,7 @@
 				<div>
 					<span :data-active="src == 'home'" @click="src = 'home'">%fa:home% %i18n:@home%</span>
 					<span :data-active="src == 'local'" @click="src = 'local'">%fa:R comments% %i18n:@local%</span>
+					<span :data-active="src == 'hybrid'" @click="src = 'hybrid'">%fa:share-alt% %i18n:@hybrid%</span>
 					<span :data-active="src == 'global'" @click="src = 'global'">%fa:globe% %i18n:@global%</span>
 					<template v-if="lists">
 						<span v-for="l in lists" :data-active="src == 'list' && list == l" @click="src = 'list'; list = l" :key="l.id">%fa:list% {{ l.title }}</span>
@@ -35,6 +37,7 @@
 		<div class="tl">
 			<x-tl v-if="src == 'home'" ref="tl" key="home" src="home"/>
 			<x-tl v-if="src == 'local'" ref="tl" key="local" src="local"/>
+			<x-tl v-if="src == 'hybrid'" ref="tl" key="hybrid" src="hybrid"/>
 			<x-tl v-if="src == 'global'" ref="tl" key="global" src="global"/>
 			<mk-user-list-timeline v-if="src == 'list'" ref="tl" :key="list.id" :list="list"/>
 		</div>

src/server/api/call.ts

@@ -8,7 +8,7 @@ export default (endpoint: string | Endpoint, user: IUser, app: IApp, data: any,
 
 	const ep = typeof endpoint == 'string' ? endpoints.find(e => e.name == endpoint) : endpoint;
 
-	if (ep.name.includes('.') {
+	if (ep.name.includes('.')) {
 		return rej('INVALID_ENDPOINT');
 	}
 

src/server/api/common/get-friends.ts

@@ -23,12 +23,16 @@ export const getFriendIds = async (me: mongodb.ObjectID, includeMe = true) => {
 	return myfollowingIds;
 };
 
-export const getFriends = async (me: mongodb.ObjectID, includeMe = true) => {
+export const getFriends = async (me: mongodb.ObjectID, includeMe = true, remoteOnly = false) => {
+	const q: any = remoteOnly ? {
+		followerId: me,
+		'_followee.host': { $ne: null }
+	} : {
+		followerId: me
+	};
 	// Fetch relation to other users who the I follows
 	const followings = await Following
-		.find({
+		.find(q);
-			followerId: me
-		});
 
 	// ID list of other users who the I follows
 	const myfollowings = followings.map(following => ({

src/server/api/endpoints.ts

@@ -543,6 +543,13 @@ const endpoints: Endpoint[] = [
 			max: 100
 		}
 	},
+	{
+		name: 'notes/hybrid-timeline',
+		limit: {
+			duration: ms('10minutes'),
+			max: 100
+		}
+	},
 	{
 		name: 'notes/global-timeline',
 		limit: {

src/server/api/endpoints/notes/hybrid-timeline.ts

@@ -0,0 +1,216 @@
+import $ from 'cafy'; import ID from '../../../../misc/cafy-id';
+import Note from '../../../../models/note';
+import Mute from '../../../../models/mute';
+import { getFriends } from '../../common/get-friends';
+import { pack } from '../../../../models/note';
+import { ILocalUser } from '../../../../models/user';
+import getParams from '../../get-params';
+
+export const meta = {
+	name: 'notes/hybrid-timeline',
+
+	desc: {
+		ja: 'ハイブリッドタイムラインを取得します。'
+	},
+
+	params: {
+		limit: $.num.optional.range(1, 100).note({
+			default: 10,
+			desc: {
+				ja: '最大数'
+			}
+		}),
+
+		sinceId: $.type(ID).optional.note({
+			desc: {
+				ja: '指定すると、この投稿を基点としてより新しい投稿を取得します'
+			}
+		}),
+
+		untilId: $.type(ID).optional.note({
+			desc: {
+				ja: '指定すると、この投稿を基点としてより古い投稿を取得します'
+			}
+		}),
+
+		sinceDate: $.num.optional.note({
+			desc: {
+				ja: '指定した時間を基点としてより新しい投稿を取得します。数値は、1970年1月1日 00:00:00 UTC から指定した日時までの経過時間をミリ秒単位で表します。'
+			}
+		}),
+
+		untilDate: $.num.optional.note({
+			desc: {
+				ja: '指定した時間を基点としてより古い投稿を取得します。数値は、1970年1月1日 00:00:00 UTC から指定した日時までの経過時間をミリ秒単位で表します。'
+			}
+		}),
+
+		includeMyRenotes: $.bool.optional.note({
+			default: true,
+			desc: {
+				ja: '自分の行ったRenoteを含めるかどうか'
+			}
+		}),
+
+		includeRenotedMyNotes: $.bool.optional.note({
+			default: true,
+			desc: {
+				ja: 'Renoteされた自分の投稿を含めるかどうか'
+			}
+		}),
+
+		mediaOnly: $.bool.optional.note({
+			desc: {
+				ja: 'true にすると、メディアが添付された投稿だけ取得します'
+			}
+		}),
+	}
+};
+
+/**
+ * Get hybrid timeline of myself
+ */
+export default async (params: any, user: ILocalUser) => {
+	const [ps, psErr] = getParams(meta, params);
+	if (psErr) throw psErr;
+
+	// Check if only one of sinceId, untilId, sinceDate, untilDate specified
+	if ([ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate].filter(x => x != null).length > 1) {
+		throw 'only one of sinceId, untilId, sinceDate, untilDate can be specified';
+	}
+
+	const [followings, mutedUserIds] = await Promise.all([
+		// フォローを取得
+		// Fetch following
+		getFriends(user._id, true, true),
+
+		// ミュートしているユーザーを取得
+		Mute.find({
+			muterId: user._id
+		}).then(ms => ms.map(m => m.muteeId))
+	]);
+
+	//#region Construct query
+	const sort = {
+		_id: -1
+	};
+
+	const followQuery = followings.map(f => f.stalk ? {
+		userId: f.id
+	} : {
+		userId: f.id,
+
+		// ストーキングしてないならリプライは含めない(ただし投稿者自身の投稿へのリプライ、自分の投稿へのリプライ、自分のリプライは含める)
+		$or: [{
+			// リプライでない
+			replyId: null
+		}, { // または
+			// リプライだが返信先が投稿者自身の投稿
+			$expr: {
+				$eq: ['$_reply.userId', '$userId']
+			}
+		}, { // または
+			// リプライだが返信先が自分(フォロワー)の投稿
+			'_reply.userId': user._id
+		}, { // または
+			// 自分(フォロワー)が送信したリプライ
+			userId: user._id
+		}]
+	});
+
+	const query = {
+		$and: [{
+			$or: [{
+				// フォローしている人の投稿
+				$or: followQuery
+			}, {
+				// local
+				'_user.host': null
+			}],
+
+			// mute
+			userId: {
+				$nin: mutedUserIds
+			},
+			'_reply.userId': {
+				$nin: mutedUserIds
+			},
+			'_renote.userId': {
+				$nin: mutedUserIds
+			},
+		}]
+	} as any;
+
+	// MongoDBではトップレベルで否定ができないため、De Morganの法則を利用してクエリします。
+	// つまり、「『自分の投稿かつRenote』ではない」を「『自分の投稿ではない』または『Renoteではない』」と表現します。
+	// for details: https://en.wikipedia.org/wiki/De_Morgan%27s_laws
+
+	if (ps.includeMyRenotes === false) {
+		query.$and.push({
+			$or: [{
+				userId: { $ne: user._id }
+			}, {
+				renoteId: null
+			}, {
+				text: { $ne: null }
+			}, {
+				mediaIds: { $ne: [] }
+			}, {
+				poll: { $ne: null }
+			}]
+		});
+	}
+
+	if (ps.includeRenotedMyNotes === false) {
+		query.$and.push({
+			$or: [{
+				'_renote.userId': { $ne: user._id }
+			}, {
+				renoteId: null
+			}, {
+				text: { $ne: null }
+			}, {
+				mediaIds: { $ne: [] }
+			}, {
+				poll: { $ne: null }
+			}]
+		});
+	}
+
+	if (ps.mediaOnly) {
+		query.$and.push({
+			mediaIds: { $exists: true, $ne: [] }
+		});
+	}
+
+	if (ps.sinceId) {
+		sort._id = 1;
+		query._id = {
+			$gt: ps.sinceId
+		};
+	} else if (ps.untilId) {
+		query._id = {
+			$lt: ps.untilId
+		};
+	} else if (ps.sinceDate) {
+		sort._id = 1;
+		query.createdAt = {
+			$gt: new Date(ps.sinceDate)
+		};
+	} else if (ps.untilDate) {
+		query.createdAt = {
+			$lt: new Date(ps.untilDate)
+		};
+	}
+	//#endregion
+
+	// Issue query
+	const timeline = await Note
+		.find(query, {
+			limit: ps.limit,
+			sort: sort
+		});
+
+	// Serialize
+	return await Promise.all(timeline.map(note => pack(note, user)));
+};

src/server/api/stream/hybrid-timeline.ts

@@ -0,0 +1,47 @@
+import * as websocket from 'websocket';
+import * as redis from 'redis';
+
+import { IUser } from '../../../models/user';
+import Mute from '../../../models/mute';
+import { pack } from '../../../models/note';
+
+export default async function(
+	request: websocket.request,
+	connection: websocket.connection,
+	subscriber: redis.RedisClient,
+	user: IUser
+) {
+	// Subscribe stream
+	subscriber.subscribe(`misskey:hybrid-timeline:${user._id}`);
+
+	const mute = await Mute.find({ muterId: user._id });
+	const mutedUserIds = mute.map(m => m.muteeId.toString());
+
+	subscriber.on('message', async (_, data) => {
+		const note = JSON.parse(data);
+
+		//#region 流れてきたNoteがミュートしているユーザーが関わるものだったら無視する
+		if (mutedUserIds.indexOf(note.userId) != -1) {
+			return;
+		}
+		if (note.reply != null && mutedUserIds.indexOf(note.reply.userId) != -1) {
+			return;
+		}
+		if (note.renote != null && mutedUserIds.indexOf(note.renote.userId) != -1) {
+			return;
+		}
+		//#endregion
+
+		// Renoteなら再pack
+		if (note.renoteId != null) {
+			note.renote = await pack(note.renoteId, user, {
+				detail: true
+			});
+		}
+
+		connection.send(JSON.stringify({
+			type: 'note',
+			body: note
+		}));
+	});
+}

src/services/note/create.ts

@@ -1,7 +1,7 @@
 import es from '../../db/elasticsearch';
 import Note, { pack, INote } from '../../models/note';
 import User, { isLocalUser, IUser, isRemoteUser, IRemoteUser, ILocalUser } from '../../models/user';
-import stream, { publishLocalTimelineStream, publishGlobalTimelineStream, publishUserListStream } from '../../stream';
+import stream, { publishLocalTimelineStream, publishHybridTimelineStream, publishGlobalTimelineStream, publishUserListStream } from '../../stream';
 import Following from '../../models/following';
 import { deliver } from '../../queue';
 import renderNote from '../../remote/activitypub/renderer/note';
@@ -266,9 +266,10 @@ export default async (user: IUser, data: {
 				// Publish event to myself's stream
 				stream(note.userId, 'note', noteObj);
 
-				// Publish note to local timeline stream
+				// Publish note to local and hybrid timeline stream
 				if (note.visibility != 'home') {
 					publishLocalTimelineStream(noteObj);
+					publishHybridTimelineStream(noteObj);
 				}
 			}
 		}
@@ -303,6 +304,10 @@ export default async (user: IUser, data: {
 
 						// Publish event to followers stream
 						stream(following.followerId, 'note', noteObj);
+
+						if (isRemoteUser(user)) {
+							publishHybridTimelineStream(following.followerId, noteObj);
+						}
 					} else {
 						//#region AP配送
 						// フォロワーがリモートユーザーかつ投稿者がローカルユーザーなら投稿を配信

src/stream.ts

@@ -49,6 +49,10 @@ class MisskeyEvent {
 		this.redisClient.publish('misskey:local-timeline', JSON.stringify(note));
 	}
 
+	public publishHybridTimelineStream(userId: ID, note: any): void {
+		this.redisClient.publish(`misskey:hybrid-timeline:${userId}`, JSON.stringify(note));
+	}
+
 	public publishGlobalTimelineStream(note: any): void {
 		this.redisClient.publish('misskey:global-timeline', JSON.stringify(note));
 	}
@@ -67,6 +71,7 @@ const ev = new MisskeyEvent();
 export default ev.publishUserStream.bind(ev);
 
 export const publishLocalTimelineStream = ev.publishLocalTimelineStream.bind(ev);
+export const publishHybridTimelineStream = ev.publishHybridTimelineStream.bind(ev);
 export const publishGlobalTimelineStream = ev.publishGlobalTimelineStream.bind(ev);
 export const publishDriveStream = ev.publishDriveStream.bind(ev);
 export const publishUserListStream = ev.publishUserListStream.bind(ev);