Improve MFM parser (#3337)

* wip

* wip

* Refactor

* Refactor

* wip

* wip

* wip

* wip

* Refactor

* Refactor

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Clean up

* Update misskey-flavored-markdown.ts

* wip

* wip

* wip

* wip

* Update parser.ts

* wip

* Add new test

* wip

* Add new test

* Add new test

* wip

* Refactor

* Update parse.ts

* Refactor

* Update parser.ts

* wip
This commit is contained in:
syuilo 2018-11-21 05:11:00 +09:00 committed by GitHub
parent 6e347e4221
commit 79ffbf95db
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
44 changed files with 1097 additions and 916 deletions

View file

@ -65,6 +65,7 @@
"@types/ms": "0.7.30",
"@types/node": "10.12.2",
"@types/oauth": "0.9.1",
"@types/parsimmon": "1.10.0",
"@types/portscanner": "2.1.0",
"@types/pug": "2.0.4",
"@types/qrcode": "1.3.0",
@ -170,6 +171,7 @@
"on-build-webpack": "0.1.0",
"os-utils": "0.0.14",
"parse5": "5.1.0",
"parsimmon": "1.12.0",
"portscanner": "2.2.0",
"postcss-loader": "3.0.0",
"progress-bar-webpack-plugin": "1.11.0",

View file

@ -41,6 +41,7 @@
if (`${url.pathname}/`.startsWith('/dev/')) app = 'dev';
if (`${url.pathname}/`.startsWith('/auth/')) app = 'auth';
if (`${url.pathname}/`.startsWith('/admin/')) app = 'admin';
if (`${url.pathname}/`.startsWith('/test/')) app = 'test';
//#endregion
// Script version

View file

@ -17,7 +17,7 @@ import forkit from './forkit.vue';
import acct from './acct.vue';
import avatar from './avatar.vue';
import nav from './nav.vue';
import misskeyFlavoredMarkdown from './misskey-flavored-markdown';
import misskeyFlavoredMarkdown from './misskey-flavored-markdown.vue';
import poll from './poll.vue';
import pollEditor from './poll-editor.vue';
import reactionIcon from './reaction-icon.vue';

View file

@ -1,11 +1,39 @@
import Vue, { VNode } from 'vue';
import { length } from 'stringz';
import { Node } from '../../../../../mfm/parser';
import parse from '../../../../../mfm/parse';
import getAcct from '../../../../../misc/acct/render';
import MkUrl from './url.vue';
import { concat } from '../../../../../prelude/array';
import MkFormula from './formula.vue';
import MkGoogle from './google.vue';
import { toUnicode } from 'punycode';
import syntaxHighlight from '../../../../../mfm/syntax-highlight';
function getText(tokens: Node[]): string {
let text = '';
const extract = (tokens: Node[]) => {
tokens.filter(x => x.name === 'text').forEach(x => {
text += x.props.text;
});
tokens.filter(x => x.children).forEach(x => {
extract(x.children);
});
};
extract(tokens);
return text;
}
function getChildrenCount(tokens: Node[]): number {
let count = 0;
const extract = (tokens: Node[]) => {
tokens.filter(x => x.children).forEach(x => {
count++;
extract(x.children);
});
};
extract(tokens);
return count;
}
export default Vue.component('misskey-flavored-markdown', {
props: {
@ -21,6 +49,10 @@ export default Vue.component('misskey-flavored-markdown', {
type: Boolean,
default: true
},
author: {
type: Object,
default: null
},
i: {
type: Object,
default: null
@ -31,23 +63,24 @@ export default Vue.component('misskey-flavored-markdown', {
},
render(createElement) {
let ast: any[];
if (this.text == null || this.text == '') return;
let ast: Node[];
if (this.ast == null) {
// Parse text to ast
ast = parse(this.text);
} else {
ast = this.ast as any[];
ast = this.ast as Node[];
}
let bigCount = 0;
let motionCount = 0;
// Parse ast to DOM
const els = concat(ast.map((token): VNode[] => {
switch (token.type) {
const genEl = (ast: Node[]) => concat(ast.map((token): VNode[] => {
switch (token.name) {
case 'text': {
const text = token.content.replace(/(\r\n|\n|\r)/g, '\n');
const text = token.props.text.replace(/(\r\n|\n|\r)/g, '\n');
if (this.shouldBreak) {
const x = text.split('\n')
@ -60,12 +93,12 @@ export default Vue.component('misskey-flavored-markdown', {
}
case 'bold': {
return [createElement('b', token.bold)];
return [createElement('b', genEl(token.children))];
}
case 'big': {
bigCount++;
const isLong = length(token.big) > 10;
const isLong = length(getText(token.children)) > 10 || getChildrenCount(token.children) > 5;
const isMany = bigCount > 3;
return (createElement as any)('strong', {
attrs: {
@ -75,12 +108,12 @@ export default Vue.component('misskey-flavored-markdown', {
name: 'animate-css',
value: { classes: 'tada', iteration: 'infinite' }
}]
}, token.big);
}, genEl(token.children));
}
case 'motion': {
motionCount++;
const isLong = length(token.motion) > 10;
const isLong = length(getText(token.children)) > 10 || getChildrenCount(token.children) > 5;
const isMany = motionCount > 3;
return (createElement as any)('span', {
attrs: {
@ -90,13 +123,14 @@ export default Vue.component('misskey-flavored-markdown', {
name: 'animate-css',
value: { classes: 'rubberBand', iteration: 'infinite' }
}]
}, token.motion);
}, genEl(token.children));
}
case 'url': {
return [createElement(MkUrl, {
key: Math.random(),
props: {
url: token.content,
url: token.props.url,
target: '_blank',
style: 'color:var(--mfmLink);'
}
@ -107,75 +141,75 @@ export default Vue.component('misskey-flavored-markdown', {
return [createElement('a', {
attrs: {
class: 'link',
href: token.url,
href: token.props.url,
target: '_blank',
title: token.url,
title: token.props.url,
style: 'color:var(--mfmLink);'
}
}, token.title)];
}, genEl(token.children))];
}
case 'mention': {
const host = token.props.host == null && this.author && this.author.host != null ? this.author.host : token.props.host;
const canonical = host != null ? `@${token.props.username}@${toUnicode(host)}` : `@${token.props.username}`;
return (createElement as any)('router-link', {
key: Math.random(),
attrs: {
to: `/${token.canonical}`,
dataIsMe: (this as any).i && getAcct((this as any).i) == getAcct(token),
to: `/${canonical}`,
// TODO
//dataIsMe: (this as any).i && getAcct((this as any).i) == getAcct(token),
style: 'color:var(--mfmMention);'
},
directives: [{
name: 'user-preview',
value: token.canonical
value: canonical
}]
}, token.canonical);
}, canonical);
}
case 'hashtag': {
return [createElement('router-link', {
key: Math.random(),
attrs: {
to: `/tags/${encodeURIComponent(token.hashtag)}`,
to: `/tags/${encodeURIComponent(token.props.hashtag)}`,
style: 'color:var(--mfmHashtag);'
}
}, token.content)];
}, `#${token.props.hashtag}`)];
}
case 'code': {
case 'blockCode': {
return [createElement('pre', {
class: 'code'
}, [
createElement('code', {
domProps: {
innerHTML: token.html
innerHTML: syntaxHighlight(token.props.code)
}
})
])];
}
case 'inline-code': {
case 'inlineCode': {
return [createElement('code', {
domProps: {
innerHTML: token.html
innerHTML: syntaxHighlight(token.props.code)
}
})];
}
case 'quote': {
const text2 = token.quote.replace(/(\r\n|\n|\r)/g, '\n');
if (this.shouldBreak) {
const x = text2.split('\n')
.map(t => [createElement('span', t), createElement('br')]);
x[x.length - 1].pop();
return [createElement('div', {
attrs: {
class: 'quote'
}
}, x)];
}, genEl(token.children))];
} else {
return [createElement('span', {
attrs: {
class: 'quote'
}
}, text2.replace(/\n/g, ' '))];
}, genEl(token.children))];
}
}
@ -184,15 +218,16 @@ export default Vue.component('misskey-flavored-markdown', {
attrs: {
class: 'title'
}
}, token.title)];
}, genEl(token.children))];
}
case 'emoji': {
const customEmojis = (this.$root.getMetaSync() || { emojis: [] }).emojis || [];
return [createElement('mk-emoji', {
key: Math.random(),
attrs: {
emoji: token.emoji,
name: token.name
emoji: token.props.emoji,
name: token.props.name
},
props: {
customEmojis: this.customEmojis || customEmojis
@ -203,8 +238,9 @@ export default Vue.component('misskey-flavored-markdown', {
case 'math': {
//const MkFormula = () => import('./formula.vue').then(m => m.default);
return [createElement(MkFormula, {
key: Math.random(),
props: {
formula: token.formula
formula: token.props.formula
}
})];
}
@ -212,22 +248,22 @@ export default Vue.component('misskey-flavored-markdown', {
case 'search': {
//const MkGoogle = () => import('./google.vue').then(m => m.default);
return [createElement(MkGoogle, {
key: Math.random(),
props: {
q: token.query
q: token.props.query
}
})];
}
default: {
console.log('unknown ast type:', token.type);
console.log('unknown ast type:', token.name);
return [];
}
}
}));
// el.tag === 'br' のとき i !== 0 が保証されるため、短絡評価により els[i - 1] は配列外参照しない
const _els = els.filter((el, i) => !(el.tag === 'br' && ['div', 'pre'].includes(els[i - 1].tag)));
return createElement('span', _els);
// Parse ast to DOM
return createElement('span', genEl(ast));
}
});

View file

@ -0,0 +1,57 @@
<template>
<mfm v-bind="$attrs" class="havbbuyv"/>
</template>
<script lang="ts">
import Vue from 'vue';
import Mfm from './mfm';
export default Vue.extend({
components: {
Mfm
}
});
</script>
<style lang="stylus" scoped>
.havbbuyv
>>> .title
display block
margin-bottom 4px
padding 4px
font-size 90%
text-align center
background var(--mfmTitleBg)
border-radius 4px
>>> .code
margin 8px 0
>>> .quote
margin 8px
padding 6px 12px
color var(--mfmQuote)
border-left solid 3px var(--mfmQuoteLine)
>>> code
padding 4px 8px
margin 0 0.5em
font-size 80%
color #525252
background #f8f8f8
border-radius 2px
>>> pre > code
padding 16px
margin 0
>>> [data-is-me]:after
content "you"
padding 0 4px
margin-left 4px
font-size 80%
color var(--primaryForeground)
background var(--primary)
border-radius 4px
</style>

View file

@ -14,7 +14,7 @@
</div>
</header>
<div class="text">
<misskey-flavored-markdown v-if="note.text" :text="note.text" :customEmojis="note.emojis"/>
<misskey-flavored-markdown v-if="note.text" :text="note.text" :author="note.user" :custom-emojis="note.emojis"/>
</div>
</div>
</div>

View file

@ -9,7 +9,7 @@
<router-link :to="user | userPage" class="name">{{ user | userName }}</router-link>
<span class="username">@{{ user | acct }}</span>
<div class="description">
<misskey-flavored-markdown v-if="user.description" :text="user.description" :i="$store.state.i"/>
<misskey-flavored-markdown v-if="user.description" :text="user.description" :author="user" :i="$store.state.i"/>
</div>
</div>
</main>

View file

@ -46,7 +46,7 @@
<div class="text">
<span v-if="appearNote.isHidden" style="opacity: 0.5">{{ $t('private') }}</span>
<span v-if="appearNote.deletedAt" style="opacity: 0.5">{{ $t('deleted') }}</span>
<misskey-flavored-markdown v-if="appearNote.text" :text="appearNote.text" :i="$store.state.i" :customEmojis="appearNote.emojis" />
<misskey-flavored-markdown v-if="appearNote.text" :text="appearNote.text" :author="appearNote.user" :i="$store.state.i" :custom-emojis="appearNote.emojis" />
</div>
<div class="files" v-if="appearNote.files.length > 0">
<mk-media-list :media-list="appearNote.files" :raw="true"/>

View file

@ -27,7 +27,7 @@
<div class="text">
<span v-if="appearNote.isHidden" style="opacity: 0.5">{{ $t('private') }}</span>
<a class="reply" v-if="appearNote.reply"><fa icon="reply"/></a>
<misskey-flavored-markdown v-if="appearNote.text" :text="appearNote.text" :i="$store.state.i" :class="$style.text" :customEmojis="appearNote.emojis"/>
<misskey-flavored-markdown v-if="appearNote.text" :text="appearNote.text" :author="appearNote.user" :i="$store.state.i" :custom-emojis="appearNote.emojis"/>
<a class="rp" v-if="appearNote.renote">RN:</a>
</div>
<div class="files" v-if="appearNote.files.length > 0">
@ -223,24 +223,6 @@ export default Vue.extend({
overflow-wrap break-word
color var(--noteText)
>>> .title
display block
margin-bottom 4px
padding 4px
font-size 90%
text-align center
background var(--mfmTitleBg)
border-radius 4px
>>> .code
margin 8px 0
>>> .quote
margin 8px
padding 6px 12px
color var(--mfmQuote)
border-left solid 3px var(--mfmQuoteLine)
> .reply
margin-right 8px
color var(--text)
@ -322,28 +304,3 @@ export default Vue.extend({
opacity 0.7
</style>
<style lang="stylus" module>
.text
code
padding 4px 8px
margin 0 0.5em
font-size 80%
color #525252
background #f8f8f8
border-radius 2px
pre > code
padding 16px
margin 0
[data-is-me]:after
content "you"
padding 0 4px
margin-left 4px
font-size 80%
color var(--primaryForeground)
background var(--primary)
border-radius 4px
</style>

View file

@ -4,7 +4,7 @@
<span v-if="note.isHidden" style="opacity: 0.5">{{ $t('private') }}</span>
<span v-if="note.deletedAt" style="opacity: 0.5">{{ $t('deleted') }}</span>
<a class="reply" v-if="note.replyId"><fa icon="reply"/></a>
<misskey-flavored-markdown v-if="note.text" :text="note.text" :i="$store.state.i" :custom-emojis="note.emojis"/>
<misskey-flavored-markdown v-if="note.text" :text="note.text" :author="note.user" :i="$store.state.i" :custom-emojis="note.emojis"/>
<a class="rp" v-if="note.renoteId" :href="`/notes/${note.renoteId}`">RN: ...</a>
</div>
<details v-if="note.files.length > 0">

View file

@ -7,7 +7,7 @@
<router-link :to="user | userPage" class="name">{{ user | userName }}</router-link>
<span class="username">@{{ user | acct }}</span>
<div class="description">
<misskey-flavored-markdown v-if="user.description" :text="user.description" :i="$store.state.i"/>
<misskey-flavored-markdown v-if="user.description" :text="user.description" :author="user" :i="$store.state.i"/>
</div>
</div>
</div>

View file

@ -22,7 +22,7 @@
</header>
<div class="info">
<div class="description">
<misskey-flavored-markdown v-if="user.description" :text="user.description" :i="$store.state.i"/>
<misskey-flavored-markdown v-if="user.description" :text="user.description" :author="user" :i="$store.state.i"/>
</div>
<div class="counts">
<div>

View file

@ -14,7 +14,7 @@
<mk-avatar class="avatar" :user="user" :disable-preview="true"/>
<div class="body">
<div class="description">
<misskey-flavored-markdown v-if="user.description" :text="user.description" :i="$store.state.i"/>
<misskey-flavored-markdown v-if="user.description" :text="user.description" :author="user" :i="$store.state.i"/>
</div>
<div class="info">
<span class="location" v-if="user.host === null && user.profile.location"><fa icon="map-marker"/> {{ user.profile.location }}</span>

View file

@ -33,7 +33,7 @@
<div class="text">
<span v-if="appearNote.isHidden" style="opacity: 0.5">({{ $t('private') }})</span>
<span v-if="appearNote.deletedAt" style="opacity: 0.5">({{ $t('deleted') }})</span>
<misskey-flavored-markdown v-if="appearNote.text" :text="appearNote.text" :i="$store.state.i" :customEmojis="appearNote.emojis"/>
<misskey-flavored-markdown v-if="appearNote.text" :text="appearNote.text" :author="appearNote.user" :i="$store.state.i" :custom-emojis="appearNote.emojis"/>
</div>
<div class="files" v-if="appearNote.files.length > 0">
<mk-media-list :media-list="appearNote.files" :raw="true"/>

View file

@ -23,7 +23,7 @@
<div class="text">
<span v-if="appearNote.isHidden" style="opacity: 0.5">({{ $t('private') }})</span>
<a class="reply" v-if="appearNote.reply"><fa icon="reply"/></a>
<misskey-flavored-markdown v-if="appearNote.text" :text="appearNote.text" :i="$store.state.i" :class="$style.text" :custom-emojis="appearNote.emojis"/>
<misskey-flavored-markdown v-if="appearNote.text" :text="appearNote.text" :author="appearNote.user" :i="$store.state.i" :custom-emojis="appearNote.emojis"/>
<a class="rp" v-if="appearNote.renote != null">RN:</a>
</div>
<div class="files" v-if="appearNote.files.length > 0">
@ -188,24 +188,6 @@ export default Vue.extend({
overflow-wrap break-word
color var(--noteText)
>>> .title
display block
margin-bottom 4px
padding 4px
font-size 90%
text-align center
background var(--mfmTitleBg)
border-radius 4px
>>> .code
margin 8px 0
>>> .quote
margin 8px
padding 6px 12px
color var(--mfmQuote)
border-left solid 3px var(--mfmQuoteLine)
> .reply
margin-right 8px
color var(--noteText)
@ -215,15 +197,6 @@ export default Vue.extend({
font-style oblique
color var(--renoteText)
[data-is-me]:after
content "you"
padding 0 4px
margin-left 4px
font-size 80%
color var(--primaryForeground)
background var(--primary)
border-radius 4px
.mk-url-preview
margin-top 8px
@ -289,18 +262,3 @@ export default Vue.extend({
opacity 0.7
</style>
<style lang="stylus" module>
.text
code
padding 4px 8px
margin 0 0.5em
font-size 80%
color #525252
background #f8f8f8
border-radius 2px
pre > code
padding 16px
margin 0
</style>

View file

@ -4,7 +4,7 @@
<span v-if="note.isHidden" style="opacity: 0.5">({{ $t('private') }})</span>
<span v-if="note.deletedAt" style="opacity: 0.5">({{ $t('deleted') }})</span>
<a class="reply" v-if="note.replyId"><fa icon="reply"/></a>
<misskey-flavored-markdown v-if="note.text" :text="note.text" :i="$store.state.i" :custom-emojis="note.emojis"/>
<misskey-flavored-markdown v-if="note.text" :text="note.text" :author="note.user" :i="$store.state.i" :custom-emojis="note.emojis"/>
<a class="rp" v-if="note.renoteId">RN: ...</a>
</div>
<details v-if="note.files.length > 0">

View file

@ -20,7 +20,7 @@
<span class="followed" v-if="user.isFollowed">{{ $t('follows-you') }}</span>
</div>
<div class="description">
<misskey-flavored-markdown v-if="user.description" :text="user.description" :i="$store.state.i"/>
<misskey-flavored-markdown v-if="user.description" :text="user.description" :author="user" :i="$store.state.i"/>
</div>
<div class="info">
<p class="location" v-if="user.host === null && user.profile.location">

View file

@ -0,0 +1,23 @@
import VueRouter from 'vue-router';
// Style
import './style.styl';
import init from '../init';
import Index from './views/index.vue';
init(launch => {
document.title = 'Misskey';
// Init router
const router = new VueRouter({
mode: 'history',
base: '/test/',
routes: [
{ path: '/', component: Index },
]
});
// Launch the app
launch(router);
});

View file

@ -0,0 +1,6 @@
@import "../app"
@import "../reset"
html
height 100%
background var(--bg)

View file

@ -0,0 +1,34 @@
<template>
<main>
<ui-card>
<div slot="title">MFM Playground</div>
<section class="fit-top">
<ui-textarea v-model="mfm">
<span>MFM</span>
</ui-textarea>
<div>
<misskey-flavored-markdown :text="mfm" :i="$store.state.i"/>
</div>
</section>
</ui-card>
</main>
</template>
<script lang="ts">
import Vue from 'vue';
export default Vue.extend({
data() {
return {
mfm: '',
};
}
});
</script>
<style lang="stylus" scoped>
main
max-width 700px
margin 0 auto
</style>

View file

@ -1,127 +1,135 @@
const { lib: emojilib } = require('emojilib');
const jsdom = require('jsdom');
const { JSDOM } = jsdom;
import config from '../config';
import { INote } from '../models/note';
import { TextElement } from './parse';
import { Node } from './parser';
import { intersperse } from '../prelude/array';
const handlers: { [key: string]: (window: any, token: any, mentionedRemoteUsers: INote['mentionedRemoteUsers']) => void } = {
bold({ document }, { bold }) {
const b = document.createElement('b');
b.textContent = bold;
document.body.appendChild(b);
},
big({ document }, { big }) {
const b = document.createElement('strong');
b.textContent = big;
document.body.appendChild(b);
},
motion({ document }, { big }) {
const b = document.createElement('strong');
b.textContent = big;
document.body.appendChild(b);
},
code({ document }, { code }) {
const pre = document.createElement('pre');
const inner = document.createElement('code');
inner.innerHTML = code;
pre.appendChild(inner);
document.body.appendChild(pre);
},
emoji({ document }, { content, emoji }) {
const found = emojilib[emoji];
const node = document.createTextNode(found ? found.char : content);
document.body.appendChild(node);
},
hashtag({ document }, { hashtag }) {
const a = document.createElement('a');
a.href = `${config.url}/tags/${hashtag}`;
a.textContent = `#${hashtag}`;
a.setAttribute('rel', 'tag');
document.body.appendChild(a);
},
'inline-code'({ document }, { code }) {
const element = document.createElement('code');
element.textContent = code;
document.body.appendChild(element);
},
math({ document }, { formula }) {
const element = document.createElement('code');
element.textContent = formula;
document.body.appendChild(element);
},
link({ document }, { url, title }) {
const a = document.createElement('a');
a.href = url;
a.textContent = title;
document.body.appendChild(a);
},
mention({ document }, { content, username, host }, mentionedRemoteUsers) {
const a = document.createElement('a');
const remoteUserInfo = mentionedRemoteUsers.find(remoteUser => remoteUser.username === username && remoteUser.host === host);
a.href = remoteUserInfo ? remoteUserInfo.uri : `${config.url}/${content}`;
a.textContent = content;
document.body.appendChild(a);
},
quote({ document }, { quote }) {
const blockquote = document.createElement('blockquote');
blockquote.textContent = quote;
document.body.appendChild(blockquote);
},
title({ document }, { content }) {
const h1 = document.createElement('h1');
h1.textContent = content;
document.body.appendChild(h1);
},
text({ document }, { content }) {
const nodes = (content as string).split('\n').map(x => document.createTextNode(x));
for (const x of intersperse('br', nodes)) {
if (x === 'br') {
document.body.appendChild(document.createElement('br'));
} else {
document.body.appendChild(x);
}
}
},
url({ document }, { url }) {
const a = document.createElement('a');
a.href = url;
a.textContent = url;
document.body.appendChild(a);
},
search({ document }, { content, query }) {
const a = document.createElement('a');
a.href = `https://www.google.com/?#q=${query}`;
a.textContent = content;
document.body.appendChild(a);
}
};
export default (tokens: TextElement[], mentionedRemoteUsers: INote['mentionedRemoteUsers'] = []) => {
export default (tokens: Node[], mentionedRemoteUsers: INote['mentionedRemoteUsers'] = []) => {
if (tokens == null) {
return null;
}
const { window } = new JSDOM('');
for (const token of tokens) {
handlers[token.type](window, token, mentionedRemoteUsers);
const doc = window.document;
function dive(nodes: Node[]): any[] {
return nodes.map(n => handlers[n.name](n));
}
return `<p>${window.document.body.innerHTML}</p>`;
const handlers: { [key: string]: (token: Node) => any } = {
bold(token) {
const el = doc.createElement('b');
dive(token.children).forEach(child => el.appendChild(child));
return el;
},
big(token) {
const el = doc.createElement('strong');
dive(token.children).forEach(child => el.appendChild(child));
return el;
},
motion(token) {
const el = doc.createElement('i');
dive(token.children).forEach(child => el.appendChild(child));
return el;
},
blockCode(token) {
const pre = doc.createElement('pre');
const inner = doc.createElement('code');
inner.innerHTML = token.props.code;
pre.appendChild(inner);
return pre;
},
emoji(token) {
return doc.createTextNode(token.props.emoji ? token.props.emoji : `:${token.props.name}:`);
},
hashtag(token) {
const a = doc.createElement('a');
a.href = `${config.url}/tags/${token.props.hashtag}`;
a.textContent = `#${token.props.hashtag}`;
a.setAttribute('rel', 'tag');
return a;
},
inlineCode(token) {
const el = doc.createElement('code');
el.textContent = token.props.code;
return el;
},
math(token) {
const el = doc.createElement('code');
el.textContent = token.props.formula;
return el;
},
link(token) {
const a = doc.createElement('a');
a.href = token.props.url;
dive(token.children).forEach(child => a.appendChild(child));
return a;
},
mention(token) {
const a = doc.createElement('a');
const { username, host, acct } = token.props;
const remoteUserInfo = mentionedRemoteUsers.find(remoteUser => remoteUser.username === username && remoteUser.host === host);
a.href = remoteUserInfo ? remoteUserInfo.uri : `${config.url}/${acct}`;
a.textContent = acct;
return a;
},
quote(token) {
const el = doc.createElement('blockquote');
dive(token.children).forEach(child => el.appendChild(child));
return el;
},
title(token) {
const el = doc.createElement('h1');
dive(token.children).forEach(child => el.appendChild(child));
return el;
},
text(token) {
const el = doc.createElement('span');
const nodes = (token.props.text as string).split('\n').map(x => doc.createTextNode(x));
for (const x of intersperse('br', nodes)) {
if (x === 'br') {
el.appendChild(doc.createElement('br'));
} else {
el.appendChild(x);
}
}
return el;
},
url(token) {
const a = doc.createElement('a');
a.href = token.props.url;
a.textContent = token.props.url;
return a;
},
search(token) {
const a = doc.createElement('a');
a.href = `https://www.google.com/?#q=${token.props.query}`;
a.textContent = token.props.content;
return a;
}
};
dive(tokens).forEach(x => {
doc.body.appendChild(x);
});
return `<p>${doc.body.innerHTML}</p>`;
};

81
src/mfm/parse.ts Normal file
View file

@ -0,0 +1,81 @@
import parser, { Node } from './parser';
import * as A from '../prelude/array';
import * as S from '../prelude/string';
export default (source: string): Node[] => {
if (source == null || source == '') {
return null;
}
let nodes: Node[] = parser.root.tryParse(source);
const combineText = (es: Node[]): Node =>
({ name: 'text', props: { text: S.concat(es.map(e => e.props.text)) } });
const concatText = (nodes: Node[]): Node[] =>
A.concat(A.groupOn(x => x.name, nodes).map(es =>
es[0].name === 'text' ? [combineText(es)] : es
));
const concatTextRecursive = (es: Node[]): void =>
es.filter(x => x.children).forEach(x => {
x.children = concatText(x.children);
concatTextRecursive(x.children);
});
nodes = concatText(nodes);
concatTextRecursive(nodes);
function getBeforeTextNode(node: Node): Node {
if (node == null) return null;
if (node.name == 'text') return node;
if (node.children) return getBeforeTextNode(node.children[node.children.length - 1]);
return null;
}
function getAfterTextNode(node: Node): Node {
if (node == null) return null;
if (node.name == 'text') return node;
if (node.children) return getBeforeTextNode(node.children[0]);
return null;
}
function isBlockNode(node: Node): boolean {
return ['blockCode', 'quote', 'title'].includes(node.name);
}
/**
* ()
* @param nodes
*/
const removeNeedlessLineBreaks = (nodes: Node[]) => {
nodes.forEach((node, i) => {
if (node.children) removeNeedlessLineBreaks(node.children);
if (isBlockNode(node)) {
const before = getBeforeTextNode(nodes[i - 1]);
const after = getAfterTextNode(nodes[i + 1]);
if (before && before.props.text.endsWith('\n')) {
before.props.text = before.props.text.substring(0, before.props.text.length - 1);
}
if (after && after.props.text.startsWith('\n')) {
after.props.text = after.props.text.substring(1);
}
}
});
};
const removeEmptyTextNodes = (nodes: Node[]) => {
nodes.forEach(n => {
if (n.children) {
n.children = removeEmptyTextNodes(n.children);
}
});
return nodes.filter(n => !(n.name == 'text' && n.props.text == ''));
};
removeNeedlessLineBreaks(nodes);
nodes = removeEmptyTextNodes(nodes);
return nodes;
};

View file

@ -1,20 +0,0 @@
/**
* Big
*/
export type TextElementBig = {
type: 'big';
content: string;
big: string;
};
export default function(text: string) {
const match = text.match(/^\*\*\*(.+?)\*\*\*/);
if (!match) return null;
const big = match[0];
return {
type: 'big',
content: big,
big: match[1]
} as TextElementBig;
}

View file

@ -1,20 +0,0 @@
/**
* Bold
*/
export type TextElementBold = {
type: 'bold';
content: string;
bold: string;
};
export default function(text: string) {
const match = text.match(/^\*\*(.+?)\*\*/);
if (!match) return null;
const bold = match[0];
return {
type: 'bold',
content: bold,
bold: match[1]
} as TextElementBold;
}

View file

@ -1,24 +0,0 @@
/**
* Code (block)
*/
import genHtml from '../core/syntax-highlighter';
export type TextElementCode = {
type: 'code';
content: string;
code: string;
html: string;
};
export default function(text: string) {
const match = text.match(/^```([\s\S]+?)```/);
if (!match) return null;
const code = match[0];
return {
type: 'code',
content: code,
code: match[1],
html: genHtml(match[1].trim())
} as TextElementCode;
}

File diff suppressed because one or more lines are too long

View file

@ -1,33 +0,0 @@
/**
* Emoji
*/
import { emojiRegex } from "./emoji.regex";
export type TextElementEmoji = {
type: 'emoji';
content: string;
emoji?: string;
name?: string;
};
export default function(text: string) {
const name = text.match(/^:([a-zA-Z0-9+_-]+):/);
if (name) {
return {
type: 'emoji',
content: name[0],
name: name[1]
} as TextElementEmoji;
}
const unicode = text.match(emojiRegex);
if (unicode) {
const [content] = unicode;
return {
type: 'emoji',
content,
emoji: content
} as TextElementEmoji;
}
return null;
}

View file

@ -1,27 +0,0 @@
/**
* Hashtag
*/
export type TextElementHashtag = {
type: 'hashtag';
content: string;
hashtag: string;
};
export default function(text: string, before: string) {
const isBegin = before == '';
if (!(/^\s#[^\s\.,!\?#]+/.test(text) || (isBegin && /^#[^\s\.,!\?#]+/.test(text)))) return null;
const isHead = text.startsWith('#');
const hashtag = text.match(/^\s?#[^\s\.,!\?#]+/)[0];
const res: any[] = !isHead ? [{
type: 'text',
content: text[0]
}] : [];
res.push({
type: 'hashtag',
content: isHead ? hashtag : hashtag.substr(1),
hashtag: isHead ? hashtag.substr(1) : hashtag.substr(2)
});
return res as TextElementHashtag[];
}

View file

@ -1,25 +0,0 @@
/**
* Code (inline)
*/
import genHtml from '../core/syntax-highlighter';
export type TextElementInlineCode = {
type: 'inline-code';
content: string;
code: string;
html: string;
};
export default function(text: string) {
const match = text.match(/^`(.+?)`/);
if (!match) return null;
if (match[1].includes('´')) return null;
const code = match[0];
return {
type: 'inline-code',
content: code,
code: match[1],
html: genHtml(match[1])
} as TextElementInlineCode;
}

View file

@ -1,27 +0,0 @@
/**
* Link
*/
export type TextElementLink = {
type: 'link';
content: string;
title: string;
url: string;
silent: boolean;
};
export default function(text: string) {
const match = text.match(/^\??\[([^\[\]]+?)\]\((https?:\/\/[\w\/:%#@\$&\?!\(\)\[\]~\.=\+\-]+?)\)/);
if (!match) return null;
const silent = text.startsWith('?');
const link = match[0];
const title = match[1];
const url = match[2];
return {
type: 'link',
content: link,
title: title,
url: url,
silent: silent
} as TextElementLink;
}

View file

@ -1,20 +0,0 @@
/**
* Math
*/
export type TextElementMath = {
type: 'math';
content: string;
formula: string;
};
export default function(text: string) {
const match = text.match(/^\\\((.+?)\\\)/);
if (!match) return null;
const math = match[0];
return {
type: 'math',
content: math,
formula: match[1]
} as TextElementMath;
}

View file

@ -1,29 +0,0 @@
/**
* Mention
*/
import parseAcct from '../../../misc/acct/parse';
import { toUnicode } from 'punycode';
export type TextElementMention = {
type: 'mention';
content: string;
canonical: string;
username: string;
host: string;
};
export default function(text: string, before: string) {
const match = text.match(/^@[a-z0-9_]+(?:@[a-z0-9\.\-]+[a-z0-9])?/i);
if (!match) return null;
if (/[a-zA-Z0-9]$/.test(before)) return null;
const mention = match[0];
const { username, host } = parseAcct(mention.substr(1));
const canonical = host != null ? `@${username}@${toUnicode(host)}` : mention;
return {
type: 'mention',
content: mention,
canonical,
username,
host
} as TextElementMention;
}

View file

@ -1,20 +0,0 @@
/**
* Motion
*/
export type TextElementMotion = {
type: 'motion';
content: string;
motion: string;
};
export default function(text: string) {
const match = text.match(/^\(\(\((.+?)\)\)\)/) || text.match(/^<motion>(.+?)<\/motion>/);
if (!match) return null;
const motion = match[0];
return {
type: 'motion',
content: motion,
motion: match[1]
} as TextElementMotion;
}

View file

@ -1,30 +0,0 @@
/**
* Quoted text
*/
export type TextElementQuote = {
type: 'quote';
content: string;
quote: string;
};
export default function(text: string, before: string) {
const isBegin = before == '';
const match = text.match(/^"([\s\S]+?)\n"/) || text.match(/^\n>([\s\S]+?)(\n\n|$)/) ||
(isBegin ? text.match(/^>([\s\S]+?)(\n\n|$)/) : null);
if (!match) return null;
const quote = match[1]
.split('\n')
.map(line => line.replace(/^>+/g, '').trim())
.join('\n')
.trim();
return {
type: 'quote',
content: match[0],
quote: quote,
} as TextElementQuote;
}

View file

@ -1,19 +0,0 @@
/**
* Search
*/
export type TextElementSearch = {
type: 'search';
content: string;
query: string;
};
export default function(text: string) {
const match = text.match(/^(.+?)( | )(検索|\[検索\]|Search|\[Search\])(\n|$)/i);
if (!match) return null;
return {
type: 'search',
content: match[0],
query: match[1]
};
}

View file

@ -1,21 +0,0 @@
/**
* Title
*/
export type TextElementTitle = {
type: 'title';
content: string;
title: string;
};
export default function(text: string, before: string) {
const isBegin = before == '';
const match = isBegin ? text.match(/^(【|\[)(.+?)(】|])\n/) : text.match(/^\n(【|\[)(.+?)(】|])\n/);
if (!match) return null;
return {
type: 'title',
content: match[0],
title: match[2]
} as TextElementTitle;
}

View file

@ -1,23 +0,0 @@
/**
* URL
*/
export type TextElementUrl = {
type: 'url';
content: string;
url: string;
};
export default function(text: string, before: string) {
const match = text.match(/^https?:\/\/[\w\/:%#@\$&\?!\(\)\[\]~\.,=\+\-]+/);
if (!match) return null;
let url = match[0];
if (url.endsWith('.')) url = url.substr(0, url.lastIndexOf('.'));
if (url.endsWith(',')) url = url.substr(0, url.lastIndexOf(','));
if (url.endsWith(')') && before.endsWith('(')) url = url.substr(0, url.lastIndexOf(')'));
return {
type: 'url',
content: url,
url: url
} as TextElementUrl;
}

View file

@ -1,100 +0,0 @@
/**
* Misskey Text Analyzer
*/
import { TextElementBold } from './elements/bold';
import { TextElementBig } from './elements/big';
import { TextElementCode } from './elements/code';
import { TextElementEmoji } from './elements/emoji';
import { TextElementHashtag } from './elements/hashtag';
import { TextElementInlineCode } from './elements/inline-code';
import { TextElementMath } from './elements/math';
import { TextElementLink } from './elements/link';
import { TextElementMention } from './elements/mention';
import { TextElementQuote } from './elements/quote';
import { TextElementSearch } from './elements/search';
import { TextElementTitle } from './elements/title';
import { TextElementUrl } from './elements/url';
import { TextElementMotion } from './elements/motion';
import { groupOn } from '../../prelude/array';
import * as A from '../../prelude/array';
import * as S from '../../prelude/string';
const elements = [
require('./elements/big'),
require('./elements/bold'),
require('./elements/title'),
require('./elements/url'),
require('./elements/link'),
require('./elements/mention'),
require('./elements/hashtag'),
require('./elements/code'),
require('./elements/inline-code'),
require('./elements/math'),
require('./elements/quote'),
require('./elements/emoji'),
require('./elements/search'),
require('./elements/motion')
].map(element => element.default as TextElementProcessor);
export type TextElement = { type: 'text', content: string }
| TextElementBold
| TextElementBig
| TextElementCode
| TextElementEmoji
| TextElementHashtag
| TextElementInlineCode
| TextElementMath
| TextElementLink
| TextElementMention
| TextElementQuote
| TextElementSearch
| TextElementTitle
| TextElementUrl
| TextElementMotion;
export type TextElementProcessor = (text: string, before: string) => TextElement | TextElement[];
export default (source: string): TextElement[] => {
if (source == null || source == '') {
return null;
}
const tokens: TextElement[] = [];
function push(token: TextElement) {
if (token != null) {
tokens.push(token);
source = source.substr(token.content.length);
}
}
// パース
while (source != '') {
const parsed = elements.some(el => {
let _tokens = el(source, tokens.map(token => token.content).join(''));
if (_tokens) {
if (!Array.isArray(_tokens)) {
_tokens = [_tokens];
}
_tokens.forEach(push);
return true;
} else {
return false;
}
});
if (!parsed) {
push({
type: 'text',
content: source[0]
});
}
}
const combineText = (es: TextElement[]): TextElement =>
({ type: 'text', content: S.concat(es.map(e => e.content)) });
return A.concat(groupOn(x => x.type, tokens).map(es =>
es[0].type === 'text' ? [combineText(es)] : es
));
};

256
src/mfm/parser.ts Normal file

File diff suppressed because one or more lines are too long

View file

@ -1,4 +1,4 @@
import { capitalize, toUpperCase } from "../../../prelude/string";
import { capitalize, toUpperCase } from "../prelude/string";
function escape(text: string) {
return text
@ -308,7 +308,7 @@ const elements: Element[] = [
];
// specify lang is todo
export default (source: string, lang?: string) => {
export default (source: string, lang?: string): string => {
let code = source;
let html = '';

View file

@ -7,7 +7,6 @@ import DriveFile, { IDriveFile } from '../../../models/drive-file';
import Note, { INote } from '../../../models/note';
import User from '../../../models/user';
import toHtml from '../misc/get-note-html';
import parseMfm from '../../../mfm/parse';
import Emoji, { IEmoji } from '../../../models/emoji';
export default async function renderNote(note: INote, dive = true): Promise<any> {
@ -95,17 +94,6 @@ export default async function renderNote(note: INote, dive = true): Promise<any>
text += `\n\nRE: ${url}`;
}
// 省略されたメンションのホストを復元する
if (text != null && text != '') {
text = parseMfm(text).map(x => {
if (x.type == 'mention' && x.host == null) {
return `${x.content}@${config.host}`;
} else {
return x.content;
}
}).join('');
}
const content = toHtml(Object.assign({}, note, { text }));
const emojis = await getEmojis(note.emojis);

View file

@ -21,8 +21,6 @@ import Meta from '../../models/meta';
import config from '../../config';
import registerHashtag from '../register-hashtag';
import isQuote from '../../misc/is-quote';
import { TextElementMention } from '../../mfm/parse/elements/mention';
import { TextElementHashtag } from '../../mfm/parse/elements/hashtag';
import notesChart from '../../chart/notes';
import perUserNotesChart from '../../chart/per-user-notes';
@ -30,7 +28,7 @@ import { erase, unique } from '../../prelude/array';
import insertNoteUnread from './unread';
import registerInstance from '../register-instance';
import Instance from '../../models/instance';
import { TextElementEmoji } from '../../mfm/parse/elements/emoji';
import { Node } from '../../mfm/parser';
type NotificationType = 'reply' | 'renote' | 'quote' | 'mention';
@ -162,7 +160,7 @@ export default async (user: IUser, data: Option, silent = false) => new Promise<
const emojis = extractEmojis(tokens);
const mentionedUsers = data.apMentions || await extractMentionedUsers(tokens);
const mentionedUsers = data.apMentions || await extractMentionedUsers(user, tokens);
if (data.reply && !user._id.equals(data.reply.userId) && !mentionedUsers.some(u => u._id.equals(data.reply.userId))) {
mentionedUsers.push(await User.findOne({ _id: data.reply.userId }));
@ -460,21 +458,41 @@ async function insertNote(user: IUser, data: Option, tags: string[], emojis: str
}
function extractHashtags(tokens: ReturnType<typeof parse>): string[] {
const hashtags: string[] = [];
const extract = (tokens: Node[]) => {
tokens.filter(x => x.name === 'hashtag').forEach(x => {
if (x.props.hashtag.length <= 100) {
hashtags.push(x.props.hashtag);
}
});
tokens.filter(x => x.children).forEach(x => {
extract(x.children);
});
};
// Extract hashtags
const hashtags = tokens
.filter(t => t.type == 'hashtag')
.map(t => (t as TextElementHashtag).hashtag)
.filter(tag => tag.length <= 100);
extract(tokens);
return unique(hashtags);
}
function extractEmojis(tokens: ReturnType<typeof parse>): string[] {
const emojis: string[] = [];
const extract = (tokens: Node[]) => {
tokens.filter(x => x.name === 'emoji').forEach(x => {
if (x.props.name && x.props.name.length <= 100) {
emojis.push(x.props.name);
}
});
tokens.filter(x => x.children).forEach(x => {
extract(x.children);
});
};
// Extract emojis
const emojis = tokens
.filter(t => t.type == 'emoji' && t.name)
.map(t => (t as TextElementEmoji).name)
.filter(emoji => emoji.length <= 100);
extract(tokens);
return unique(emojis);
}
@ -638,16 +656,27 @@ function incNotesCount(user: IUser) {
}
}
async function extractMentionedUsers(tokens: ReturnType<typeof parse>): Promise<IUser[]> {
async function extractMentionedUsers(user: IUser, tokens: ReturnType<typeof parse>): Promise<IUser[]> {
if (tokens == null) return [];
const mentionTokens = tokens
.filter(t => t.type == 'mention') as TextElementMention[];
const mentions: any[] = [];
const extract = (tokens: Node[]) => {
tokens.filter(x => x.name === 'mention').forEach(x => {
mentions.push(x.props);
});
tokens.filter(x => x.children).forEach(x => {
extract(x.children);
});
};
// Extract hashtags
extract(tokens);
let mentionedUsers =
erase(null, await Promise.all(mentionTokens.map(async m => {
erase(null, await Promise.all(mentions.map(async m => {
try {
return await resolveUser(m.username, m.host);
return await resolveUser(m.username, m.host ? m.host : user.host);
} catch (e) {
return null;
}

View file

@ -6,102 +6,158 @@ import * as assert from 'assert';
import analyze from '../src/mfm/parse';
import toHtml from '../src/mfm/html';
import syntaxhighlighter from '../src/mfm/parse/core/syntax-highlighter';
function _node(name: string, children: any[], props: any) {
return children ? { name, children, props } : { name, props };
}
function node(name: string, props?: any) {
return _node(name, null, props);
}
function nodeWithChildren(name: string, children: any[], props?: any) {
return _node(name, children, props);
}
function text(text: string) {
return node('text', { text });
}
describe('Text', () => {
it('can be analyzed', () => {
const tokens = analyze('@himawari @hima_sub@namori.net お腹ペコい :cat: #yryr');
assert.deepEqual([
{ type: 'mention', content: '@himawari', canonical: '@himawari', username: 'himawari', host: null },
{ type: 'text', content: ' ' },
{ type: 'mention', content: '@hima_sub@namori.net', canonical: '@hima_sub@namori.net', username: 'hima_sub', host: 'namori.net' },
{ type: 'text', content: ' お腹ペコい ' },
{ type: 'emoji', content: ':cat:', name: 'cat' },
{ type: 'text', content: ' ' },
{ type: 'hashtag', content: '#yryr', hashtag: 'yryr' }
node('mention', { acct: '@himawari', canonical: '@himawari', username: 'himawari', host: null }),
text(' '),
node('mention', { acct: '@hima_sub@namori.net', canonical: '@hima_sub@namori.net', username: 'hima_sub', host: 'namori.net' }),
text(' お腹ペコい '),
node('emoji', { name: 'cat' }),
text(' '),
node('hashtag', { hashtag: 'yryr' }),
], tokens);
});
it('can be inverted', () => {
const text = '@himawari @hima_sub@namori.net お腹ペコい :cat: #yryr';
assert.equal(analyze(text).map(x => x.content).join(''), text);
});
describe('elements', () => {
it('bold', () => {
const tokens = analyze('**Strawberry** Pasta');
assert.deepEqual([
{ type: 'bold', content: '**Strawberry**', bold: 'Strawberry' },
{ type: 'text', content: ' Pasta' }
], tokens);
describe('bold', () => {
it('simple', () => {
const tokens = analyze('**foo**');
assert.deepEqual([
nodeWithChildren('bold', [
text('foo')
]),
], tokens);
});
it('with other texts', () => {
const tokens = analyze('bar**foo**bar');
assert.deepEqual([
text('bar'),
nodeWithChildren('bold', [
text('foo')
]),
text('bar'),
], tokens);
});
});
it('big', () => {
const tokens = analyze('***Strawberry*** Pasta');
assert.deepEqual([
{ type: 'big', content: '***Strawberry***', big: 'Strawberry' },
{ type: 'text', content: ' Pasta' }
nodeWithChildren('big', [
text('Strawberry')
]),
text(' Pasta'),
], tokens);
});
it('motion', () => {
const tokens1 = analyze('(((Strawberry))) Pasta');
assert.deepEqual([
{ type: 'motion', content: '(((Strawberry)))', motion: 'Strawberry' },
{ type: 'text', content: ' Pasta' }
], tokens1);
describe('motion', () => {
it('by triple brackets', () => {
const tokens = analyze('(((foo)))');
assert.deepEqual([
nodeWithChildren('motion', [
text('foo')
]),
], tokens);
});
const tokens2 = analyze('<motion>Strawberry</motion> Pasta');
assert.deepEqual([
{ type: 'motion', content: '<motion>Strawberry</motion>', motion: 'Strawberry' },
{ type: 'text', content: ' Pasta' }
], tokens2);
it('by triple brackets (with other texts)', () => {
const tokens = analyze('bar(((foo)))bar');
assert.deepEqual([
text('bar'),
nodeWithChildren('motion', [
text('foo')
]),
text('bar'),
], tokens);
});
it('by <motion> tag', () => {
const tokens = analyze('<motion>foo</motion>');
assert.deepEqual([
nodeWithChildren('motion', [
text('foo')
]),
], tokens);
});
it('by <motion> tag (with other texts)', () => {
const tokens = analyze('bar<motion>foo</motion>bar');
assert.deepEqual([
text('bar'),
nodeWithChildren('motion', [
text('foo')
]),
text('bar'),
], tokens);
});
});
describe('mention', () => {
it('local', () => {
const tokens = analyze('@himawari お腹ペコい');
const tokens = analyze('@himawari foo');
assert.deepEqual([
{ type: 'mention', content: '@himawari', canonical: '@himawari', username: 'himawari', host: null },
{ type: 'text', content: ' お腹ペコい' }
node('mention', { acct: '@himawari', canonical: '@himawari', username: 'himawari', host: null }),
text(' foo')
], tokens);
});
it('remote', () => {
const tokens = analyze('@hima_sub@namori.net お腹ペコい');
const tokens = analyze('@hima_sub@namori.net foo');
assert.deepEqual([
{ type: 'mention', content: '@hima_sub@namori.net', canonical: '@hima_sub@namori.net', username: 'hima_sub', host: 'namori.net' },
{ type: 'text', content: ' お腹ペコい' }
node('mention', { acct: '@hima_sub@namori.net', canonical: '@hima_sub@namori.net', username: 'hima_sub', host: 'namori.net' }),
text(' foo')
], tokens);
});
it('remote punycode', () => {
const tokens = analyze('@hima_sub@xn--q9j5bya.xn--zckzah お腹ペコい');
const tokens = analyze('@hima_sub@xn--q9j5bya.xn--zckzah foo');
assert.deepEqual([
{ type: 'mention', content: '@hima_sub@xn--q9j5bya.xn--zckzah', canonical: '@hima_sub@なもり.テスト', username: 'hima_sub', host: 'xn--q9j5bya.xn--zckzah' },
{ type: 'text', content: ' お腹ペコい' }
node('mention', { acct: '@hima_sub@xn--q9j5bya.xn--zckzah', canonical: '@hima_sub@なもり.テスト', username: 'hima_sub', host: 'xn--q9j5bya.xn--zckzah' }),
text(' foo')
], tokens);
});
it('ignore', () => {
const tokens = analyze('idolm@ster');
assert.deepEqual([
{ type: 'text', content: 'idolm@ster' }
text('idolm@ster')
], tokens);
const tokens2 = analyze('@a\n@b\n@c');
assert.deepEqual([
{ type: 'mention', content: '@a', canonical: '@a', username: 'a', host: null },
{ type: 'text', content: '\n' },
{ type: 'mention', content: '@b', canonical: '@b', username: 'b', host: null },
{ type: 'text', content: '\n' },
{ type: 'mention', content: '@c', canonical: '@c', username: 'c', host: null }
node('mention', { acct: '@a', canonical: '@a', username: 'a', host: null }),
text('\n'),
node('mention', { acct: '@b', canonical: '@b', username: 'b', host: null }),
text('\n'),
node('mention', { acct: '@c', canonical: '@c', username: 'c', host: null })
], tokens2);
const tokens3 = analyze('**x**@a');
assert.deepEqual([
{ type: 'bold', content: '**x**', bold: 'x' },
{ type: 'mention', content: '@a', canonical: '@a', username: 'a', host: null }
nodeWithChildren('bold', [
text('x')
]),
node('mention', { acct: '@a', canonical: '@a', username: 'a', host: null })
], tokens3);
});
});
@ -109,172 +165,294 @@ describe('Text', () => {
it('hashtag', () => {
const tokens1 = analyze('Strawberry Pasta #alice');
assert.deepEqual([
{ type: 'text', content: 'Strawberry Pasta ' },
{ type: 'hashtag', content: '#alice', hashtag: 'alice' }
text('Strawberry Pasta '),
node('hashtag', { hashtag: 'alice' })
], tokens1);
const tokens2 = analyze('Foo #bar, baz #piyo.');
assert.deepEqual([
{ type: 'text', content: 'Foo ' },
{ type: 'hashtag', content: '#bar', hashtag: 'bar' },
{ type: 'text', content: ', baz ' },
{ type: 'hashtag', content: '#piyo', hashtag: 'piyo' },
{ type: 'text', content: '.' }
text('Foo '),
node('hashtag', { hashtag: 'bar' }),
text(', baz '),
node('hashtag', { hashtag: 'piyo' }),
text('.'),
], tokens2);
const tokens3 = analyze('#Foo!');
assert.deepEqual([
{ type: 'hashtag', content: '#Foo', hashtag: 'Foo' },
{ type: 'text', content: '!' },
node('hashtag', { hashtag: 'Foo' }),
text('!'),
], tokens3);
});
it('quote', () => {
const tokens1 = analyze('> foo\nbar\nbaz');
assert.deepEqual([
{ type: 'quote', content: '> foo\nbar\nbaz', quote: 'foo\nbar\nbaz' }
], tokens1);
describe('quote', () => {
it('basic', () => {
const tokens1 = analyze('> foo');
assert.deepEqual([
nodeWithChildren('quote', [
text('foo')
])
], tokens1);
const tokens2 = analyze('before\n> foo\nbar\nbaz\n\nafter');
assert.deepEqual([
{ type: 'text', content: 'before' },
{ type: 'quote', content: '\n> foo\nbar\nbaz\n\n', quote: 'foo\nbar\nbaz' },
{ type: 'text', content: 'after' }
], tokens2);
const tokens2 = analyze('>foo');
assert.deepEqual([
nodeWithChildren('quote', [
text('foo')
])
], tokens2);
});
const tokens3 = analyze('piyo> foo\nbar\nbaz');
assert.deepEqual([
{ type: 'text', content: 'piyo> foo\nbar\nbaz' }
], tokens3);
it('series', () => {
const tokens = analyze('> foo\n\n> bar');
assert.deepEqual([
nodeWithChildren('quote', [
text('foo')
]),
nodeWithChildren('quote', [
text('bar')
]),
], tokens);
});
const tokens4 = analyze('> foo\n> bar\n> baz');
assert.deepEqual([
{ type: 'quote', content: '> foo\n> bar\n> baz', quote: 'foo\nbar\nbaz' }
], tokens4);
it('trailing line break', () => {
const tokens1 = analyze('> foo\n');
assert.deepEqual([
nodeWithChildren('quote', [
text('foo')
]),
], tokens1);
const tokens5 = analyze('"\nfoo\nbar\nbaz\n"');
assert.deepEqual([
{ type: 'quote', content: '"\nfoo\nbar\nbaz\n"', quote: 'foo\nbar\nbaz' }
], tokens5);
const tokens2 = analyze('> foo\n\n');
assert.deepEqual([
nodeWithChildren('quote', [
text('foo')
]),
text('\n')
], tokens2);
});
it('multiline', () => {
const tokens1 = analyze('>foo\n>bar');
assert.deepEqual([
nodeWithChildren('quote', [
text('foo\nbar')
])
], tokens1);
const tokens2 = analyze('> foo\n> bar');
assert.deepEqual([
nodeWithChildren('quote', [
text('foo\nbar')
])
], tokens2);
});
it('multiline with trailing line break', () => {
const tokens1 = analyze('> foo\n> bar\n');
assert.deepEqual([
nodeWithChildren('quote', [
text('foo\nbar')
]),
], tokens1);
const tokens2 = analyze('> foo\n> bar\n\n');
assert.deepEqual([
nodeWithChildren('quote', [
text('foo\nbar')
]),
text('\n')
], tokens2);
});
it('with before and after texts', () => {
const tokens = analyze('before\n> foo\nafter');
assert.deepEqual([
text('before'),
nodeWithChildren('quote', [
text('foo')
]),
text('after'),
], tokens);
});
it('require line break before ">"', () => {
const tokens = analyze('foo>bar');
assert.deepEqual([
text('foo>bar'),
], tokens);
});
it('nested', () => {
const tokens = analyze('>> foo\n> bar');
assert.deepEqual([
nodeWithChildren('quote', [
nodeWithChildren('quote', [
text('foo')
]),
text('bar')
])
], tokens);
});
it('trim line breaks', () => {
const tokens = analyze('foo\n\n>a\n>>b\n>>\n>>>\n>>>c\n>>>\n>d\n\n');
assert.deepEqual([
text('foo\n'),
nodeWithChildren('quote', [
text('a'),
nodeWithChildren('quote', [
text('b\n'),
nodeWithChildren('quote', [
text('\nc\n')
])
]),
text('d')
]),
text('\n'),
], tokens);
});
});
describe('url', () => {
it('simple', () => {
const tokens = analyze('https://example.com');
assert.deepEqual([{
type: 'url',
content: 'https://example.com',
url: 'https://example.com'
}], tokens);
assert.deepEqual([
node('url', { url: 'https://example.com' })
], tokens);
});
it('ignore trailing period', () => {
const tokens = analyze('https://example.com.');
assert.deepEqual([{
type: 'url',
content: 'https://example.com',
url: 'https://example.com'
}, {
type: 'text', content: '.'
}], tokens);
assert.deepEqual([
node('url', { url: 'https://example.com' }),
text('.')
], tokens);
});
it('with comma', () => {
const tokens = analyze('https://example.com/foo?bar=a,b');
assert.deepEqual([{
type: 'url',
content: 'https://example.com/foo?bar=a,b',
url: 'https://example.com/foo?bar=a,b'
}], tokens);
assert.deepEqual([
node('url', { url: 'https://example.com/foo?bar=a,b' })
], tokens);
});
it('ignore trailing comma', () => {
const tokens = analyze('https://example.com/foo, bar');
assert.deepEqual([{
type: 'url',
content: 'https://example.com/foo',
url: 'https://example.com/foo'
}, {
type: 'text', content: ', bar'
}], tokens);
assert.deepEqual([
node('url', { url: 'https://example.com/foo' }),
text(', bar')
], tokens);
});
it('with brackets', () => {
const tokens = analyze('https://example.com/foo(bar)');
assert.deepEqual([{
type: 'url',
content: 'https://example.com/foo(bar)',
url: 'https://example.com/foo(bar)'
}], tokens);
assert.deepEqual([
node('url', { url: 'https://example.com/foo(bar)' })
], tokens);
});
it('ignore parent brackets', () => {
const tokens = analyze('(https://example.com/foo)');
assert.deepEqual([{
type: 'text', content: '('
}, {
type: 'url',
content: 'https://example.com/foo',
url: 'https://example.com/foo'
}, {
type: 'text', content: ')'
}], tokens);
assert.deepEqual([
text('('),
node('url', { url: 'https://example.com/foo' }),
text(')')
], tokens);
});
it('ignore parent brackets with internal brackets', () => {
const tokens = analyze('(https://example.com/foo(bar))');
assert.deepEqual([{
type: 'text', content: '('
}, {
type: 'url',
content: 'https://example.com/foo(bar)',
url: 'https://example.com/foo(bar)'
}, {
type: 'text', content: ')'
}], tokens);
assert.deepEqual([
text('('),
node('url', { url: 'https://example.com/foo(bar)' }),
text(')')
], tokens);
});
});
it('link', () => {
const tokens = analyze('[ひまさく](https://himasaku.net)');
assert.deepEqual([{
type: 'link',
content: '[ひまさく](https://himasaku.net)',
title: 'ひまさく',
url: 'https://himasaku.net',
silent: false
}], tokens);
const tokens = analyze('[foo](https://example.com)');
assert.deepEqual([
nodeWithChildren('link', [
text('foo')
], { url: 'https://example.com', silent: false })
], tokens);
});
it('emoji', () => {
const tokens1 = analyze(':cat:');
assert.deepEqual([
{ type: 'emoji', content: ':cat:', name: 'cat' }
node('emoji', { name: 'cat' })
], tokens1);
const tokens2 = analyze(':cat::cat::cat:');
assert.deepEqual([
{ type: 'emoji', content: ':cat:', name: 'cat' },
{ type: 'emoji', content: ':cat:', name: 'cat' },
{ type: 'emoji', content: ':cat:', name: 'cat' }
node('emoji', { name: 'cat' }),
node('emoji', { name: 'cat' }),
node('emoji', { name: 'cat' })
], tokens2);
const tokens3 = analyze('🍎');
assert.deepEqual([
{ type: 'emoji', content: '🍎', emoji: '🍎' }
node('emoji', { emoji: '🍎' })
], tokens3);
});
it('block code', () => {
const tokens = analyze('```\nvar x = "Strawberry Pasta";\n```');
assert.equal(tokens[0].type, 'code');
assert.equal(tokens[0].content, '```\nvar x = "Strawberry Pasta";\n```');
describe('block code', () => {
it('simple', () => {
const tokens = analyze('```\nvar x = "Strawberry Pasta";\n```');
assert.deepEqual([
node('blockCode', { code: 'var x = "Strawberry Pasta";', lang: null })
], tokens);
});
it('can specify language', () => {
const tokens = analyze('``` json\n{ "x": 42 }\n```');
assert.deepEqual([
node('blockCode', { code: '{ "x": 42 }', lang: 'json' })
], tokens);
});
it('require line break before "```"', () => {
const tokens = analyze('before```\nfoo\n```');
assert.deepEqual([
text('before'),
node('inlineCode', { code: '`' }),
text('\nfoo\n'),
node('inlineCode', { code: '`' })
], tokens);
});
it('series', () => {
const tokens = analyze('```\nfoo\n```\n```\nbar\n```\n```\nbaz\n```');
assert.deepEqual([
node('blockCode', { code: 'foo', lang: null }),
node('blockCode', { code: 'bar', lang: null }),
node('blockCode', { code: 'baz', lang: null }),
], tokens);
});
it('ignore internal marker', () => {
const tokens = analyze('```\naaa```bbb\n```');
assert.deepEqual([
node('blockCode', { code: 'aaa```bbb', lang: null })
], tokens);
});
it('trim after line break', () => {
const tokens = analyze('```\nfoo\n```\nbar');
assert.deepEqual([
node('blockCode', { code: 'foo', lang: null }),
text('bar')
], tokens);
});
});
it('inline code', () => {
const tokens = analyze('`var x = "Strawberry Pasta";`');
assert.equal(tokens[0].type, 'inline-code');
assert.equal(tokens[0].content, '`var x = "Strawberry Pasta";`');
assert.deepEqual([
node('inlineCode', { code: 'var x = "Strawberry Pasta";' })
], tokens);
});
it('math', () => {
@ -282,82 +460,88 @@ describe('Text', () => {
const text = `\\(${fomula}\\)`;
const tokens = analyze(text);
assert.deepEqual([
{ type: 'math', content: text, formula: fomula }
node('math', { formula: fomula })
], tokens);
});
it('search', () => {
const tokens1 = analyze('a b c 検索');
assert.deepEqual([
{ type: 'search', content: 'a b c 検索', query: 'a b c' }
node('search', { content: 'a b c 検索', query: 'a b c' })
], tokens1);
const tokens2 = analyze('a b c Search');
assert.deepEqual([
{ type: 'search', content: 'a b c Search', query: 'a b c' }
node('search', { content: 'a b c Search', query: 'a b c' })
], tokens2);
const tokens3 = analyze('a b c search');
assert.deepEqual([
{ type: 'search', content: 'a b c search', query: 'a b c' }
node('search', { content: 'a b c search', query: 'a b c' })
], tokens3);
const tokens4 = analyze('a b c SEARCH');
assert.deepEqual([
{ type: 'search', content: 'a b c SEARCH', query: 'a b c' }
node('search', { content: 'a b c SEARCH', query: 'a b c' })
], tokens4);
});
it('title', () => {
const tokens1 = analyze('【yee】\nhaw');
assert.deepEqual(
{ type: 'title', content: '【yee】\n', title: 'yee' }
, tokens1[0]);
describe('title', () => {
it('simple', () => {
const tokens = analyze('【foo】');
assert.deepEqual([
nodeWithChildren('title', [
text('foo')
])
], tokens);
});
const tokens2 = analyze('[yee]\nhaw');
assert.deepEqual(
{ type: 'title', content: '[yee]\n', title: 'yee' }
, tokens2[0]);
it('require line break', () => {
const tokens = analyze('a【foo】');
assert.deepEqual([
text('a【foo】')
], tokens);
});
const tokens3 = analyze('a [a]\nb [b]\nc [c]');
assert.deepEqual(
{ type: 'text', content: 'a [a]\nb [b]\nc [c]' }
, tokens3[0]);
const tokens4 = analyze('foo\n【bar】\nbuzz');
assert.deepEqual([
{ type: 'text', content: 'foo' },
{ type: 'title', content: '\n【bar】\n', title: 'bar' },
{ type: 'text', content: 'buzz' },
], tokens4);
});
});
describe('syntax highlighting', () => {
it('comment', () => {
const html1 = syntaxhighlighter('// Strawberry pasta');
assert.equal(html1, '<span class="comment">// Strawberry pasta</span>');
const html2 = syntaxhighlighter('x // x\ny // y');
assert.equal(html2, 'x <span class="comment">// x\n</span>y <span class="comment">// y</span>');
});
it('regexp', () => {
const html = syntaxhighlighter('/.*/');
assert.equal(html, '<span class="regexp">/.*/</span>');
});
it('slash', () => {
const html = syntaxhighlighter('/');
assert.equal(html, '<span class="symbol">/</span>');
it('with before and after texts', () => {
const tokens = analyze('before\n【foo】\nafter');
assert.deepEqual([
text('before'),
nodeWithChildren('title', [
text('foo')
]),
text('after')
], tokens);
});
});
});
describe('toHtml', () => {
it('br', () => {
const input = 'foo\nbar\nbaz';
const output = '<p>foo<br>bar<br>baz</p>';
const output = '<p><span>foo<br>bar<br>baz</span></p>';
assert.equal(toHtml(analyze(input)), output);
});
});
it('code block with quote', () => {
const tokens = analyze('> foo\n```\nbar\n```');
assert.deepEqual([
nodeWithChildren('quote', [
text('foo')
]),
node('blockCode', { code: 'bar', lang: null })
], tokens);
});
it('quote between two code blocks', () => {
const tokens = analyze('```\nbefore\n```\n> foo\n```\nafter\n```');
assert.deepEqual([
node('blockCode', { code: 'before', lang: null }),
nodeWithChildren('quote', [
text('foo')
]),
node('blockCode', { code: 'after', lang: null })
], tokens);
});
});

View file

@ -38,6 +38,7 @@ module.exports = {
dev: './src/client/app/dev/script.ts',
auth: './src/client/app/auth/script.ts',
admin: './src/client/app/admin/script.ts',
test: './src/client/app/test/script.ts',
sw: './src/client/app/sw.js'
},
module: {