Store nodeinfo per federated instances (#5578)

* Store nodeinfo per federated instances

* Update fetch-nodeinfo.ts

* Update fetch-nodeinfo.ts

* update
This commit is contained in:
syuilo 2019-11-05 22:14:42 +09:00 committed by GitHub
parent 2f8992f98a
commit 77c9b90e6d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 173 additions and 10 deletions

View file

@ -0,0 +1,29 @@
import {MigrationInterface, QueryRunner} from "typeorm";
export class nodeinfo1572760203493 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<any> {
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "system"`, undefined);
await queryRunner.query(`ALTER TABLE "instance" ADD "softwareName" character varying(64) DEFAULT null`, undefined);
await queryRunner.query(`ALTER TABLE "instance" ADD "softwareVersion" character varying(64) DEFAULT null`, undefined);
await queryRunner.query(`ALTER TABLE "instance" ADD "openRegistrations" boolean DEFAULT null`, undefined);
await queryRunner.query(`ALTER TABLE "instance" ADD "name" character varying(256) DEFAULT null`, undefined);
await queryRunner.query(`ALTER TABLE "instance" ADD "description" character varying(4096) DEFAULT null`, undefined);
await queryRunner.query(`ALTER TABLE "instance" ADD "maintainerName" character varying(128) DEFAULT null`, undefined);
await queryRunner.query(`ALTER TABLE "instance" ADD "maintainerEmail" character varying(256) DEFAULT null`, undefined);
await queryRunner.query(`ALTER TABLE "instance" ADD "infoUpdatedAt" TIMESTAMP WITH TIME ZONE`, undefined);
}
public async down(queryRunner: QueryRunner): Promise<any> {
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "infoUpdatedAt"`, undefined);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "maintainerEmail"`, undefined);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "maintainerName"`, undefined);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "description"`, undefined);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "name"`, undefined);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "openRegistrations"`, undefined);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "softwareVersion"`, undefined);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "softwareName"`, undefined);
await queryRunner.query(`ALTER TABLE "instance" ADD "system" character varying(64)`, undefined);
}
}

View file

@ -20,3 +20,7 @@ const lock: (key: string, timeout?: number) => Promise<() => void>
export function getApLock(uri: string, timeout = 30 * 1000) { export function getApLock(uri: string, timeout = 30 * 1000) {
return lock(`ap-object:${uri}`, timeout); return lock(`ap-object:${uri}`, timeout);
} }
export function getNodeinfoLock(host: string, timeout = 30 * 1000) {
return lock(`nodeinfo:${host}`, timeout);
}

View file

@ -25,15 +25,6 @@ export class Instance {
}) })
public host: string; public host: string;
/**
* (MastodonとかMisskeyとかPleromaとか)
*/
@Column('varchar', {
length: 64, nullable: true,
comment: 'The system of the Instance.'
})
public system: string | null;
/** /**
* *
*/ */
@ -129,4 +120,45 @@ export class Instance {
default: false default: false
}) })
public isMarkedAsClosed: boolean; public isMarkedAsClosed: boolean;
@Column('varchar', {
length: 64, nullable: true, default: null,
comment: 'The software of the Instance.'
})
public softwareName: string | null;
@Column('varchar', {
length: 64, nullable: true, default: null,
})
public softwareVersion: string | null;
@Column('boolean', {
nullable: true, default: null,
})
public openRegistrations: boolean | null;
@Column('varchar', {
length: 256, nullable: true, default: null,
})
public name: string | null;
@Column('varchar', {
length: 4096, nullable: true, default: null,
})
public description: string | null;
@Column('varchar', {
length: 128, nullable: true, default: null,
})
public maintainerName: string | null;
@Column('varchar', {
length: 256, nullable: true, default: null,
})
public maintainerEmail: string | null;
@Column('timestamp with time zone', {
nullable: true,
})
public infoUpdatedAt: Date | null;
} }

View file

@ -4,6 +4,7 @@ import { registerOrFetchInstanceDoc } from '../../services/register-or-fetch-ins
import Logger from '../../services/logger'; import Logger from '../../services/logger';
import { Instances } from '../../models'; import { Instances } from '../../models';
import { instanceChart } from '../../services/chart'; import { instanceChart } from '../../services/chart';
import { fetchNodeinfo } from '../../services/fetch-nodeinfo';
const logger = new Logger('deliver'); const logger = new Logger('deliver');
@ -28,6 +29,8 @@ export default async (job: Bull.Job) => {
isNotResponding: false isNotResponding: false
}); });
fetchNodeinfo(i);
instanceChart.requestSent(i.host, true); instanceChart.requestSent(i.host, true);
}); });

View file

@ -13,6 +13,7 @@ import { fetchMeta } from '../../misc/fetch-meta';
import { toPuny } from '../../misc/convert-host'; import { toPuny } from '../../misc/convert-host';
import { validActor } from '../../remote/activitypub/type'; import { validActor } from '../../remote/activitypub/type';
import { ensure } from '../../prelude/ensure'; import { ensure } from '../../prelude/ensure';
import { fetchNodeinfo } from '../../services/fetch-nodeinfo';
const logger = new Logger('inbox'); const logger = new Logger('inbox');
@ -105,6 +106,8 @@ export default async (job: Bull.Job): Promise<void> => {
isNotResponding: false isNotResponding: false
}); });
fetchNodeinfo(i);
instanceChart.requestReceived(i.host); instanceChart.requestReceived(i.host);
}); });

View file

@ -27,6 +27,7 @@ import { validActor } from '../../../remote/activitypub/type';
import { getConnection } from 'typeorm'; import { getConnection } from 'typeorm';
import { ensure } from '../../../prelude/ensure'; import { ensure } from '../../../prelude/ensure';
import { toArray } from '../../../prelude/array'; import { toArray } from '../../../prelude/array';
import { fetchNodeinfo } from '../../../services/fetch-nodeinfo';
const logger = apLogger; const logger = apLogger;
@ -191,6 +192,7 @@ export async function createPerson(uri: string, resolver?: Resolver): Promise<Us
registerOrFetchInstanceDoc(host).then(i => { registerOrFetchInstanceDoc(host).then(i => {
Instances.increment({ id: i.id }, 'usersCount', 1); Instances.increment({ id: i.id }, 'usersCount', 1);
instanceChart.newUser(i.host); instanceChart.newUser(i.host);
fetchNodeinfo(i);
}); });
usersChart.update(user!, true); usersChart.update(user!, true);

View file

@ -0,0 +1,91 @@
import * as request from 'request-promise-native';
import { Instance } from '../models/entities/instance';
import { Instances } from '../models';
import config from '../config';
import { getNodeinfoLock } from '../misc/app-lock';
import Logger from '../services/logger';
export const logger = new Logger('nodeinfo', 'cyan');
export async function fetchNodeinfo(instance: Instance) {
const unlock = await getNodeinfoLock(instance.host);
const _instance = await Instances.findOne({ host: instance.host });
const now = Date.now();
if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24)) {
unlock();
return;
}
logger.info(`Fetching nodeinfo of ${instance.host} ...`);
try {
const wellknown = await request({
url: 'https://' + instance.host + '/.well-known/nodeinfo',
proxy: config.proxy,
timeout: 1000 * 10,
forever: true,
headers: {
'User-Agent': config.userAgent,
Accept: 'application/json, */*'
},
json: true
}).catch(e => {
if (e.statusCode === 404) {
throw 'No nodeinfo provided';
} else {
throw e.statusCode || e.message;
}
});
if (wellknown.links == null || !Array.isArray(wellknown.links)) {
throw 'No wellknown links';
}
const links = wellknown.links as any[];
const lnik1_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/1.0');
const lnik2_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.0');
const lnik2_1 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.1');
const link = lnik2_1 || lnik2_0 || lnik1_0;
if (link == null) {
throw 'No nodeinfo link provided';
}
const info = await request({
url: link.href,
proxy: config.proxy,
timeout: 1000 * 10,
forever: true,
headers: {
'User-Agent': config.userAgent,
Accept: 'application/json, */*'
},
json: true
}).catch(e => {
throw e.statusCode || e.message;
});
await Instances.update(instance.id, {
infoUpdatedAt: new Date(),
softwareName: info.software.name.toLowerCase(),
softwareVersion: info.software.version,
openRegistrations: info.openRegistrations,
name: info.metadata ? (info.metadata.nodeName || info.metadata.name || null) : null,
description: info.metadata ? (info.metadata.nodeDescription || info.metadata.description || null) : null,
maintainerName: info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.name || null) : null : null,
maintainerEmail: info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.email || null) : null : null,
});
logger.succ(`Successfuly fetched nodeinfo of ${instance.host}`);
} catch (e) {
logger.error(`Failed to fetch nodeinfo of ${instance.host}: ${e}`);
await Instances.update(instance.id, {
infoUpdatedAt: new Date(),
});
} finally {
unlock();
}
}

View file

@ -15,7 +15,6 @@ export async function registerOrFetchInstanceDoc(host: string): Promise<Instance
host, host,
caughtAt: new Date(), caughtAt: new Date(),
lastCommunicatedAt: new Date(), lastCommunicatedAt: new Date(),
system: null // TODO
}); });
federationChart.update(true); federationChart.update(true);