mirror of
https://git.joinsharkey.org/Sharkey/Sharkey.git
synced 2024-11-23 06:43:08 +02:00
Store nodeinfo per federated instances (#5578)
* Store nodeinfo per federated instances * Update fetch-nodeinfo.ts * Update fetch-nodeinfo.ts * update
This commit is contained in:
parent
2f8992f98a
commit
77c9b90e6d
8 changed files with 173 additions and 10 deletions
29
migration/1572760203493-nodeinfo.ts
Normal file
29
migration/1572760203493-nodeinfo.ts
Normal file
|
@ -0,0 +1,29 @@
|
||||||
|
import {MigrationInterface, QueryRunner} from "typeorm";
|
||||||
|
|
||||||
|
export class nodeinfo1572760203493 implements MigrationInterface {
|
||||||
|
|
||||||
|
public async up(queryRunner: QueryRunner): Promise<any> {
|
||||||
|
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "system"`, undefined);
|
||||||
|
await queryRunner.query(`ALTER TABLE "instance" ADD "softwareName" character varying(64) DEFAULT null`, undefined);
|
||||||
|
await queryRunner.query(`ALTER TABLE "instance" ADD "softwareVersion" character varying(64) DEFAULT null`, undefined);
|
||||||
|
await queryRunner.query(`ALTER TABLE "instance" ADD "openRegistrations" boolean DEFAULT null`, undefined);
|
||||||
|
await queryRunner.query(`ALTER TABLE "instance" ADD "name" character varying(256) DEFAULT null`, undefined);
|
||||||
|
await queryRunner.query(`ALTER TABLE "instance" ADD "description" character varying(4096) DEFAULT null`, undefined);
|
||||||
|
await queryRunner.query(`ALTER TABLE "instance" ADD "maintainerName" character varying(128) DEFAULT null`, undefined);
|
||||||
|
await queryRunner.query(`ALTER TABLE "instance" ADD "maintainerEmail" character varying(256) DEFAULT null`, undefined);
|
||||||
|
await queryRunner.query(`ALTER TABLE "instance" ADD "infoUpdatedAt" TIMESTAMP WITH TIME ZONE`, undefined);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async down(queryRunner: QueryRunner): Promise<any> {
|
||||||
|
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "infoUpdatedAt"`, undefined);
|
||||||
|
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "maintainerEmail"`, undefined);
|
||||||
|
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "maintainerName"`, undefined);
|
||||||
|
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "description"`, undefined);
|
||||||
|
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "name"`, undefined);
|
||||||
|
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "openRegistrations"`, undefined);
|
||||||
|
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "softwareVersion"`, undefined);
|
||||||
|
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "softwareName"`, undefined);
|
||||||
|
await queryRunner.query(`ALTER TABLE "instance" ADD "system" character varying(64)`, undefined);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -20,3 +20,7 @@ const lock: (key: string, timeout?: number) => Promise<() => void>
|
||||||
export function getApLock(uri: string, timeout = 30 * 1000) {
|
export function getApLock(uri: string, timeout = 30 * 1000) {
|
||||||
return lock(`ap-object:${uri}`, timeout);
|
return lock(`ap-object:${uri}`, timeout);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function getNodeinfoLock(host: string, timeout = 30 * 1000) {
|
||||||
|
return lock(`nodeinfo:${host}`, timeout);
|
||||||
|
}
|
||||||
|
|
|
@ -25,15 +25,6 @@ export class Instance {
|
||||||
})
|
})
|
||||||
public host: string;
|
public host: string;
|
||||||
|
|
||||||
/**
|
|
||||||
* インスタンスのシステム (MastodonとかMisskeyとかPleromaとか)
|
|
||||||
*/
|
|
||||||
@Column('varchar', {
|
|
||||||
length: 64, nullable: true,
|
|
||||||
comment: 'The system of the Instance.'
|
|
||||||
})
|
|
||||||
public system: string | null;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* インスタンスのユーザー数
|
* インスタンスのユーザー数
|
||||||
*/
|
*/
|
||||||
|
@ -129,4 +120,45 @@ export class Instance {
|
||||||
default: false
|
default: false
|
||||||
})
|
})
|
||||||
public isMarkedAsClosed: boolean;
|
public isMarkedAsClosed: boolean;
|
||||||
|
|
||||||
|
@Column('varchar', {
|
||||||
|
length: 64, nullable: true, default: null,
|
||||||
|
comment: 'The software of the Instance.'
|
||||||
|
})
|
||||||
|
public softwareName: string | null;
|
||||||
|
|
||||||
|
@Column('varchar', {
|
||||||
|
length: 64, nullable: true, default: null,
|
||||||
|
})
|
||||||
|
public softwareVersion: string | null;
|
||||||
|
|
||||||
|
@Column('boolean', {
|
||||||
|
nullable: true, default: null,
|
||||||
|
})
|
||||||
|
public openRegistrations: boolean | null;
|
||||||
|
|
||||||
|
@Column('varchar', {
|
||||||
|
length: 256, nullable: true, default: null,
|
||||||
|
})
|
||||||
|
public name: string | null;
|
||||||
|
|
||||||
|
@Column('varchar', {
|
||||||
|
length: 4096, nullable: true, default: null,
|
||||||
|
})
|
||||||
|
public description: string | null;
|
||||||
|
|
||||||
|
@Column('varchar', {
|
||||||
|
length: 128, nullable: true, default: null,
|
||||||
|
})
|
||||||
|
public maintainerName: string | null;
|
||||||
|
|
||||||
|
@Column('varchar', {
|
||||||
|
length: 256, nullable: true, default: null,
|
||||||
|
})
|
||||||
|
public maintainerEmail: string | null;
|
||||||
|
|
||||||
|
@Column('timestamp with time zone', {
|
||||||
|
nullable: true,
|
||||||
|
})
|
||||||
|
public infoUpdatedAt: Date | null;
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,6 +4,7 @@ import { registerOrFetchInstanceDoc } from '../../services/register-or-fetch-ins
|
||||||
import Logger from '../../services/logger';
|
import Logger from '../../services/logger';
|
||||||
import { Instances } from '../../models';
|
import { Instances } from '../../models';
|
||||||
import { instanceChart } from '../../services/chart';
|
import { instanceChart } from '../../services/chart';
|
||||||
|
import { fetchNodeinfo } from '../../services/fetch-nodeinfo';
|
||||||
|
|
||||||
const logger = new Logger('deliver');
|
const logger = new Logger('deliver');
|
||||||
|
|
||||||
|
@ -28,6 +29,8 @@ export default async (job: Bull.Job) => {
|
||||||
isNotResponding: false
|
isNotResponding: false
|
||||||
});
|
});
|
||||||
|
|
||||||
|
fetchNodeinfo(i);
|
||||||
|
|
||||||
instanceChart.requestSent(i.host, true);
|
instanceChart.requestSent(i.host, true);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
@ -13,6 +13,7 @@ import { fetchMeta } from '../../misc/fetch-meta';
|
||||||
import { toPuny } from '../../misc/convert-host';
|
import { toPuny } from '../../misc/convert-host';
|
||||||
import { validActor } from '../../remote/activitypub/type';
|
import { validActor } from '../../remote/activitypub/type';
|
||||||
import { ensure } from '../../prelude/ensure';
|
import { ensure } from '../../prelude/ensure';
|
||||||
|
import { fetchNodeinfo } from '../../services/fetch-nodeinfo';
|
||||||
|
|
||||||
const logger = new Logger('inbox');
|
const logger = new Logger('inbox');
|
||||||
|
|
||||||
|
@ -105,6 +106,8 @@ export default async (job: Bull.Job): Promise<void> => {
|
||||||
isNotResponding: false
|
isNotResponding: false
|
||||||
});
|
});
|
||||||
|
|
||||||
|
fetchNodeinfo(i);
|
||||||
|
|
||||||
instanceChart.requestReceived(i.host);
|
instanceChart.requestReceived(i.host);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
@ -27,6 +27,7 @@ import { validActor } from '../../../remote/activitypub/type';
|
||||||
import { getConnection } from 'typeorm';
|
import { getConnection } from 'typeorm';
|
||||||
import { ensure } from '../../../prelude/ensure';
|
import { ensure } from '../../../prelude/ensure';
|
||||||
import { toArray } from '../../../prelude/array';
|
import { toArray } from '../../../prelude/array';
|
||||||
|
import { fetchNodeinfo } from '../../../services/fetch-nodeinfo';
|
||||||
|
|
||||||
const logger = apLogger;
|
const logger = apLogger;
|
||||||
|
|
||||||
|
@ -191,6 +192,7 @@ export async function createPerson(uri: string, resolver?: Resolver): Promise<Us
|
||||||
registerOrFetchInstanceDoc(host).then(i => {
|
registerOrFetchInstanceDoc(host).then(i => {
|
||||||
Instances.increment({ id: i.id }, 'usersCount', 1);
|
Instances.increment({ id: i.id }, 'usersCount', 1);
|
||||||
instanceChart.newUser(i.host);
|
instanceChart.newUser(i.host);
|
||||||
|
fetchNodeinfo(i);
|
||||||
});
|
});
|
||||||
|
|
||||||
usersChart.update(user!, true);
|
usersChart.update(user!, true);
|
||||||
|
|
91
src/services/fetch-nodeinfo.ts
Normal file
91
src/services/fetch-nodeinfo.ts
Normal file
|
@ -0,0 +1,91 @@
|
||||||
|
import * as request from 'request-promise-native';
|
||||||
|
import { Instance } from '../models/entities/instance';
|
||||||
|
import { Instances } from '../models';
|
||||||
|
import config from '../config';
|
||||||
|
import { getNodeinfoLock } from '../misc/app-lock';
|
||||||
|
import Logger from '../services/logger';
|
||||||
|
|
||||||
|
export const logger = new Logger('nodeinfo', 'cyan');
|
||||||
|
|
||||||
|
export async function fetchNodeinfo(instance: Instance) {
|
||||||
|
const unlock = await getNodeinfoLock(instance.host);
|
||||||
|
|
||||||
|
const _instance = await Instances.findOne({ host: instance.host });
|
||||||
|
const now = Date.now();
|
||||||
|
if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24)) {
|
||||||
|
unlock();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`Fetching nodeinfo of ${instance.host} ...`);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const wellknown = await request({
|
||||||
|
url: 'https://' + instance.host + '/.well-known/nodeinfo',
|
||||||
|
proxy: config.proxy,
|
||||||
|
timeout: 1000 * 10,
|
||||||
|
forever: true,
|
||||||
|
headers: {
|
||||||
|
'User-Agent': config.userAgent,
|
||||||
|
Accept: 'application/json, */*'
|
||||||
|
},
|
||||||
|
json: true
|
||||||
|
}).catch(e => {
|
||||||
|
if (e.statusCode === 404) {
|
||||||
|
throw 'No nodeinfo provided';
|
||||||
|
} else {
|
||||||
|
throw e.statusCode || e.message;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
if (wellknown.links == null || !Array.isArray(wellknown.links)) {
|
||||||
|
throw 'No wellknown links';
|
||||||
|
}
|
||||||
|
|
||||||
|
const links = wellknown.links as any[];
|
||||||
|
|
||||||
|
const lnik1_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/1.0');
|
||||||
|
const lnik2_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.0');
|
||||||
|
const lnik2_1 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.1');
|
||||||
|
const link = lnik2_1 || lnik2_0 || lnik1_0;
|
||||||
|
|
||||||
|
if (link == null) {
|
||||||
|
throw 'No nodeinfo link provided';
|
||||||
|
}
|
||||||
|
|
||||||
|
const info = await request({
|
||||||
|
url: link.href,
|
||||||
|
proxy: config.proxy,
|
||||||
|
timeout: 1000 * 10,
|
||||||
|
forever: true,
|
||||||
|
headers: {
|
||||||
|
'User-Agent': config.userAgent,
|
||||||
|
Accept: 'application/json, */*'
|
||||||
|
},
|
||||||
|
json: true
|
||||||
|
}).catch(e => {
|
||||||
|
throw e.statusCode || e.message;
|
||||||
|
});
|
||||||
|
|
||||||
|
await Instances.update(instance.id, {
|
||||||
|
infoUpdatedAt: new Date(),
|
||||||
|
softwareName: info.software.name.toLowerCase(),
|
||||||
|
softwareVersion: info.software.version,
|
||||||
|
openRegistrations: info.openRegistrations,
|
||||||
|
name: info.metadata ? (info.metadata.nodeName || info.metadata.name || null) : null,
|
||||||
|
description: info.metadata ? (info.metadata.nodeDescription || info.metadata.description || null) : null,
|
||||||
|
maintainerName: info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.name || null) : null : null,
|
||||||
|
maintainerEmail: info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.email || null) : null : null,
|
||||||
|
});
|
||||||
|
|
||||||
|
logger.succ(`Successfuly fetched nodeinfo of ${instance.host}`);
|
||||||
|
} catch (e) {
|
||||||
|
logger.error(`Failed to fetch nodeinfo of ${instance.host}: ${e}`);
|
||||||
|
|
||||||
|
await Instances.update(instance.id, {
|
||||||
|
infoUpdatedAt: new Date(),
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
unlock();
|
||||||
|
}
|
||||||
|
}
|
|
@ -15,7 +15,6 @@ export async function registerOrFetchInstanceDoc(host: string): Promise<Instance
|
||||||
host,
|
host,
|
||||||
caughtAt: new Date(),
|
caughtAt: new Date(),
|
||||||
lastCommunicatedAt: new Date(),
|
lastCommunicatedAt: new Date(),
|
||||||
system: null // TODO
|
|
||||||
});
|
});
|
||||||
|
|
||||||
federationChart.update(true);
|
federationChart.update(true);
|
||||||
|
|
Loading…
Reference in a new issue