mirror of
https://git.joinsharkey.org/Sharkey/Sharkey.git
synced 2024-11-30 03:03:08 +02:00
feat(server): Fetch icon url of an instance (#6591)
* feat(server): Fetch icon url of an instance Resolve #6589 * chore: Rename the function
This commit is contained in:
parent
cf9266eab9
commit
f1ef85b636
9 changed files with 183 additions and 80 deletions
14
migration/1595676934834-instance-icon-url.ts
Normal file
14
migration/1595676934834-instance-icon-url.ts
Normal file
|
@ -0,0 +1,14 @@
|
||||||
|
import {MigrationInterface, QueryRunner} from "typeorm";
|
||||||
|
|
||||||
|
export class instanceIconUrl1595676934834 implements MigrationInterface {
|
||||||
|
name = 'instanceIconUrl1595676934834'
|
||||||
|
|
||||||
|
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||||
|
await queryRunner.query(`ALTER TABLE "instance" ADD "iconUrl" character varying(256) DEFAULT null`);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||||
|
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "iconUrl"`);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -21,8 +21,8 @@ export function getApLock(uri: string, timeout = 30 * 1000) {
|
||||||
return lock(`ap-object:${uri}`, timeout);
|
return lock(`ap-object:${uri}`, timeout);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getNodeinfoLock(host: string, timeout = 30 * 1000) {
|
export function getFetchInstanceMetadataLock(host: string, timeout = 30 * 1000) {
|
||||||
return lock(`nodeinfo:${host}`, timeout);
|
return lock(`instance:${host}`, timeout);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getChartInsertLock(lockKey: string, timeout = 30 * 1000) {
|
export function getChartInsertLock(lockKey: string, timeout = 30 * 1000) {
|
||||||
|
|
|
@ -27,6 +27,27 @@ export async function getJson(url: string, accept = 'application/json, */*', tim
|
||||||
return await res.json();
|
return await res.json();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: HeadersInit) {
|
||||||
|
const res = await fetch(url, {
|
||||||
|
headers: Object.assign({
|
||||||
|
'User-Agent': config.userAgent,
|
||||||
|
Accept: accept
|
||||||
|
}, headers || {}),
|
||||||
|
timeout,
|
||||||
|
agent: getAgentByUrl,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!res.ok) {
|
||||||
|
throw {
|
||||||
|
name: `StatusError`,
|
||||||
|
statusCode: res.status,
|
||||||
|
message: `${res.status} ${res.statusText}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return await res.text();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get http non-proxy agent
|
* Get http non-proxy agent
|
||||||
*/
|
*/
|
||||||
|
|
|
@ -158,6 +158,11 @@ export class Instance {
|
||||||
})
|
})
|
||||||
public maintainerEmail: string | null;
|
public maintainerEmail: string | null;
|
||||||
|
|
||||||
|
@Column('varchar', {
|
||||||
|
length: 256, nullable: true, default: null,
|
||||||
|
})
|
||||||
|
public iconUrl: string | null;
|
||||||
|
|
||||||
@Column('timestamp with time zone', {
|
@Column('timestamp with time zone', {
|
||||||
nullable: true,
|
nullable: true,
|
||||||
})
|
})
|
||||||
|
|
|
@ -4,7 +4,7 @@ import { registerOrFetchInstanceDoc } from '../../services/register-or-fetch-ins
|
||||||
import Logger from '../../services/logger';
|
import Logger from '../../services/logger';
|
||||||
import { Instances } from '../../models';
|
import { Instances } from '../../models';
|
||||||
import { instanceChart } from '../../services/chart';
|
import { instanceChart } from '../../services/chart';
|
||||||
import { fetchNodeinfo } from '../../services/fetch-nodeinfo';
|
import { fetchInstanceMetadata } from '../../services/fetch-instance-metadata';
|
||||||
import { fetchMeta } from '../../misc/fetch-meta';
|
import { fetchMeta } from '../../misc/fetch-meta';
|
||||||
import { toPuny } from '../../misc/convert-host';
|
import { toPuny } from '../../misc/convert-host';
|
||||||
|
|
||||||
|
@ -48,7 +48,7 @@ export default async (job: Bull.Job) => {
|
||||||
isNotResponding: false
|
isNotResponding: false
|
||||||
});
|
});
|
||||||
|
|
||||||
fetchNodeinfo(i);
|
fetchInstanceMetadata(i);
|
||||||
|
|
||||||
instanceChart.requestSent(i.host, true);
|
instanceChart.requestSent(i.host, true);
|
||||||
});
|
});
|
||||||
|
|
|
@ -8,7 +8,7 @@ import { instanceChart } from '../../services/chart';
|
||||||
import { fetchMeta } from '../../misc/fetch-meta';
|
import { fetchMeta } from '../../misc/fetch-meta';
|
||||||
import { toPuny, extractDbHost } from '../../misc/convert-host';
|
import { toPuny, extractDbHost } from '../../misc/convert-host';
|
||||||
import { getApId } from '../../remote/activitypub/type';
|
import { getApId } from '../../remote/activitypub/type';
|
||||||
import { fetchNodeinfo } from '../../services/fetch-nodeinfo';
|
import { fetchInstanceMetadata } from '../../services/fetch-instance-metadata';
|
||||||
import { InboxJobData } from '..';
|
import { InboxJobData } from '..';
|
||||||
import DbResolver from '../../remote/activitypub/db-resolver';
|
import DbResolver from '../../remote/activitypub/db-resolver';
|
||||||
import { resolvePerson } from '../../remote/activitypub/models/person';
|
import { resolvePerson } from '../../remote/activitypub/models/person';
|
||||||
|
@ -126,7 +126,7 @@ export default async (job: Bull.Job<InboxJobData>): Promise<string> => {
|
||||||
isNotResponding: false
|
isNotResponding: false
|
||||||
});
|
});
|
||||||
|
|
||||||
fetchNodeinfo(i);
|
fetchInstanceMetadata(i);
|
||||||
|
|
||||||
instanceChart.requestReceived(i.host);
|
instanceChart.requestReceived(i.host);
|
||||||
});
|
});
|
||||||
|
|
|
@ -26,7 +26,7 @@ import { validActor } from '../../../remote/activitypub/type';
|
||||||
import { getConnection } from 'typeorm';
|
import { getConnection } from 'typeorm';
|
||||||
import { ensure } from '../../../prelude/ensure';
|
import { ensure } from '../../../prelude/ensure';
|
||||||
import { toArray } from '../../../prelude/array';
|
import { toArray } from '../../../prelude/array';
|
||||||
import { fetchNodeinfo } from '../../../services/fetch-nodeinfo';
|
import { fetchInstanceMetadata } from '../../../services/fetch-instance-metadata';
|
||||||
|
|
||||||
const logger = apLogger;
|
const logger = apLogger;
|
||||||
|
|
||||||
|
@ -204,7 +204,7 @@ export async function createPerson(uri: string, resolver?: Resolver): Promise<Us
|
||||||
registerOrFetchInstanceDoc(host).then(i => {
|
registerOrFetchInstanceDoc(host).then(i => {
|
||||||
Instances.increment({ id: i.id }, 'usersCount', 1);
|
Instances.increment({ id: i.id }, 'usersCount', 1);
|
||||||
instanceChart.newUser(i.host);
|
instanceChart.newUser(i.host);
|
||||||
fetchNodeinfo(i);
|
fetchInstanceMetadata(i);
|
||||||
});
|
});
|
||||||
|
|
||||||
usersChart.update(user!, true);
|
usersChart.update(user!, true);
|
||||||
|
|
135
src/services/fetch-instance-metadata.ts
Normal file
135
src/services/fetch-instance-metadata.ts
Normal file
|
@ -0,0 +1,135 @@
|
||||||
|
import { JSDOM } from 'jsdom';
|
||||||
|
import fetch from 'node-fetch';
|
||||||
|
import { getJson, getHtml, getAgentByUrl } from '../misc/fetch';
|
||||||
|
import { Instance } from '../models/entities/instance';
|
||||||
|
import { Instances } from '../models';
|
||||||
|
import { getFetchInstanceMetadataLock } from '../misc/app-lock';
|
||||||
|
import Logger from './logger';
|
||||||
|
import { URL } from 'url';
|
||||||
|
|
||||||
|
const logger = new Logger('metadata', 'cyan');
|
||||||
|
|
||||||
|
export async function fetchInstanceMetadata(instance: Instance): Promise<void> {
|
||||||
|
const unlock = await getFetchInstanceMetadataLock(instance.host);
|
||||||
|
|
||||||
|
const _instance = await Instances.findOne({ host: instance.host });
|
||||||
|
const now = Date.now();
|
||||||
|
if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24)) {
|
||||||
|
unlock();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`Fetching metadata of ${instance.host} ...`);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const [info, icon] = await Promise.all([
|
||||||
|
fetchNodeinfo(instance).catch(() => null),
|
||||||
|
fetchIconUrl(instance).catch(() => null),
|
||||||
|
]);
|
||||||
|
|
||||||
|
logger.succ(`Successfuly fetched metadata of ${instance.host}`);
|
||||||
|
|
||||||
|
const updates = {
|
||||||
|
infoUpdatedAt: new Date(),
|
||||||
|
} as Record<string, any>;
|
||||||
|
|
||||||
|
if (info) {
|
||||||
|
updates.softwareName = info.software.name.toLowerCase();
|
||||||
|
updates.softwareVersion = info.software.version;
|
||||||
|
updates.openRegistrations = info.openRegistrations;
|
||||||
|
updates.name = info.metadata ? (info.metadata.nodeName || info.metadata.name || null) : null;
|
||||||
|
updates.description = info.metadata ? (info.metadata.nodeDescription || info.metadata.description || null) : null;
|
||||||
|
updates.maintainerName = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.name || null) : null : null;
|
||||||
|
updates.maintainerEmail = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.email || null) : null : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (icon) {
|
||||||
|
updates.iconUrl = icon;
|
||||||
|
}
|
||||||
|
|
||||||
|
await Instances.update(instance.id, updates);
|
||||||
|
|
||||||
|
logger.succ(`Successfuly updated metadata of ${instance.host}`);
|
||||||
|
} catch (e) {
|
||||||
|
logger.error(`Failed to update metadata of ${instance.host}: ${e}`);
|
||||||
|
} finally {
|
||||||
|
unlock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchNodeinfo(instance: Instance): Promise<Record<string, any>> {
|
||||||
|
logger.info(`Fetching nodeinfo of ${instance.host} ...`);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const wellknown = await getJson('https://' + instance.host + '/.well-known/nodeinfo')
|
||||||
|
.catch(e => {
|
||||||
|
if (e.statusCode === 404) {
|
||||||
|
throw 'No nodeinfo provided';
|
||||||
|
} else {
|
||||||
|
throw e.statusCode || e.message;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
if (wellknown.links == null || !Array.isArray(wellknown.links)) {
|
||||||
|
throw 'No wellknown links';
|
||||||
|
}
|
||||||
|
|
||||||
|
const links = wellknown.links as any[];
|
||||||
|
|
||||||
|
const lnik1_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/1.0');
|
||||||
|
const lnik2_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.0');
|
||||||
|
const lnik2_1 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.1');
|
||||||
|
const link = lnik2_1 || lnik2_0 || lnik1_0;
|
||||||
|
|
||||||
|
if (link == null) {
|
||||||
|
throw 'No nodeinfo link provided';
|
||||||
|
}
|
||||||
|
|
||||||
|
const info = await getJson(link.href)
|
||||||
|
.catch(e => {
|
||||||
|
throw e.statusCode || e.message;
|
||||||
|
});
|
||||||
|
|
||||||
|
logger.succ(`Successfuly fetched nodeinfo of ${instance.host}`);
|
||||||
|
|
||||||
|
return info;
|
||||||
|
} catch (e) {
|
||||||
|
logger.error(`Failed to fetch nodeinfo of ${instance.host}: ${e}`);
|
||||||
|
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchIconUrl(instance: Instance): Promise<string | null> {
|
||||||
|
logger.info(`Fetching icon URL of ${instance.host} ...`);
|
||||||
|
|
||||||
|
const url = 'https://' + instance.host;
|
||||||
|
|
||||||
|
const html = await getHtml(url);
|
||||||
|
|
||||||
|
const { window } = new JSDOM(html);
|
||||||
|
const doc = window.document;
|
||||||
|
|
||||||
|
const hrefAppleTouchIconPrecomposed = doc.querySelector('link[rel="apple-touch-icon-precomposed"]')?.getAttribute('href');
|
||||||
|
const hrefAppleTouchIcon = doc.querySelector('link[rel="apple-touch-icon"]')?.getAttribute('href');
|
||||||
|
const hrefIcon = doc.querySelector('link[rel="icon"]')?.getAttribute('href');
|
||||||
|
|
||||||
|
const href = hrefAppleTouchIconPrecomposed || hrefAppleTouchIcon || hrefIcon;
|
||||||
|
|
||||||
|
if (href) {
|
||||||
|
return (new URL(href, url)).href;
|
||||||
|
}
|
||||||
|
|
||||||
|
const faviconUrl = url + '/favicon.ico';
|
||||||
|
|
||||||
|
const favicon = await fetch(faviconUrl, {
|
||||||
|
timeout: 10000,
|
||||||
|
agent: getAgentByUrl,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (favicon.ok) {
|
||||||
|
return faviconUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
|
@ -1,72 +0,0 @@
|
||||||
import { getJson } from '../misc/fetch';
|
|
||||||
import { Instance } from '../models/entities/instance';
|
|
||||||
import { Instances } from '../models';
|
|
||||||
import { getNodeinfoLock } from '../misc/app-lock';
|
|
||||||
import Logger from '../services/logger';
|
|
||||||
|
|
||||||
export const logger = new Logger('nodeinfo', 'cyan');
|
|
||||||
|
|
||||||
export async function fetchNodeinfo(instance: Instance) {
|
|
||||||
const unlock = await getNodeinfoLock(instance.host);
|
|
||||||
|
|
||||||
const _instance = await Instances.findOne({ host: instance.host });
|
|
||||||
const now = Date.now();
|
|
||||||
if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24)) {
|
|
||||||
unlock();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info(`Fetching nodeinfo of ${instance.host} ...`);
|
|
||||||
|
|
||||||
try {
|
|
||||||
const wellknown = await getJson('https://' + instance.host + '/.well-known/nodeinfo')
|
|
||||||
.catch(e => {
|
|
||||||
if (e.statusCode === 404) {
|
|
||||||
throw 'No nodeinfo provided';
|
|
||||||
} else {
|
|
||||||
throw e.statusCode || e.message;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
if (wellknown.links == null || !Array.isArray(wellknown.links)) {
|
|
||||||
throw 'No wellknown links';
|
|
||||||
}
|
|
||||||
|
|
||||||
const links = wellknown.links as any[];
|
|
||||||
|
|
||||||
const lnik1_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/1.0');
|
|
||||||
const lnik2_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.0');
|
|
||||||
const lnik2_1 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.1');
|
|
||||||
const link = lnik2_1 || lnik2_0 || lnik1_0;
|
|
||||||
|
|
||||||
if (link == null) {
|
|
||||||
throw 'No nodeinfo link provided';
|
|
||||||
}
|
|
||||||
|
|
||||||
const info = await getJson(link.href)
|
|
||||||
.catch(e => {
|
|
||||||
throw e.statusCode || e.message;
|
|
||||||
});
|
|
||||||
|
|
||||||
await Instances.update(instance.id, {
|
|
||||||
infoUpdatedAt: new Date(),
|
|
||||||
softwareName: info.software.name.toLowerCase(),
|
|
||||||
softwareVersion: info.software.version,
|
|
||||||
openRegistrations: info.openRegistrations,
|
|
||||||
name: info.metadata ? (info.metadata.nodeName || info.metadata.name || null) : null,
|
|
||||||
description: info.metadata ? (info.metadata.nodeDescription || info.metadata.description || null) : null,
|
|
||||||
maintainerName: info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.name || null) : null : null,
|
|
||||||
maintainerEmail: info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.email || null) : null : null,
|
|
||||||
});
|
|
||||||
|
|
||||||
logger.succ(`Successfuly fetched nodeinfo of ${instance.host}`);
|
|
||||||
} catch (e) {
|
|
||||||
logger.error(`Failed to fetch nodeinfo of ${instance.host}: ${e}`);
|
|
||||||
|
|
||||||
await Instances.update(instance.id, {
|
|
||||||
infoUpdatedAt: new Date(),
|
|
||||||
});
|
|
||||||
} finally {
|
|
||||||
unlock();
|
|
||||||
}
|
|
||||||
}
|
|
Loading…
Reference in a new issue