egirlskey/packages/backend/src/core/FetchInstanceMetadataService.ts

315 lines
9.6 KiB
TypeScript
Raw Normal View History

/*
* SPDX-FileCopyrightText: syuilo and other misskey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
2022-09-17 18:27:08 +00:00
import { URL } from 'node:url';
import { Inject, Injectable } from '@nestjs/common';
import { JSDOM } from 'jsdom';
import tinycolor from 'tinycolor2';
2023-07-19 02:27:50 +00:00
import * as Redis from 'ioredis';
import type { MiInstance } from '@/models/Instance.js';
2022-09-18 14:07:41 +00:00
import type Logger from '@/logger.js';
2022-09-17 18:27:08 +00:00
import { DI } from '@/di-symbols.js';
2022-09-18 14:07:41 +00:00
import { LoggerService } from '@/core/LoggerService.js';
2022-12-04 01:16:03 +00:00
import { HttpRequestService } from '@/core/HttpRequestService.js';
2022-12-04 08:05:32 +00:00
import { bindThis } from '@/decorators.js';
2023-04-22 11:05:36 +00:00
import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
2022-09-17 18:27:08 +00:00
import type { DOMWindow } from 'jsdom';
type NodeInfo = {
2022-09-23 22:12:11 +00:00
openRegistrations?: unknown;
2022-09-17 18:27:08 +00:00
software?: {
2022-09-23 22:12:11 +00:00
name?: unknown;
version?: unknown;
2022-09-17 18:27:08 +00:00
};
metadata?: {
2022-09-23 22:12:11 +00:00
name?: unknown;
nodeName?: unknown;
nodeDescription?: unknown;
description?: unknown;
2022-09-17 18:27:08 +00:00
maintainer?: {
2022-09-23 22:12:11 +00:00
name?: unknown;
email?: unknown;
2022-09-17 18:27:08 +00:00
};
2022-09-23 22:12:11 +00:00
themeColor?: unknown;
2022-09-17 18:27:08 +00:00
};
};
@Injectable()
export class FetchInstanceMetadataService {
2022-09-18 18:11:50 +00:00
private logger: Logger;
2022-09-18 14:07:41 +00:00
2022-09-17 18:27:08 +00:00
constructor(
private httpRequestService: HttpRequestService,
2022-09-18 14:07:41 +00:00
private loggerService: LoggerService,
private federatedInstanceService: FederatedInstanceService,
@Inject(DI.redis)
private redisClient: Redis.Redis,
2022-09-17 18:27:08 +00:00
) {
2022-09-18 18:11:50 +00:00
this.logger = this.loggerService.getLogger('metadata', 'cyan');
2022-09-17 18:27:08 +00:00
}
@bindThis
public async tryLock(host: string): Promise<boolean> {
const mutex = await this.redisClient.set(`fetchInstanceMetadata:mutex:${host}`, '1', 'GET');
return mutex !== '1';
}
@bindThis
public unlock(host: string): Promise<'OK'> {
return this.redisClient.set(`fetchInstanceMetadata:mutex:${host}`, '0');
}
@bindThis
public async fetchInstanceMetadata(instance: MiInstance, force = false): Promise<void> {
const host = instance.host;
// Acquire mutex to ensure no parallel runs
if (!await this.tryLock(host)) return;
try {
if (!force) {
const _instance = await this.federatedInstanceService.fetch(host);
const now = Date.now();
if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24)) {
// unlock at the finally caluse
return;
}
2022-09-17 18:27:08 +00:00
}
this.logger.info(`Fetching metadata of ${instance.host} ...`);
2022-09-17 18:27:08 +00:00
const [info, dom, manifest] = await Promise.all([
2022-09-18 18:11:50 +00:00
this.fetchNodeinfo(instance).catch(() => null),
this.fetchDom(instance).catch(() => null),
this.fetchManifest(instance).catch(() => null),
2022-09-17 18:27:08 +00:00
]);
2022-09-17 18:27:08 +00:00
const [favicon, icon, themeColor, name, description] = await Promise.all([
2022-09-18 18:11:50 +00:00
this.fetchFaviconUrl(instance, dom).catch(() => null),
this.fetchIconUrl(instance, dom, manifest).catch(() => null),
this.getThemeColor(info, dom, manifest).catch(() => null),
this.getSiteName(info, dom, manifest).catch(() => null),
this.getDescription(info, dom, manifest).catch(() => null),
2022-09-17 18:27:08 +00:00
]);
2022-09-18 18:11:50 +00:00
this.logger.succ(`Successfuly fetched metadata of ${instance.host}`);
2022-09-17 18:27:08 +00:00
const updates = {
infoUpdatedAt: new Date(),
} as Record<string, any>;
2022-09-17 18:27:08 +00:00
if (info) {
2022-09-23 22:12:11 +00:00
updates.softwareName = typeof info.software?.name === 'string' ? info.software.name.toLowerCase() : '?';
2022-09-17 18:27:08 +00:00
updates.softwareVersion = info.software?.version;
updates.openRegistrations = info.openRegistrations;
updates.maintainerName = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.name ?? null) : null : null;
updates.maintainerEmail = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.email ?? null) : null : null;
}
2022-09-17 18:27:08 +00:00
if (name) updates.name = name;
if (description) updates.description = description;
if (icon ?? favicon) updates.iconUrl = (icon && !icon.includes('data:image/png;base64')) ? icon : favicon;
2022-09-17 18:27:08 +00:00
if (favicon) updates.faviconUrl = favicon;
if (themeColor) updates.themeColor = themeColor;
await this.federatedInstanceService.update(instance.id, updates);
2022-09-18 18:11:50 +00:00
this.logger.succ(`Successfuly updated metadata of ${instance.host}`);
2022-09-17 18:27:08 +00:00
} catch (e) {
2022-09-18 18:11:50 +00:00
this.logger.error(`Failed to update metadata of ${instance.host}: ${e}`);
2022-09-17 18:27:08 +00:00
} finally {
await this.unlock(host);
2022-09-17 18:27:08 +00:00
}
}
@bindThis
private async fetchNodeinfo(instance: MiInstance): Promise<NodeInfo> {
2022-09-18 18:11:50 +00:00
this.logger.info(`Fetching nodeinfo of ${instance.host} ...`);
2022-09-17 18:27:08 +00:00
try {
const wellknown = await this.httpRequestService.getJson('https://' + instance.host + '/.well-known/nodeinfo')
.catch(err => {
if (err.statusCode === 404) {
throw new Error('No nodeinfo provided');
2022-09-17 18:27:08 +00:00
} else {
throw err.statusCode ?? err.message;
}
}) as Record<string, unknown>;
2022-09-17 18:27:08 +00:00
if (wellknown.links == null || !Array.isArray(wellknown.links)) {
throw new Error('No wellknown links');
2022-09-17 18:27:08 +00:00
}
2022-09-17 18:27:08 +00:00
const links = wellknown.links as any[];
const link1_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/1.0');
const link2_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.0');
const link2_1 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.1');
const link = link2_1 ?? link2_0 ?? link1_0;
2022-09-17 18:27:08 +00:00
if (link == null) {
throw new Error('No nodeinfo link provided');
2022-09-17 18:27:08 +00:00
}
2022-09-17 18:27:08 +00:00
const info = await this.httpRequestService.getJson(link.href)
.catch(err => {
throw err.statusCode ?? err.message;
});
2022-09-18 18:11:50 +00:00
this.logger.succ(`Successfuly fetched nodeinfo of ${instance.host}`);
2022-09-17 18:27:08 +00:00
return info as NodeInfo;
} catch (err) {
2022-09-18 18:11:50 +00:00
this.logger.error(`Failed to fetch nodeinfo of ${instance.host}: ${err}`);
2022-09-17 18:27:08 +00:00
throw err;
}
}
@bindThis
private async fetchDom(instance: MiInstance): Promise<DOMWindow['document']> {
2022-09-18 18:11:50 +00:00
this.logger.info(`Fetching HTML of ${instance.host} ...`);
2022-09-17 18:27:08 +00:00
const url = 'https://' + instance.host;
2022-09-17 18:27:08 +00:00
const html = await this.httpRequestService.getHtml(url);
2022-09-17 18:27:08 +00:00
const { window } = new JSDOM(html);
const doc = window.document;
2022-09-17 18:27:08 +00:00
return doc;
}
@bindThis
private async fetchManifest(instance: MiInstance): Promise<Record<string, unknown> | null> {
2022-09-17 18:27:08 +00:00
const url = 'https://' + instance.host;
2022-09-17 18:27:08 +00:00
const manifestUrl = url + '/manifest.json';
2022-09-17 18:27:08 +00:00
const manifest = await this.httpRequestService.getJson(manifestUrl) as Record<string, unknown>;
2022-09-17 18:27:08 +00:00
return manifest;
}
@bindThis
private async fetchFaviconUrl(instance: MiInstance, doc: DOMWindow['document'] | null): Promise<string | null> {
2022-09-17 18:27:08 +00:00
const url = 'https://' + instance.host;
2022-09-17 18:27:08 +00:00
if (doc) {
// https://github.com/misskey-dev/misskey/pull/8220#issuecomment-1025104043
const href = Array.from(doc.getElementsByTagName('link')).reverse().find(link => link.relList.contains('icon'))?.href;
2022-09-17 18:27:08 +00:00
if (href) {
return (new URL(href, url)).href;
}
}
2022-09-17 18:27:08 +00:00
const faviconUrl = url + '/favicon.ico';
const favicon = await this.httpRequestService.send(faviconUrl, {
method: 'HEAD',
}, { throwErrorWhenResponseNotOk: false });
2022-09-17 18:27:08 +00:00
if (favicon.ok) {
return faviconUrl;
}
2022-09-17 18:27:08 +00:00
return null;
}
@bindThis
private async fetchIconUrl(instance: MiInstance, doc: DOMWindow['document'] | null, manifest: Record<string, any> | null): Promise<string | null> {
2022-09-17 18:27:08 +00:00
if (manifest && manifest.icons && manifest.icons.length > 0 && manifest.icons[0].src) {
const url = 'https://' + instance.host;
return (new URL(manifest.icons[0].src, url)).href;
}
2022-09-17 18:27:08 +00:00
if (doc) {
const url = 'https://' + instance.host;
2022-09-17 18:27:08 +00:00
// https://github.com/misskey-dev/misskey/pull/8220#issuecomment-1025104043
const links = Array.from(doc.getElementsByTagName('link')).reverse();
// https://github.com/misskey-dev/misskey/pull/8220/files/0ec4eba22a914e31b86874f12448f88b3e58dd5a#r796487559
const href =
2022-09-17 18:27:08 +00:00
[
links.find(link => link.relList.contains('apple-touch-icon-precomposed'))?.href,
links.find(link => link.relList.contains('apple-touch-icon'))?.href,
links.find(link => link.relList.contains('icon'))?.href,
]
.find(href => href);
2022-09-17 18:27:08 +00:00
if (href) {
return (new URL(href, url)).href;
}
}
2022-09-17 18:27:08 +00:00
return null;
}
@bindThis
2022-09-18 18:11:50 +00:00
private async getThemeColor(info: NodeInfo | null, doc: DOMWindow['document'] | null, manifest: Record<string, any> | null): Promise<string | null> {
2022-09-17 18:27:08 +00:00
const themeColor = info?.metadata?.themeColor ?? doc?.querySelector('meta[name="theme-color"]')?.getAttribute('content') ?? manifest?.theme_color;
2022-09-17 18:27:08 +00:00
if (themeColor) {
const color = new tinycolor(themeColor);
if (color.isValid()) return color.toHexString();
}
2022-09-17 18:27:08 +00:00
return null;
}
@bindThis
2022-09-18 18:11:50 +00:00
private async getSiteName(info: NodeInfo | null, doc: DOMWindow['document'] | null, manifest: Record<string, any> | null): Promise<string | null> {
2022-09-17 18:27:08 +00:00
if (info && info.metadata) {
2022-09-23 22:12:11 +00:00
if (typeof info.metadata.nodeName === 'string') {
return info.metadata.nodeName;
} else if (typeof info.metadata.name === 'string') {
return info.metadata.name;
2022-09-17 18:27:08 +00:00
}
}
2022-09-17 18:27:08 +00:00
if (doc) {
const og = doc.querySelector('meta[property="og:title"]')?.getAttribute('content');
2022-09-17 18:27:08 +00:00
if (og) {
return og;
}
}
2022-09-17 18:27:08 +00:00
if (manifest) {
return manifest.name ?? manifest.short_name;
}
2022-09-17 18:27:08 +00:00
return null;
}
@bindThis
2022-09-18 18:11:50 +00:00
private async getDescription(info: NodeInfo | null, doc: DOMWindow['document'] | null, manifest: Record<string, any> | null): Promise<string | null> {
2022-09-17 18:27:08 +00:00
if (info && info.metadata) {
2022-09-23 22:12:11 +00:00
if (typeof info.metadata.nodeDescription === 'string') {
return info.metadata.nodeDescription;
} else if (typeof info.metadata.description === 'string') {
return info.metadata.description;
2022-09-17 18:27:08 +00:00
}
}
2022-09-17 18:27:08 +00:00
if (doc) {
const meta = doc.querySelector('meta[name="description"]')?.getAttribute('content');
if (meta) {
return meta;
}
2022-09-17 18:27:08 +00:00
const og = doc.querySelector('meta[property="og:description"]')?.getAttribute('content');
if (og) {
return og;
}
}
2022-09-17 18:27:08 +00:00
if (manifest) {
return manifest.name ?? manifest.short_name;
}
2022-09-17 18:27:08 +00:00
return null;
}
}