profxadke/misskey
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'develop' into pr/ThatOneCalculator/8764

Browse source
This commit is contained in:
tamaina 2022-07-05 05:16:06 +00:00
commit 9cd1526073
249 changed files with 4798 additions and 6822 deletions
Download patch file Download diff file Expand all files Collapse all files

13
packages/backend/migration/1655813815729-driveCapacityOverrideMb.js Normal file
View file

@ -0,0 +1,13 @@
export class driveCapacityOverrideMb1655813815729 {
name = 'driveCapacityOverrideMb1655813815729'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "user" ADD "driveCapacityOverrideMb" integer`);
await queryRunner.query(`COMMENT ON COLUMN "user"."driveCapacityOverrideMb" IS 'Overrides user drive capacity limit'`);
}
async down(queryRunner) {
await queryRunner.query(`COMMENT ON COLUMN "user"."driveCapacityOverrideMb" IS 'Overrides user drive capacity limit'`);
await queryRunner.query(`ALTER TABLE "user" DROP COLUMN "driveCapacityOverrideMb"`);
}
}

17
packages/backend/migration/1655918165614-user-ip.js Normal file
View file

@ -0,0 +1,17 @@
export class userIp1655918165614 {
name = 'userIp1655918165614'
async up(queryRunner) {
await queryRunner.query(`CREATE TABLE "user_ip" ("id" SERIAL NOT NULL, "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL, "userId" character varying(32) NOT NULL, "ip" character varying(128) NOT NULL, CONSTRAINT "PK_2c44ddfbf7c0464d028dcef325e" PRIMARY KEY ("id"))`);
await queryRunner.query(`CREATE INDEX "IDX_7f7f1c66f48e9a8e18a33bc515" ON "user_ip" ("userId") `);
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_361b500e06721013c124b7b6c5" ON "user_ip" ("userId", "ip") `);
await queryRunner.query(`ALTER TABLE "user_ip" ADD CONSTRAINT "FK_7f7f1c66f48e9a8e18a33bc5150" FOREIGN KEY ("userId") REFERENCES "user"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "user_ip" DROP CONSTRAINT "FK_7f7f1c66f48e9a8e18a33bc5150"`);
await queryRunner.query(`DROP INDEX "public"."IDX_361b500e06721013c124b7b6c5"`);
await queryRunner.query(`DROP INDEX "public"."IDX_7f7f1c66f48e9a8e18a33bc515"`);
await queryRunner.query(`DROP TABLE "user_ip"`);
}
}

13
packages/backend/migration/1656122560740-file-ip.js Normal file
View file

@ -0,0 +1,13 @@
export class fileIp1656122560740 {
name = 'fileIp1656122560740'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "drive_file" ADD "requestHeaders" jsonb DEFAULT '{}'`);
await queryRunner.query(`ALTER TABLE "drive_file" ADD "requestIp" character varying(128)`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "drive_file" DROP COLUMN "requestIp"`);
await queryRunner.query(`ALTER TABLE "drive_file" DROP COLUMN "requestHeaders"`);
}
}

13
packages/backend/migration/1656328812281-ip-2.js Normal file
View file

@ -0,0 +1,13 @@
export class ip21656328812281 {
name = 'ip21656328812281'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "user_ip" DROP CONSTRAINT "FK_7f7f1c66f48e9a8e18a33bc5150"`);
await queryRunner.query(`ALTER TABLE "meta" ADD "enableIpLogging" boolean NOT NULL DEFAULT false`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "enableIpLogging"`);
await queryRunner.query(`ALTER TABLE "user_ip" ADD CONSTRAINT "FK_7f7f1c66f48e9a8e18a33bc5150" FOREIGN KEY ("userId") REFERENCES "user"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`);
}
}

11
packages/backend/migration/1656772790599-user-moderation-note.js Normal file
View file

@ -0,0 +1,11 @@
export class userModerationNote1656772790599 {
name = 'userModerationNote1656772790599'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "user_profile" ADD "moderationNote" character varying(8192) NOT NULL DEFAULT ''`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "user_profile" DROP COLUMN "moderationNote"`);
}
}

44
packages/backend/package.json
View file

@ -18,9 +18,9 @@
"lodash": "^4.17.21"
},
"dependencies": {
"@bull-board/api": "^3.11.1",
"@bull-board/koa": "3.10.4",
"@bull-board/ui": "^3.11.1",
"@bull-board/api": "4.0.0",
"@bull-board/koa": "4.0.0",
"@bull-board/ui": "4.0.0",
"@discordapp/twemoji": "14.0.2",
"@elastic/elasticsearch": "7.17.0",
"@koa/cors": "3.3.0",
@ -34,10 +34,10 @@
"archiver": "5.3.1",
"autobind-decorator": "2.4.0",
"autwh": "0.1.0",
"aws-sdk": "2.1152.0",
"aws-sdk": "2.1165.0",
"bcryptjs": "2.4.3",
"blurhash": "1.1.5",
"bull": "4.8.3",
"bull": "4.8.4",
"cacheable-lookup": "6.0.4",
"cbor": "8.1.0",
"chalk": "5.0.1",
@ -57,7 +57,7 @@
"ip-cidr": "3.0.10",
"is-svg": "4.3.2",
"js-yaml": "4.1.0",
"jsdom": "19.0.0",
"jsdom": "20.0.0",
"json5": "2.2.1",
"json5-loader": "4.0.1",
"jsonld": "6.0.0",
@ -79,26 +79,27 @@
"multer": "1.4.4",
"nested-property": "4.0.0",
"node-fetch": "3.2.6",
"nodemailer": "6.7.5",
"nodemailer": "6.7.6",
"oauth": "^0.9.15",
"os-utils": "0.0.14",
"parse5": "6.0.1",
"parse5": "7.0.0",
"pg": "8.7.3",
"private-ip": "2.3.3",
"probe-image-size": "7.2.3",
"promise-limit": "2.7.0",
"pug": "3.0.2",
"punycode": "2.1.1",
"pureimage": "0.3.8",
"pureimage": "0.3.14",
"qrcode": "1.5.0",
"random-seed": "0.3.0",
"ratelimiter": "3.4.1",
"re2": "1.17.4",
"re2": "1.17.7",
"redis-lock": "0.1.4",
"reflect-metadata": "0.1.13",
"rename": "1.0.4",
"require-all": "3.0.0",
"rndstr": "1.0.0",
"rss-parser": "3.12.0",
"s-age": "1.1.2",
"sanitize-html": "2.7.0",
"semver": "7.3.7",
@ -109,15 +110,15 @@
"style-loader": "3.3.1",
"summaly": "2.6.0",
"syslog-pro": "1.0.0",
"systeminformation": "5.11.16",
"systeminformation": "5.11.22",
"tinycolor2": "1.4.2",
"tmp": "0.2.1",
"ts-loader": "9.3.0",
"ts-loader": "9.3.1",
"ts-node": "10.8.1",
"tsc-alias": "1.6.9",
"tsc-alias": "1.6.11",
"tsconfig-paths": "4.0.0",
"twemoji-parser": "14.0.0",
"typeorm": "0.3.6",
"typeorm": "0.3.7",
"ulid": "2.3.0",
"unzipper": "0.10.11",
"uuid": "8.3.2",
@ -150,11 +151,10 @@
"@types/koa__multer": "2.0.4",
"@types/koa__router": "8.0.11",
"@types/mocha": "9.1.1",
"@types/node": "17.0.41",
"@types/node": "18.0.0",
"@types/node-fetch": "3.0.3",
"@types/nodemailer": "6.4.4",
"@types/oauth": "0.9.1",
"@types/parse5": "6.0.3",
"@types/pug": "2.0.6",
"@types/punycode": "2.1.0",
"@types/qrcode": "1.4.2",
@ -163,8 +163,8 @@
"@types/redis": "4.0.11",
"@types/rename": "1.0.4",
"@types/sanitize-html": "2.6.2",
"@types/semver": "7.3.9",
"@types/sharp": "0.30.2",
"@types/semver": "7.3.10",
"@types/sharp": "0.30.4",
"@types/sinonjs__fake-timers": "8.1.2",
"@types/speakeasy": "2.0.7",
"@types/tinycolor2": "1.4.3",
@ -173,13 +173,13 @@
"@types/web-push": "3.3.2",
"@types/websocket": "1.0.5",
"@types/ws": "8.5.3",
"@typescript-eslint/eslint-plugin": "5.27.1",
"@typescript-eslint/parser": "5.27.1",
"@typescript-eslint/eslint-plugin": "5.30.0",
"@typescript-eslint/parser": "5.30.0",
"cross-env": "7.0.3",
"eslint": "8.17.0",
"eslint": "8.18.0",
"eslint-plugin-import": "2.26.0",
"execa": "6.1.0",
"form-data": "^4.0.0",
"typescript": "4.7.2"
"typescript": "4.7.4"
}
}

4
packages/backend/src/db/postgre.ts
View file

@ -68,9 +68,10 @@ import { RegistryItem } from '@/models/entities/registry-item.js';
import { Ad } from '@/models/entities/ad.js';
import { PasswordResetRequest } from '@/models/entities/password-reset-request.js';
import { UserPending } from '@/models/entities/user-pending.js';
import { Webhook } from '@/models/entities/webhook.js';
import { UserIp } from '@/models/entities/user-ip.js';
import { entities as charts } from '@/services/chart/entities.js';
import { Webhook } from '@/models/entities/webhook.js';
import { envOption } from '../env.js';
import { dbLogger } from './logger.js';
import { redisClient } from './redis.js';
@ -173,6 +174,7 @@ export const entities = [
PasswordResetRequest,
UserPending,
Webhook,
UserIp,
...charts,
];

16
packages/backend/src/mfm/from-html.ts
View file

@ -1,8 +1,10 @@
import * as parse5 from 'parse5';
import treeAdapter from 'parse5/lib/tree-adapters/default.js';
import { URL } from 'node:url';
import * as parse5 from 'parse5';
import * as TreeAdapter from '../../node_modules/parse5/dist/tree-adapters/default.js';
const urlRegex = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+/;
const treeAdapter = TreeAdapter.defaultTreeAdapter;
const urlRegex = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+/;
const urlRegexFull = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+$/;
export function fromHtml(html: string, hashtagNames?: string[]): string {
@ -19,7 +21,7 @@ export function fromHtml(html: string, hashtagNames?: string[]): string {
return text.trim();
function getText(node: parse5.Node): string {
function getText(node: TreeAdapter.Node): string {
if (treeAdapter.isTextNode(node)) return node.value;
if (!treeAdapter.isElementNode(node)) return '';
if (node.nodeName === 'br') return '\n';
@ -31,7 +33,7 @@ export function fromHtml(html: string, hashtagNames?: string[]): string {
return '';
}
function appendChildren(childNodes: parse5.ChildNode[]): void {
function appendChildren(childNodes: TreeAdapter.ChildNode[]): void {
if (childNodes) {
for (const n of childNodes) {
analyze(n);
@ -39,7 +41,7 @@ export function fromHtml(html: string, hashtagNames?: string[]): string {
}
}
function analyze(node: parse5.Node) {
function analyze(node: TreeAdapter.Node) {
if (treeAdapter.isTextNode(node)) {
text += node.value;
return;
@ -170,7 +172,7 @@ export function fromHtml(html: string, hashtagNames?: string[]): string {
const t = getText(node);
if (t) {
text += '\n> ';
text += t.split('\n').join(`\n> `);
text += t.split('\n').join('\n> ');
}
break;
}

13
packages/backend/src/models/entities/drive-file.ts
View file

@ -1,7 +1,7 @@
import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm';
import { id } from '../id.js';
import { User } from './user.js';
import { DriveFolder } from './drive-folder.js';
import { id } from '../id.js';
@Entity()
@Index(['userId', 'folderId', 'id'])
@ -165,4 +165,15 @@ export class DriveFile {
comment: 'Whether the DriveFile is direct link to remote server.',
})
public isLink: boolean;
@Column('jsonb', {
default: {},
nullable: true,
})
public requestHeaders: Record<string, string> | null;
@Column('varchar', {
length: 128, nullable: true,
})
public requestIp: string | null;
}

7
packages/backend/src/models/entities/meta.ts
View file

@ -1,6 +1,6 @@
import { Entity, Column, PrimaryColumn, ManyToOne, JoinColumn } from 'typeorm';
import { User } from './user.js';
import { id } from '../id.js';
import { User } from './user.js';
import { Clip } from './clip.js';
@Entity()
@ -427,4 +427,9 @@ export class Meta {
default: true,
})
public objectStorageS3ForcePathStyle: boolean;
@Column('boolean', {
default: false,
})
public enableIpLogging: boolean;
}

24
packages/backend/src/models/entities/user-ip.ts Normal file
View file

@ -0,0 +1,24 @@
import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne, PrimaryGeneratedColumn } from 'typeorm';
import { id } from '../id.js';
import { Note } from './note.js';
import { User } from './user.js';
@Entity()
@Index(['userId', 'ip'], { unique: true })
export class UserIp {
@PrimaryGeneratedColumn()
public id: string;
@Column('timestamp with time zone', {
})
public createdAt: Date;
@Index()
@Column(id())
public userId: User['id'];
@Column('varchar', {
length: 128,
})
public ip: string;
}

7
packages/backend/src/models/entities/user-profile.ts
View file

@ -1,8 +1,8 @@
import { Entity, Column, Index, OneToOne, JoinColumn, PrimaryColumn } from 'typeorm';
import { ffVisibility, notificationTypes } from '@/types.js';
import { id } from '../id.js';
import { User } from './user.js';
import { Page } from './page.js';
import { ffVisibility, notificationTypes } from '@/types.js';
// TODO: このテーブルで管理している情報すべてレジストリで管理するようにしても良いかも
// ただ、「emailVerified が true なユーザーを find する」のようなクエリは書けなくなるからウーン
@ -117,6 +117,11 @@ export class UserProfile {
})
public password: string | null;
@Column('varchar', {
length: 8192, default: '',
})
public moderationNote: string | null;
// TODO: そのうち消す
@Column('jsonb', {
default: {},

6
packages/backend/src/models/entities/user.ts
View file

@ -218,6 +218,12 @@ export class User {
})
public token: string | null;
@Column('integer', {
nullable: true,
comment: 'Overrides user drive capacity limit',
})
public driveCapacityOverrideMb: number | null;
constructor(data: Partial<User>) {
if (data == null) return;

2
packages/backend/src/models/index.ts
View file

@ -65,6 +65,7 @@ import { PasswordResetRequest } from './entities/password-reset-request.js';
import { UserPending } from './entities/user-pending.js';
import { InstanceRepository } from './repositories/instance.js';
import { Webhook } from './entities/webhook.js';
import { UserIp } from './entities/user-ip.js';
export const Announcements = db.getRepository(Announcement);
export const AnnouncementReads = db.getRepository(AnnouncementRead);
@ -90,6 +91,7 @@ export const UserGroups = (UserGroupRepository);
export const UserGroupJoinings = db.getRepository(UserGroupJoining);
export const UserGroupInvitations = (UserGroupInvitationRepository);
export const UserNotePinings = db.getRepository(UserNotePining);
export const UserIps = db.getRepository(UserIp);
export const UsedUsernames = db.getRepository(UsedUsername);
export const Followings = (FollowingRepository);
export const FollowRequests = (FollowRequestRepository);

1
packages/backend/src/models/repositories/user.ts
View file

@ -315,6 +315,7 @@ export const UserRepository = db.getRepository(User).extend({
} : undefined) : undefined,
emojis: populateEmojis(user.emojis, user.host),
onlineStatus: this.getOnlineStatus(user),
driveCapacityOverrideMb: user.driveCapacityOverrideMb,
...(opts.detail ? {
url: profile!.url,

18
packages/backend/src/queue/index.ts
View file

@ -2,6 +2,9 @@ import httpSignature from '@peertube/http-signature';
import { v4 as uuid } from 'uuid';
import config from '@/config/index.js';
import { DriveFile } from '@/models/entities/drive-file.js';
import { IActivity } from '@/remote/activitypub/type.js';
import { Webhook, webhookEventTypes } from '@/models/entities/webhook.js';
import { envOption } from '../env.js';
import processDeliver from './processors/deliver.js';
@ -12,18 +15,15 @@ import processSystemQueue from './processors/system/index.js';
import processWebhookDeliver from './processors/webhook-deliver.js';
import { endedPollNotification } from './processors/ended-poll-notification.js';
import { queueLogger } from './logger.js';
import { DriveFile } from '@/models/entities/drive-file.js';
import { getJobInfo } from './get-job-info.js';
import { systemQueue, dbQueue, deliverQueue, inboxQueue, objectStorageQueue, endedPollNotificationQueue, webhookDeliverQueue } from './queues.js';
import { ThinUser } from './types.js';
import { IActivity } from '@/remote/activitypub/type.js';
import { Webhook, webhookEventTypes } from '@/models/entities/webhook.js';
function renderError(e: Error): any {
return {
stack: e?.stack,
message: e?.message,
name: e?.name,
stack: e.stack,
message: e.message,
name: e.name,
};
}
@ -314,6 +314,12 @@ export default function() {
removeOnComplete: true,
});
systemQueue.add('clean', {
}, {
repeat: { cron: '0 0 * * *' },
removeOnComplete: true,
});
systemQueue.add('checkExpiredMutings', {
}, {
repeat: { cron: '*/5 * * * *' },

18
packages/backend/src/queue/processors/system/clean.ts Normal file
View file

@ -0,0 +1,18 @@
import Bull from 'bull';
import { LessThan } from 'typeorm';
import { UserIps } from '@/models/index.js';
import { queueLogger } from '../../logger.js';
const logger = queueLogger.createSubLogger('clean');
export async function clean(job: Bull.Job<Record<string, unknown>>, done: any): Promise<void> {
logger.info('Cleaning...');
UserIps.delete({
createdAt: LessThan(new Date(Date.now() - (1000 * 60 * 60 * 24 * 90))),
});
logger.succ('Cleaned.');
done();
}

2
packages/backend/src/queue/processors/system/index.ts
View file

@ -3,12 +3,14 @@ import { tickCharts } from './tick-charts.js';
import { resyncCharts } from './resync-charts.js';
import { cleanCharts } from './clean-charts.js';
import { checkExpiredMutings } from './check-expired-mutings.js';
import { clean } from './clean.js';
const jobs = {
tickCharts,
resyncCharts,
cleanCharts,
checkExpiredMutings,
clean,
} as Record<string, Bull.ProcessCallbackFunction<Record<string, unknown>> | Bull.ProcessPromiseFunction<Record<string, unknown>>>;
export default function(dbQueue: Bull.Queue<Record<string, unknown>>) {

34
packages/backend/src/server/api/api-handler.ts
View file

@ -1,10 +1,19 @@
import Koa from 'koa';
import { User } from '@/models/entities/user.js';
import { UserIps } from '@/models/index.js';
import { fetchMeta } from '@/misc/fetch-meta.js';
import { IEndpoint } from './endpoints.js';
import authenticate, { AuthenticationError } from './authenticate.js';
import call from './call.js';
import { ApiError } from './error.js';
const userIpHistories = new Map<User['id'], Set<string>>();
setInterval(() => {
userIpHistories.clear();
}, 1000 * 60 * 60);
export default (endpoint: IEndpoint, ctx: Koa.Context) => new Promise<void>((res) => {
const body = ctx.is('multipart/form-data')
? (ctx.request as any).body
@ -44,6 +53,31 @@ export default (endpoint: IEndpoint, ctx: Koa.Context) => new Promise<void>((res
}).catch((e: ApiError) => {
reply(e.httpStatusCode ? e.httpStatusCode : e.kind === 'client' ? 400 : 500, e);
});
// Log IP
if (user) {
fetchMeta().then(meta => {
if (!meta.enableIpLogging) return;
const ip = ctx.ip;
const ips = userIpHistories.get(user.id);
if (ips == null || !ips.has(ip)) {
if (ips == null) {
userIpHistories.set(user.id, new Set([ip]));
} else {
ips.add(ip);
}
try {
UserIps.insert({
createdAt: new Date(),
userId: user.id,
ip: ip,
});
} catch {
}
}
});
}
}).catch(e => {
if (e instanceof AuthenticationError) {
reply(403, new ApiError({

2
packages/backend/src/server/api/call.ts
View file

@ -116,7 +116,7 @@ export default async (endpoint: string, user: CacheableLocalUser | null | undefi
// API invoking
const before = performance.now();
return await ep.exec(data, user, token, ctx?.file).catch((e: Error) => {
return await ep.exec(data, user, token, ctx?.file, ctx?.ip, ctx?.headers).catch((e: Error) => {
if (e instanceof ApiError) {
throw e;
} else {

21
packages/backend/src/server/api/common/generate-muted-user-query.ts
View file

@ -51,26 +51,7 @@ export function generateMutedUserQueryForUsers(q: SelectQueryBuilder<any>, me: {
.select('muting.muteeId')
.where('muting.muterId = :muterId', { muterId: me.id });
const mutingInstanceQuery = UserProfiles.createQueryBuilder('user_profile')
.select('user_profile.mutedInstances')
.where('user_profile.userId = :muterId', { muterId: me.id });
q
.andWhere(`user.id NOT IN (${ mutingQuery.getQuery() })`)
// mute instances
.andWhere(new Brackets(qb => { qb
.andWhere('note.userHost IS NULL')
.orWhere(`NOT ((${ mutingInstanceQuery.getQuery() })::jsonb ? note.userHost)`);
}))
.andWhere(new Brackets(qb => { qb
.where('note.replyUserHost IS NULL')
.orWhere(`NOT ((${ mutingInstanceQuery.getQuery() })::jsonb ? note.replyUserHost)`);
}))
.andWhere(new Brackets(qb => { qb
.where('note.renoteUserHost IS NULL')
.orWhere(`NOT ((${ mutingInstanceQuery.getQuery() })::jsonb ? note.renoteUserHost)`);
}));
q.andWhere(`user.id NOT IN (${ mutingQuery.getQuery() })`);
q.setParameters(mutingQuery.getParameters());
q.setParameters(mutingInstanceQuery.getParameters());
}

34
packages/backend/src/server/api/define.ts
View file

@ -1,16 +1,16 @@
import * as fs from 'node:fs';
import Ajv from 'ajv';
import { CacheableLocalUser, ILocalUser } from '@/models/entities/user.js';
import { IEndpointMeta } from './endpoints.js';
import { ApiError } from './error.js';
import { Schema, SchemaType } from '@/misc/schema.js';
import { AccessToken } from '@/models/entities/access-token.js';
import { IEndpointMeta } from './endpoints.js';
import { ApiError } from './error.js';
export type Response = Record<string, any> | void;
// TODO: paramsの型をT['params']のスキーマ定義から推論する
type executor<T extends IEndpointMeta, Ps extends Schema> =
(params: SchemaType<Ps>, user: T['requireCredential'] extends true ? CacheableLocalUser : CacheableLocalUser | null, token: AccessToken | null, file?: any, cleanup?: () => any) =>
(params: SchemaType<Ps>, user: T['requireCredential'] extends true ? CacheableLocalUser : CacheableLocalUser | null, token: AccessToken | null, file?: any, cleanup?: () => any, ip?: string | null, headers?: Record<string, string> | null) =>
Promise<T['res'] extends undefined ? Response : SchemaType<NonNullable<T['res']>>>;
const ajv = new Ajv({
@ -20,23 +20,27 @@ const ajv = new Ajv({
ajv.addFormat('misskey:id', /^[a-zA-Z0-9]+$/);
export default function <T extends IEndpointMeta, Ps extends Schema>(meta: T, paramDef: Ps, cb: executor<T, Ps>)
: (params: any, user: T['requireCredential'] extends true ? CacheableLocalUser : CacheableLocalUser | null, token: AccessToken | null, file?: any) => Promise<any> {
: (params: any, user: T['requireCredential'] extends true ? CacheableLocalUser : CacheableLocalUser | null, token: AccessToken | null, file?: any, ip?: string | null, headers?: Record<string, string> | null) => Promise<any> {
const validate = ajv.compile(paramDef);
return (params: any, user: T['requireCredential'] extends true ? CacheableLocalUser : CacheableLocalUser | null, token: AccessToken | null, file?: any) => {
function cleanup() {
fs.unlink(file.path, () => {});
}
return (params: any, user: T['requireCredential'] extends true ? CacheableLocalUser : CacheableLocalUser | null, token: AccessToken | null, file?: any, ip?: string | null, headers?: Record<string, string> | null) => {
let cleanup: undefined | (() => void) = undefined;
if (meta.requireFile && file == null) return Promise.reject(new ApiError({
message: 'File required.',
code: 'FILE_REQUIRED',
id: '4267801e-70d1-416a-b011-4ee502885d8b',
}));
if (meta.requireFile) {
cleanup = () => {
fs.unlink(file.path, () => {});
};
if (file == null) return Promise.reject(new ApiError({
message: 'File required.',
code: 'FILE_REQUIRED',
id: '4267801e-70d1-416a-b011-4ee502885d8b',
}));
}
const valid = validate(params);
if (!valid) {
if (file) cleanup();
if (file) cleanup!();
const errors = validate.errors!;
const err = new ApiError({
@ -50,6 +54,6 @@ export default function <T extends IEndpointMeta, Ps extends Schema>(meta: T, pa
return Promise.reject(err);
}
return cb(params as SchemaType<Ps>, user, token, file, cleanup);
return cb(params as SchemaType<Ps>, user, token, file, cleanup, ip, headers);
};
}

8
packages/backend/src/server/api/endpoints.ts
View file

@ -35,6 +35,7 @@ import * as ep___admin_federation_removeAllFollowing from './endpoints/admin/fed
import * as ep___admin_federation_updateInstance from './endpoints/admin/federation/update-instance.js';
import * as ep___admin_getIndexStats from './endpoints/admin/get-index-stats.js';
import * as ep___admin_getTableStats from './endpoints/admin/get-table-stats.js';
import * as ep___admin_getUserIps from './endpoints/admin/get-user-ips.js';
import * as ep___admin_invite from './endpoints/admin/invite.js';
import * as ep___admin_moderators_add from './endpoints/admin/moderators/add.js';
import * as ep___admin_moderators_remove from './endpoints/admin/moderators/remove.js';
@ -60,6 +61,7 @@ import * as ep___admin_unsuspendUser from './endpoints/admin/unsuspend-user.js';
import * as ep___admin_updateMeta from './endpoints/admin/update-meta.js';
import * as ep___admin_vacuum from './endpoints/admin/vacuum.js';
import * as ep___admin_deleteAccount from './endpoints/admin/delete-account.js';
import * as ep___admin_updateUserNote from './endpoints/admin/update-user-note.js';
import * as ep___announcements from './endpoints/announcements.js';
import * as ep___antennas_create from './endpoints/antennas/create.js';
import * as ep___antennas_delete from './endpoints/antennas/delete.js';
@ -311,6 +313,8 @@ import * as ep___users_searchByUsernameAndHost from './endpoints/users/search-by
import * as ep___users_search from './endpoints/users/search.js';
import * as ep___users_show from './endpoints/users/show.js';
import * as ep___users_stats from './endpoints/users/stats.js';
import * as ep___fetchRss from './endpoints/fetch-rss.js';
import * as ep___admin_driveCapOverride from './endpoints/admin/drive-capacity-override.js';
const eps = [
['admin/meta', ep___admin_meta],
@ -348,6 +352,7 @@ const eps = [
['admin/federation/update-instance', ep___admin_federation_updateInstance],
['admin/get-index-stats', ep___admin_getIndexStats],
['admin/get-table-stats', ep___admin_getTableStats],
['admin/get-user-ips', ep___admin_getUserIps],
['admin/invite', ep___admin_invite],
['admin/moderators/add', ep___admin_moderators_add],
['admin/moderators/remove', ep___admin_moderators_remove],
@ -373,6 +378,7 @@ const eps = [
['admin/update-meta', ep___admin_updateMeta],
['admin/vacuum', ep___admin_vacuum],
['admin/delete-account', ep___admin_deleteAccount],
['admin/update-user-note', ep___admin_updateUserNote],
['announcements', ep___announcements],
['antennas/create', ep___antennas_create],
['antennas/delete', ep___antennas_delete],
@ -624,6 +630,8 @@ const eps = [
['users/search', ep___users_search],
['users/show', ep___users_show],
['users/stats', ep___users_stats],
['admin/drive-capacity-override', ep___admin_driveCapOverride],
['fetch-rss', ep___fetchRss],
];
export interface IEndpointMeta {

47
packages/backend/src/server/api/endpoints/admin/drive-capacity-override.ts Normal file
View file

@ -0,0 +1,47 @@
import define from '../../define.js';
import { Users } from '@/models/index.js';
import { User } from '@/models/entities/user.js';
import { insertModerationLog } from '@/services/insert-moderation-log.js';
export const meta = {
tags: ['admin'],
requireCredential: true,
requireModerator: true,
} as const;
export const paramDef = {
type: 'object',
properties: {
userId: { type: 'string', format: 'misskey:id' },
overrideMb: { type: 'number', nullable: true },
},
required: ['userId', 'overrideMb'],
} as const;
// eslint-disable-next-line import/no-default-export
export default define(meta, paramDef, async (ps, me) => {
const user = await Users.findOneBy({ id: ps.userId });
if (user == null) {
throw new Error('user not found');
}
if (!Users.isLocalUser(user)) {
throw new Error('user is not local user');
}
/*if (user.isAdmin) {
throw new Error('cannot suspend admin');
}
if (user.isModerator) {
throw new Error('cannot suspend moderator');
}*/
await Users.update(user.id, {
driveCapacityOverrideMb: ps.overrideMb,
});
insertModerationLog(me, 'change-drive-capacity-override', {
targetId: user.id,
});
});

7
packages/backend/src/server/api/endpoints/admin/drive/show-file.ts
View file

@ -1,6 +1,6 @@
import { DriveFiles } from '@/models/index.js';
import define from '../../../define.js';
import { ApiError } from '../../../error.js';
import { DriveFiles } from '@/models/index.js';
export const meta = {
tags: ['admin'],
@ -184,5 +184,10 @@ export default define(meta, paramDef, async (ps, me) => {
throw new ApiError(meta.errors.noSuchFile);
}
if (!me.isAdmin) {
delete file.requestIp;
delete file.requestHeaders;
}
return file;
});

31
packages/backend/src/server/api/endpoints/admin/get-user-ips.ts Normal file
View file

@ -0,0 +1,31 @@
import { UserIps } from '@/models/index.js';
import define from '../../define.js';
export const meta = {
tags: ['admin'],
requireCredential: true,
requireAdmin: true,
} as const;
export const paramDef = {
type: 'object',
properties: {
userId: { type: 'string', format: 'misskey:id' },
},
required: ['userId'],
} as const;
// eslint-disable-next-line import/no-default-export
export default define(meta, paramDef, async (ps, me) => {
const ips = await UserIps.find({
where: { userId: ps.userId },
order: { createdAt: 'DESC' },
take: 30,
});
return ips.map(x => ({
ip: x.ip,
createdAt: x.createdAt.toISOString(),
}));
});

8
packages/backend/src/server/api/endpoints/admin/meta.ts
View file

@ -1,7 +1,7 @@
import config from '@/config/index.js';
import define from '../../define.js';
import { fetchMeta } from '@/misc/fetch-meta.js';
import { MAX_NOTE_TEXT_LENGTH } from '@/const.js';
import define from '../../define.js';
export const meta = {
tags: ['meta'],
@ -304,6 +304,10 @@ export const meta = {
type: 'boolean',
optional: true, nullable: false,
},
enableIpLogging: {
type: 'boolean',
optional: true, nullable: false,
},
},
},
} as const;
@ -360,7 +364,6 @@ export default define(meta, paramDef, async (ps, me) => {
pinnedPages: instance.pinnedPages,
pinnedClipId: instance.pinnedClipId,
cacheRemoteFiles: instance.cacheRemoteFiles,
useStarForReactionFallback: instance.useStarForReactionFallback,
pinnedUsers: instance.pinnedUsers,
hiddenTags: instance.hiddenTags,
@ -397,5 +400,6 @@ export default define(meta, paramDef, async (ps, me) => {
objectStorageS3ForcePathStyle: instance.objectStorageS3ForcePathStyle,
deeplAuthKey: instance.deeplAuthKey,
deeplIsPro: instance.deeplIsPro,
enableIpLogging: instance.enableIpLogging,
};
});

4
packages/backend/src/server/api/endpoints/admin/show-user.ts
View file

@ -25,7 +25,7 @@ export const paramDef = {
export default define(meta, paramDef, async (ps, me) => {
const [user, profile] = await Promise.all([
Users.findOneBy({ id: ps.userId }),
UserProfiles.findOneBy({ userId: ps.userId })
UserProfiles.findOneBy({ userId: ps.userId }),
]);
if (user == null || profile == null) {
@ -68,6 +68,8 @@ export default define(meta, paramDef, async (ps, me) => {
isModerator: user.isModerator,
isSilenced: user.isSilenced,
isSuspended: user.isSuspended,
lastActiveDate: user.lastActiveDate,
moderationNote: profile.moderationNote,
signins,
};
});

7
packages/backend/src/server/api/endpoints/admin/update-meta.ts
View file

@ -1,8 +1,8 @@
import define from '../../define.js';
import { Meta } from '@/models/entities/meta.js';
import { insertModerationLog } from '@/services/insert-moderation-log.js';
import { DB_MAX_NOTE_TEXT_LENGTH } from '@/misc/hard-limits.js';
import { db } from '@/db/postgre.js';
import define from '../../define.js';
export const meta = {
tags: ['admin'],
@ -96,6 +96,7 @@ export const paramDef = {
objectStorageUseProxy: { type: 'boolean' },
objectStorageSetPublicRead: { type: 'boolean' },
objectStorageS3ForcePathStyle: { type: 'boolean' },
enableIpLogging: { type: 'boolean' },
},
required: [],
} as const;
@ -396,6 +397,10 @@ export default define(meta, paramDef, async (ps, me) => {
set.deeplIsPro = ps.deeplIsPro;
}
if (ps.enableIpLogging !== undefined) {
set.enableIpLogging = ps.enableIpLogging;
}
await db.transaction(async transactionalEntityManager => {
const metas = await transactionalEntityManager.find(Meta, {
order: {

31
packages/backend/src/server/api/endpoints/admin/update-user-note.ts Normal file
View file

@ -0,0 +1,31 @@
import { UserProfiles, Users } from '@/models/index.js';
import define from '../../define.js';
export const meta = {
tags: ['admin'],
requireCredential: true,
requireModerator: true,
} as const;
export const paramDef = {
type: 'object',
properties: {
userId: { type: 'string', format: 'misskey:id' },
text: { type: 'string' },
},
required: ['userId', 'text'],
} as const;
// eslint-disable-next-line import/no-default-export
export default define(meta, paramDef, async (ps, me) => {
const user = await Users.findOneBy({ id: ps.userId });
if (user == null) {
throw new Error('user not found');
}
await UserProfiles.update({ userId: user.id }, {
moderationNote: ps.text,
});
});

2
packages/backend/src/server/api/endpoints/drive.ts
View file

@ -39,7 +39,7 @@ export default define(meta, paramDef, async (ps, user) => {
const usage = await DriveFiles.calcDriveUsageOf(user.id);
return {
capacity: 1024 * 1024 * instance.localDriveCapacityMb,
capacity: 1024 * 1024 * (user.driveCapacityOverrideMb || instance.localDriveCapacityMb),
usage: usage,
};
});

21
packages/backend/src/server/api/endpoints/drive/files/create.ts
View file

@ -1,10 +1,11 @@
import ms from 'ms';
import { addFile } from '@/services/drive/add-file.js';
import { DriveFiles } from '@/models/index.js';
import { DB_MAX_IMAGE_COMMENT_LENGTH } from '@/misc/hard-limits.js';
import { fetchMeta } from '@/misc/fetch-meta.js';
import define from '../../../define.js';
import { apiLogger } from '../../../logger.js';
import { ApiError } from '../../../error.js';
import { DriveFiles } from '@/models/index.js';
import { DB_MAX_IMAGE_COMMENT_LENGTH } from '@/misc/hard-limits.js';
export const meta = {
tags: ['drive'],
@ -50,7 +51,7 @@ export const paramDef = {
} as const;
// eslint-disable-next-line import/no-default-export
export default define(meta, paramDef, async (ps, user, _, file, cleanup) => {
export default define(meta, paramDef, async (ps, user, _, file, cleanup, ip, headers) => {
// Get 'name' parameter
let name = ps.name || file.originalname;
if (name !== undefined && name !== null) {
@ -66,9 +67,21 @@ export default define(meta, paramDef, async (ps, user, _, file, cleanup) => {
name = null;
}
const meta = await fetchMeta();
try {
// Create file
const driveFile = await addFile({ user, path: file.path, name, comment: ps.comment, folderId: ps.folderId, force: ps.force, sensitive: ps.isSensitive });
const driveFile = await addFile({
user,
path: file.path,
name,
comment: ps.comment,
folderId: ps.folderId,
force: ps.force,
sensitive: ps.isSensitive,
requestIp: meta.enableIpLogging ? ip : null,
requestHeaders: meta.enableIpLogging ? headers : null,
});
return await DriveFiles.pack(driveFile, { self: true });
} catch (e) {
if (e instanceof Error || typeof e === 'string') {

6
packages/backend/src/server/api/endpoints/drive/files/upload-from-url.ts
View file

@ -1,9 +1,9 @@
import ms from 'ms';
import { uploadFromUrl } from '@/services/drive/upload-from-url.js';
import define from '../../../define.js';
import { DriveFiles } from '@/models/index.js';
import { publishMainStream } from '@/services/stream.js';
import { DB_MAX_IMAGE_COMMENT_LENGTH } from '@/misc/hard-limits.js';
import define from '../../../define.js';
export const meta = {
tags: ['drive'],
@ -34,8 +34,8 @@ export const paramDef = {
} as const;
// eslint-disable-next-line import/no-default-export
export default define(meta, paramDef, async (ps, user) => {
uploadFromUrl({ url: ps.url, user, folderId: ps.folderId, sensitive: ps.isSensitive, force: ps.force, comment: ps.comment }).then(file => {
export default define(meta, paramDef, async (ps, user, _1, _2, _3, ip, headers) => {
uploadFromUrl({ url: ps.url, user, folderId: ps.folderId, sensitive: ps.isSensitive, force: ps.force, comment: ps.comment, requestIp: ip, requestHeaders: headers }).then(file => {
DriveFiles.pack(file, { self: true }).then(packedFile => {
publishMainStream(user.id, 'urlUploadFinished', {
marker: ps.marker,

7
packages/backend/src/server/api/endpoints/federation/stats.ts
View file

@ -15,6 +15,7 @@ export const meta = {
export const paramDef = {
type: 'object',
properties: {
limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 },
},
required: [],
} as const;
@ -29,7 +30,7 @@ export default define(meta, paramDef, async (ps) => {
order: {
followersCount: 'DESC',
},
take: 10,
take: ps.limit,
}),
Instances.find({
where: {
@ -38,7 +39,7 @@ export default define(meta, paramDef, async (ps) => {
order: {
followingCount: 'DESC',
},
take: 10,
take: ps.limit,
}),
Followings.count({
where: {
@ -53,7 +54,7 @@ export default define(meta, paramDef, async (ps) => {
]);
const gotSubCount = topSubInstances.map(x => x.followersCount).reduce((a, b) => a + b, 0);
const gotPubCount = topSubInstances.map(x => x.followingCount).reduce((a, b) => a + b, 0);
const gotPubCount = topPubInstances.map(x => x.followingCount).reduce((a, b) => a + b, 0);
return await awaitAll({
topSubInstances: Instances.packMany(topSubInstances),

39
packages/backend/src/server/api/endpoints/fetch-rss.ts Normal file
View file

@ -0,0 +1,39 @@
import Parser from 'rss-parser';
import { getResponse } from '@/misc/fetch.js';
import config from '@/config/index.js';
import define from '../define.js';
const rssParser = new Parser();
export const meta = {
tags: ['meta'],
requireCredential: false,
allowGet: true,
cacheSec: 60 * 3,
} as const;
export const paramDef = {
type: 'object',
properties: {
url: { type: 'string' },
},
required: ['url'],
} as const;
// eslint-disable-next-line import/no-default-export
export default define(meta, paramDef, async (ps) => {
const res = await getResponse({
url: ps.url,
method: 'GET',
headers: Object.assign({
'User-Agent': config.userAgent,
Accept: 'application/rss+xml, */*',
}),
timeout: 5000,
});
const text = await res.text();
return rssParser.parseString(text);
});

15
packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts
View file

@ -60,12 +60,21 @@ export default define(meta, paramDef, async (ps, user) => {
query.setParameters(mutingQuery.getParameters());
//#endregion
const polls = await query.take(ps.limit).skip(ps.offset).getMany();
const polls = await query
.orderBy('poll.noteId', 'DESC')
.take(ps.limit)
.skip(ps.offset)
.getMany();
if (polls.length === 0) return [];
const notes = await Notes.findBy({
id: In(polls.map(poll => poll.noteId)),
const notes = await Notes.find({
where: {
id: In(polls.map(poll => poll.noteId)),
},
order: {
createdAt: 'DESC',
},
});
return await Notes.packMany(notes, user, {

10
packages/backend/src/server/api/endpoints/users.ts
View file

@ -27,6 +27,12 @@ export const paramDef = {
sort: { type: 'string', enum: ['+follower', '-follower', '+createdAt', '-createdAt', '+updatedAt', '-updatedAt'] },
state: { type: 'string', enum: ['all', 'admin', 'moderator', 'adminOrModerator', 'alive'], default: 'all' },
origin: { type: 'string', enum: ['combined', 'local', 'remote'], default: 'local' },
hostname: {
type: 'string',
nullable: true,
default: null,
description: 'The local host is represented with `null`.',
},
},
required: [],
} as const;
@ -48,6 +54,10 @@ export default define(meta, paramDef, async (ps, me) => {
case 'remote': query.andWhere('user.host IS NOT NULL'); break;
}
if (ps.hostname) {
query.andWhere('user.host = :hostname', { hostname: ps.hostname.toLowerCase() });
}
switch (ps.sort) {
case '+follower': query.orderBy('user.followersCount', 'DESC'); break;
case '-follower': query.orderBy('user.followersCount', 'ASC'); break;

2
packages/backend/src/server/api/stream/channels/local-timeline.ts
View file

@ -51,7 +51,7 @@ export default class extends Channel {
}
// 流れてきたNoteがミュートしているユーザーが関わるものだったら無視する
if (iUserRelated(note, this.muting)) return;
if (isUserRelated(note, this.muting)) return;
// 流れてきたNoteがブロックされているユーザーが関わるものだったら無視する
if (isUserRelated(note, this.blocking)) return;

62
packages/backend/src/services/drive/add-file.ts
View file

@ -2,26 +2,26 @@ import * as fs from 'node:fs';
import { v4 as uuid } from 'uuid';
import S3 from 'aws-sdk/clients/s3.js';
import sharp from 'sharp';
import { IsNull } from 'typeorm';
import { publishMainStream, publishDriveStream } from '@/services/stream.js';
import { deleteFile } from './delete-file.js';
import { fetchMeta } from '@/misc/fetch-meta.js';
import { GenerateVideoThumbnail } from './generate-video-thumbnail.js';
import { driveLogger } from './logger.js';
import { IImage, convertSharpToJpeg, convertSharpToWebp, convertSharpToPng } from './image-processor.js';
import { contentDisposition } from '@/misc/content-disposition.js';
import { getFileInfo } from '@/misc/get-file-info.js';
import { DriveFiles, DriveFolders, Users, Instances, UserProfiles } from '@/models/index.js';
import { InternalStorage } from './internal-storage.js';
import { DriveFile } from '@/models/entities/drive-file.js';
import { IRemoteUser, User } from '@/models/entities/user.js';
import { driveChart, perUserDriveChart, instanceChart } from '@/services/chart/index.js';
import { genId } from '@/misc/gen-id.js';
import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js';
import S3 from 'aws-sdk/clients/s3.js';
import { getS3 } from './s3.js';
import sharp from 'sharp';
import { FILE_TYPE_BROWSERSAFE } from '@/const.js';
import { IsNull } from 'typeorm';
import { getS3 } from './s3.js';
import { InternalStorage } from './internal-storage.js';
import { IImage, convertSharpToJpeg, convertSharpToWebp, convertSharpToPng } from './image-processor.js';
import { driveLogger } from './logger.js';
import { GenerateVideoThumbnail } from './generate-video-thumbnail.js';
import { deleteFile } from './delete-file.js';
const logger = driveLogger.createSubLogger('register', 'yellow');
@ -171,7 +171,7 @@ export async function generateAlts(path: string, type: string, generateWeb: bool
}
if (!['image/jpeg', 'image/png', 'image/webp', 'image/svg+xml'].includes(type)) {
logger.debug(`web image and thumbnail not created (not an required file)`);
logger.debug('web image and thumbnail not created (not an required file)');
return {
webpublic: null,
thumbnail: null,
@ -212,7 +212,7 @@ export async function generateAlts(path: string, type: string, generateWeb: bool
let webpublic: IImage | null = null;
if (generateWeb && !satisfyWebpublic) {
logger.info(`creating web image`);
logger.info('creating web image');
try {
if (['image/jpeg', 'image/webp'].includes(type)) {
@ -222,14 +222,14 @@ export async function generateAlts(path: string, type: string, generateWeb: bool
} else if (['image/svg+xml'].includes(type)) {
webpublic = await convertSharpToPng(img, 2048, 2048);
} else {
logger.debug(`web image not created (not an required image)`);
logger.debug('web image not created (not an required image)');
}
} catch (err) {
logger.warn(`web image not created (an error occured)`, err as Error);
logger.warn('web image not created (an error occured)', err as Error);
}
} else {
if (satisfyWebpublic) logger.info(`web image not created (original satisfies webpublic)`);
else logger.info(`web image not created (from remote)`);
if (satisfyWebpublic) logger.info('web image not created (original satisfies webpublic)');
else logger.info('web image not created (from remote)');
}
// #endregion webpublic
@ -240,10 +240,10 @@ export async function generateAlts(path: string, type: string, generateWeb: bool
if (['image/jpeg', 'image/webp', 'image/png', 'image/svg+xml'].includes(type)) {
thumbnail = await convertSharpToWebp(img, 498, 280);
} else {
logger.debug(`thumbnail not created (not an required file)`);
logger.debug('thumbnail not created (not an required file)');
}
} catch (err) {
logger.warn(`thumbnail not created (an error occured)`, err as Error);
logger.warn('thumbnail not created (an error occured)', err as Error);
}
// #endregion thumbnail
@ -276,7 +276,7 @@ async function upload(key: string, stream: fs.ReadStream | Buffer, type: string,
const s3 = getS3(meta);
const upload = s3.upload(params, {
partSize: s3.endpoint?.hostname === 'storage.googleapis.com' ? 500 * 1024 * 1024 : 8 * 1024 * 1024,
partSize: s3.endpoint.hostname === 'storage.googleapis.com' ? 500 * 1024 * 1024 : 8 * 1024 * 1024,
});
const result = await upload.promise();
@ -307,7 +307,7 @@ async function deleteOldFile(user: IRemoteUser) {
type AddFileArgs = {
/** User who wish to add file */
user: { id: User['id']; host: User['host'] } | null;
user: { id: User['id']; host: User['host']; driveCapacityOverrideMb: User['driveCapacityOverrideMb'] } | null;
/** File path */
path: string;
/** Name */
@ -326,6 +326,9 @@ type AddFileArgs = {
uri?: string | null;
/** Mark file as sensitive */
sensitive?: boolean | null;
requestIp?: string | null;
requestHeaders?: Record<string, string> | null;
};
/**
@ -342,7 +345,9 @@ export async function addFile({
isLink = false,
url = null,
uri = null,
sensitive = null
sensitive = null,
requestIp = null,
requestHeaders = null,
}: AddFileArgs): Promise<DriveFile> {
const info = await getFileInfo(path);
logger.info(`${JSON.stringify(info)}`);
@ -366,9 +371,16 @@ export async function addFile({
//#region Check drive usage
if (user && !isLink) {
const usage = await DriveFiles.calcDriveUsageOf(user);
const u = await Users.findOneBy({ id: user.id });
const instance = await fetchMeta();
const driveCapacity = 1024 * 1024 * (Users.isLocalUser(user) ? instance.localDriveCapacityMb : instance.remoteDriveCapacityMb);
let driveCapacity = 1024 * 1024 * (Users.isLocalUser(user) ? instance.localDriveCapacityMb : instance.remoteDriveCapacityMb);
if (Users.isLocalUser(user) && u?.driveCapacityOverrideMb != null) {
driveCapacity = 1024 * 1024 * u.driveCapacityOverrideMb;
logger.debug('drive capacity override applied');
logger.debug(`overrideCap: ${driveCapacity}bytes, usage: ${usage}bytes, u+s: ${usage + info.size}bytes`);
}
logger.debug(`drive usage is ${usage} (max: ${driveCapacity})`);
@ -427,11 +439,13 @@ export async function addFile({
file.properties = properties;
file.blurhash = info.blurhash || null;
file.isLink = isLink;
file.requestIp = requestIp;
file.requestHeaders = requestHeaders;
file.isSensitive = user
? Users.isLocalUser(user) && profile!.alwaysMarkNsfw ? true :
(sensitive !== null && sensitive !== undefined)
? sensitive
: false
(sensitive !== null && sensitive !== undefined)
? sensitive
: false
: false;
if (url !== null) {

10
packages/backend/src/services/drive/upload-from-url.ts
View file

@ -1,12 +1,12 @@
import { URL } from 'node:url';
import { addFile } from './add-file.js';
import { User } from '@/models/entities/user.js';
import { driveLogger } from './logger.js';
import { createTemp } from '@/misc/create-temp.js';
import { downloadUrl } from '@/misc/download-url.js';
import { DriveFolder } from '@/models/entities/drive-folder.js';
import { DriveFile } from '@/models/entities/drive-file.js';
import { DriveFiles } from '@/models/index.js';
import { driveLogger } from './logger.js';
import { addFile } from './add-file.js';
const logger = driveLogger.createSubLogger('downloader');
@ -19,6 +19,8 @@ type Args = {
force?: boolean;
isLink?: boolean;
comment?: string | null;
requestIp?: string | null;
requestHeaders?: Record<string, string> | null;
};
export async function uploadFromUrl({
@ -30,6 +32,8 @@ export async function uploadFromUrl({
force = false,
isLink = false,
comment = null,
requestIp = null,
requestHeaders = null,
}: Args): Promise<DriveFile> {
let name = new URL(url).pathname.split('/').pop() || null;
if (name == null || !DriveFiles.validateFileName(name)) {
@ -49,7 +53,7 @@ export async function uploadFromUrl({
// write content at URL to temp file
await downloadUrl(url, path);
const driveFile = await addFile({ user, path, name, comment, folderId, force, isLink, url, uri, sensitive });
const driveFile = await addFile({ user, path, name, comment, folderId, force, isLink, url, uri, sensitive, requestIp, requestHeaders });
logger.succ(`Got: ${driveFile.id}`);
return driveFile!;
} catch (e) {

1295
packages/backend/test/streaming.ts
View file

File diff suppressed because it is too large Load diff

6
packages/backend/test/utils.ts
View file

@ -186,7 +186,7 @@ export function connectStream(user: any, channel: string, listener: (message: Re
});
}
export const waitFire = async (user: any, channel: string, trgr: () => any, cond: (msg: Record<string, any>) => boolean) => {
export const waitFire = async (user: any, channel: string, trgr: () => any, cond: (msg: Record<string, any>) => boolean, params?: any) => {
return new Promise<boolean>(async (res, rej) => {
let timer: NodeJS.Timeout;
@ -198,7 +198,7 @@ export const waitFire = async (user: any, channel: string, trgr: () => any, cond
if (timer) clearTimeout(timer);
res(true);
}
});
}, params);
} catch (e) {
rej(e);
}
@ -208,7 +208,7 @@ export const waitFire = async (user: any, channel: string, trgr: () => any, cond
timer = setTimeout(() => {
ws.close();
res(false);
}, 5000);
}, 3000);
try {
await trgr();