heartles/egirlskey
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity

Merge branch 'develop' into future

Browse source
This commit is contained in:
dakkar 2024-05-31 11:24:00 +01:00
commit 4ddee7b01e
23 changed files with 557 additions and 84 deletions
Download patch file Download diff file Expand all files Collapse all files

1
packages/backend/src/boot/common.ts
View file

@ -36,7 +36,6 @@ export async function jobQueue() {
});
jobQueue.get(QueueProcessorService).start();
jobQueue.get(ChartManagementService).start();
return jobQueue;
}

2
packages/backend/src/boot/entry.ts
View file

@ -75,7 +75,7 @@ async function main() {
ev.mount();
}
}
if (cluster.isWorker || envOption.disableClustering) {
if (cluster.isWorker) {
await workerMain();
}

122
packages/backend/src/config.ts
View file

@ -212,6 +212,8 @@ export function loadConfig(): Config {
{} as Source,
) as Source;
applyEnvOverrides(config);
const url = tryCreateUrl(config.url);
const version = meta.version;
const host = url.host;
@ -304,3 +306,123 @@ function convertRedisOptions(options: RedisOptionsSource, host: string): RedisOp
db: options.db ?? 0,
};
}
/*
this function allows overriding any string-valued config option with
a sensible-named environment variable
e.g. `MK_CONFIG_MEILISEARCH_APIKEY` sets `config.meilisearch.apikey`
you can also override a single `dbSlave` value,
e.g. `MK_CONFIG_DBSLAVES_1_PASS` sets the password for the 2nd
database replica (the first one would be
`MK_CONFIG_DBSLAVES_0_PASS`); in this case, `config.dbSlaves` must
be set to an array of the right size already in the file
values can be read from files, too: setting `MK_DB_PASS_FILE` to
`/some/file` would set the main database password to the contents of
`/some/file` (trimmed of whitespaces)
*/
function applyEnvOverrides(config: Source) {
// these inner functions recurse through the config structure, using
// the given steps, building the env variable name
function _apply_top(steps: (string | number)[]) {
_walk('', [], steps);
}
function _walk(name: string, path: (string | number)[], steps: (string | number)[]) {
// are there more steps after this one? recurse
if (steps.length > 1) {
const thisStep = steps.shift();
if (thisStep === null || thisStep === undefined) return;
// if a step is not a simple value, iterate through it
if (typeof thisStep === 'object') {
for (const thisOneStep of thisStep) {
_descend(name, path, thisOneStep, steps);
}
} else {
_descend(name, path, thisStep, steps);
}
// the actual override has happened at the bottom of the
// recursion, we're done
return;
}
// this is the last step, same thing as above
const lastStep = steps[0];
if (typeof lastStep === 'object') {
for (const lastOneStep of lastStep) {
_lastBit(name, path, lastOneStep);
}
} else {
_lastBit(name, path, lastStep);
}
}
function _step2name(step: string|number): string {
return step.toString().replaceAll(/[^a-z0-9]+/gi,'').toUpperCase();
}
// this recurses down, bailing out if there's no config to override
function _descend(name: string, path: (string | number)[], thisStep: string | number, steps: (string | number)[]) {
name = `${name}${_step2name(thisStep)}_`;
path = [ ...path, thisStep ];
_walk(name, path, steps);
}
// this is the bottom of the recursion: look at the environment and
// set the value
function _lastBit(name: string, path: (string | number)[], lastStep: string | number) {
name = `MK_CONFIG_${name}${_step2name(lastStep)}`;
const val = process.env[name];
if (val != null && val != undefined) {
_assign(path, lastStep, val);
}
const file = process.env[`${name}_FILE`];
if (file) {
_assign(path, lastStep, fs.readFileSync(file, 'utf-8').trim());
}
}
const alwaysStrings = { 'chmodSocket': 1 };
function _assign(path: (string | number)[], lastStep: string | number, value: string) {
let thisConfig = config;
for (const step of path) {
if (!thisConfig[step]) {
thisConfig[step] = {};
}
thisConfig = thisConfig[step];
}
if (!alwaysStrings[lastStep]) {
if (value.match(/^[0-9]+$/)) {
value = parseInt(value);
} else if (value.match(/^(true|false)$/i)) {
value = !!value.match(/^true$/i);
}
}
thisConfig[lastStep] = value;
}
// these are all the settings that can be overridden
_apply_top([['url', 'port', 'socket', 'chmodSocket', 'disableHsts']]);
_apply_top(['db', ['host', 'port', 'db', 'user', 'pass']]);
_apply_top(['dbSlaves', config.dbSlaves?.keys(), ['host', 'port', 'db', 'user', 'pass']]);
_apply_top([
['redis', 'redisForPubsub', 'redisForJobQueue', 'redisForTimelines'],
['host','port','username','pass','db','prefix'],
]);
_apply_top(['meilisearch', ['host', 'port', 'apikey', 'ssl', 'index', 'scope']]);
_apply_top([['clusterLimit', 'deliverJobConcurrency', 'inboxJobConcurrency', 'relashionshipJobConcurrency', 'deliverJobPerSec', 'inboxJobPerSec', 'relashionshipJobPerSec', 'deliverJobMaxAttempts', 'inboxJobMaxAttempts']]);
_apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaProxy', 'videoThumbnailGenerator']]);
_apply_top([['maxFileSize', 'maxNoteLength', 'pidFile']]);
}

8
packages/backend/src/core/AntennaService.ts
View file

@ -133,13 +133,17 @@ export class AntennaService implements OnApplicationShutdown {
const { username, host } = Acct.parse(x);
return this.utilityService.getFullApAccount(username, host).toLowerCase();
});
if (!accts.includes(this.utilityService.getFullApAccount(noteUser.username, noteUser.host).toLowerCase())) return false;
const matchUser = this.utilityService.getFullApAccount(noteUser.username, noteUser.host).toLowerCase();
const matchWildcard = this.utilityService.getFullApAccount('*', noteUser.host).toLowerCase();
if (!accts.includes(matchUser) && !accts.includes(matchWildcard)) return false;
} else if (antenna.src === 'users_blacklist') {
const accts = antenna.users.map(x => {
const { username, host } = Acct.parse(x);
return this.utilityService.getFullApAccount(username, host).toLowerCase();
});
if (accts.includes(this.utilityService.getFullApAccount(noteUser.username, noteUser.host).toLowerCase())) return false;
const matchUser = this.utilityService.getFullApAccount(noteUser.username, noteUser.host).toLowerCase();
const matchWildcard = this.utilityService.getFullApAccount('*', noteUser.host).toLowerCase();
if (accts.includes(matchUser) || accts.includes(matchWildcard)) return false;
}
const keywords = antenna.keywords

3
packages/backend/src/core/DriveService.ts
View file

@ -632,7 +632,8 @@ export class DriveService {
@bindThis
public async updateFile(file: MiDriveFile, values: Partial<MiDriveFile>, updater: MiUser) {
const alwaysMarkNsfw = (await this.roleService.getUserPolicies(file.userId)).alwaysMarkNsfw;
const profile = await this.userProfilesRepository.findOneBy({ userId: file.userId });
const alwaysMarkNsfw = (await this.roleService.getUserPolicies(file.userId)).alwaysMarkNsfw || (profile !== null && profile!.alwaysMarkNsfw);
if (values.name != null && !this.driveFileEntityService.validateFileName(values.name)) {
throw new DriveService.InvalidFileNameError();

18
packages/backend/src/core/NoteEditService.ts
View file

@ -699,6 +699,24 @@ export class NoteEditService implements OnApplicationShutdown {
dm.addFollowersRecipe();
}
if (['public', 'home'].includes(note.visibility)) {
// Send edit event to all users who replied to,
// renoted a post or reacted to a note.
const noteId = note.id;
const users = await this.usersRepository.createQueryBuilder()
.where(
'id IN (SELECT "userId" FROM note WHERE "replyId" = :noteId OR "renoteId" = :noteId UNION SELECT "userId" FROM note_reaction WHERE "noteId" = :noteId)',
{ noteId },
)
.andWhere('host IS NOT NULL')
.getMany();
for (const u of users) {
// User was verified to be remote by checking
// whether host IS NOT NULL in SQL query.
dm.addDirectRecipe(u as MiRemoteUser);
}
}
if (['public'].includes(note.visibility)) {
this.relayService.deliverToRelays(user, noteActivity);
}

31
packages/backend/src/server/api/endpoints/channels/timeline.ts
View file

@ -51,6 +51,12 @@ export const paramDef = {
sinceDate: { type: 'integer' },
untilDate: { type: 'integer' },
allowPartial: { type: 'boolean', default: false }, // true is recommended but for compatibility false by default
withRenotes: { type: 'boolean', default: true },
withFiles: {
type: 'boolean',
default: false,
description: 'Only show notes that have attached files.',
},
},
required: ['channelId'],
} as const;
@ -89,7 +95,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
if (me) this.activeUsersChart.read(me);
if (!serverSettings.enableFanoutTimeline) {
return await this.noteEntityService.packMany(await this.getFromDb({ untilId, sinceId, limit: ps.limit, channelId: channel.id }, me), me);
return await this.noteEntityService.packMany(await this.getFromDb({ untilId, sinceId, limit: ps.limit, channelId: channel.id, withFiles: ps.withFiles, withRenotes: ps.withRenotes }, me), me);
}
return await this.fanoutTimelineEndpointService.timeline({
@ -100,9 +106,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
me,
useDbFallback: true,
redisTimelines: [`channelTimeline:${channel.id}`],
excludePureRenotes: false,
excludePureRenotes: !ps.withRenotes,
excludeNoFiles: ps.withFiles,
dbFallback: async (untilId, sinceId, limit) => {
return await this.getFromDb({ untilId, sinceId, limit, channelId: channel.id }, me);
return await this.getFromDb({ untilId, sinceId, limit, channelId: channel.id, withFiles: ps.withFiles, withRenotes: ps.withRenotes }, me);
},
});
});
@ -112,7 +119,9 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
untilId: string | null,
sinceId: string | null,
limit: number,
channelId: string
channelId: string,
withFiles: boolean,
withRenotes: boolean,
}, me: MiLocalUser | null) {
//#region fallback to database
const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId)
@ -128,6 +137,20 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
this.queryService.generateMutedUserQuery(query, me);
this.queryService.generateBlockedUserQuery(query, me);
}
if (ps.withRenotes === false) {
query.andWhere(new Brackets(qb => {
qb.orWhere('note.renoteId IS NULL');
qb.orWhere(new Brackets(qb => {
qb.orWhere('note.text IS NOT NULL');
qb.orWhere('note.fileIds != \'{}\'');
}));
}));
}
if (ps.withFiles) {
query.andWhere('note.fileIds != \'{}\'');
}
//#endregion
return await query.limit(ps.limit).getMany();

4
packages/backend/src/server/api/mastodon/converters.ts
View file

@ -146,8 +146,8 @@ export class MastoConverters {
display_name: user.name ?? user.username,
locked: user.isLocked,
created_at: this.idService.parse(user.id).date.toISOString(),
followers_count: user.followersCount,
following_count: user.followingCount,
followers_count: profile?.followersVisibility === 'public' ? user.followersCount : 0,
following_count: profile?.followingVisibility === 'public' ? user.followingCount : 0,
statuses_count: user.notesCount,
note: profile?.description ?? '',
url: user.uri ?? acctUrl,

8
packages/backend/src/server/api/stream/channels/channel.ts
View file

@ -15,6 +15,8 @@ class ChannelChannel extends Channel {
public static shouldShare = false;
public static requireCredential = false as const;
private channelId: string;
private withFiles: boolean;
private withRenotes: boolean;
constructor(
private noteEntityService: NoteEntityService,
@ -29,6 +31,8 @@ class ChannelChannel extends Channel {
@bindThis
public async init(params: any) {
this.channelId = params.channelId as string;
this.withFiles = params.withFiles ?? false;
this.withRenotes = params.withRenotes ?? true;
// Subscribe stream
this.subscriber.on('notesStream', this.onNote);
@ -38,6 +42,10 @@ class ChannelChannel extends Channel {
private async onNote(note: Packed<'Note'>) {
if (note.channelId !== this.channelId) return;
if (this.withFiles && (note.fileIds == null || note.fileIds.length === 0)) return;
if (note.renote && note.text == null && (note.fileIds == null || note.fileIds.length === 0) && !this.withRenotes) return;
if (this.isNoteMutedOrBlocked(note)) return;
if (this.user && isRenotePacked(note) && !isQuotePacked(note)) {