feat: Introduce Meilisearch (#10755)

* wip

* wip

* Update SearchService.ts

* Update SearchService.ts

* wip

* wip

* Update SearchService.ts

* Update CHANGELOG.md

* wip

* Update SearchService.ts

* Update docker-compose.yml.example
This commit is contained in:
syuilo 2023-05-05 08:52:14 +09:00 committed by GitHub
parent 5f62cefe31
commit 5c08f2b93b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 257 additions and 91 deletions

View file

@ -95,15 +95,13 @@ redis:
# #prefix: example-prefix
# #db: 1
# ┌─────────────────────────────
#───┘ Elasticsearch configuration └─────────────────────────────
# ┌───────────────────────────
#───┘ MeiliSearch configuration └─────────────────────────────
#elasticsearch:
# host: localhost
# port: 9200
# ssl: false
# user:
# pass:
#meilisearch:
# host: meilisearch
# port: 7700
# apiKey: ''
# ┌───────────────┐
#───┘ ID generation └───────────────────────────────────────────

View file

@ -95,15 +95,13 @@ redis:
# #prefix: example-prefix
# #db: 1
# ┌─────────────────────────────
#───┘ Elasticsearch configuration └─────────────────────────────
# ┌───────────────────────────
#───┘ MeiliSearch configuration └─────────────────────────────
#elasticsearch:
#meilisearch:
# host: localhost
# port: 9200
# ssl: false
# user:
# pass:
# port: 7700
# apiKey: ''
# ┌───────────────┐
#───┘ ID generation └───────────────────────────────────────────

View file

@ -95,15 +95,13 @@ redis:
# #prefix: example-prefix
# #db: 1
# ┌─────────────────────────────
#───┘ Elasticsearch configuration └─────────────────────────────
# ┌───────────────────────────
#───┘ MeiliSearch configuration └─────────────────────────────
#elasticsearch:
# host: localhost
# port: 9200
# ssl: false
# user:
# pass:
#meilisearch:
# host: meilisearch
# port: 7700
# apiKey: ''
# ┌───────────────┐
#───┘ ID generation └───────────────────────────────────────────

View file

@ -8,7 +8,6 @@ build/
built/
db/
docker-compose.yml
elasticsearch/
node_modules/
packages/*/node_modules
redis/

2
.gitignore vendored
View file

@ -44,7 +44,7 @@ built
/data
/.cache-loader
/db
/elasticsearch
/meili_data
npm-debug.log
*.pem
run.bat

View file

@ -17,6 +17,7 @@
- Node.js 18.6.0以上が必要になりました
### General
- Meilisearchを全文検索に使用できるようになりました
- 新規登録前に簡潔なルールをユーザーに表示できる、サーバールール機能を追加
- ユーザーへの自分用メモ機能
* ユーザーに対して、自分だけが見られるメモを追加できるようになりました。

View file

@ -116,15 +116,13 @@ redis:
# #prefix: example-prefix
# #db: 1
# ┌─────────────────────────────
#───┘ Elasticsearch configuration └─────────────────────────────
# ┌───────────────────────────
#───┘ MeiliSearch configuration └─────────────────────────────
#elasticsearch:
#meilisearch:
# host: localhost
# port: 9200
# ssl: false
# user:
# pass:
# port: 7700
# apiKey: ''
# ┌───────────────┐
#───┘ ID generation └───────────────────────────────────────────

View file

@ -7,7 +7,7 @@ services:
links:
- db
- redis
# - es
# - meilisearch
depends_on:
db:
condition: service_healthy
@ -48,16 +48,18 @@ services:
interval: 5s
retries: 20
# es:
# meilisearch:
# restart: always
# image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.4.2
# image: getmeili/meilisearch:v1.1.1
# environment:
# - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
# - "TAKE_FILE_OWNERSHIP=111"
# - MEILI_NO_ANALYTICS=true
# - MEILI_ENV=production
# env_file:
# - .config/meilisearch.env
# networks:
# - internal_network
# volumes:
# - ./elasticsearch:/usr/share/elasticsearch/data
# - ./meili_data:/meili_data
networks:
internal_network:

View file

@ -91,6 +91,7 @@
"jsdom": "21.1.1",
"json5": "2.2.3",
"jsonld": "8.1.1",
"meilisearch": "0.32.3",
"jsrsasign": "10.8.6",
"mfm-js": "0.23.3",
"mime-types": "2.1.35",

View file

@ -2,6 +2,7 @@ import { setTimeout } from 'node:timers/promises';
import { Global, Inject, Module } from '@nestjs/common';
import * as Redis from 'ioredis';
import { DataSource } from 'typeorm';
import { MeiliSearch } from 'meilisearch';
import { DI } from './di-symbols.js';
import { loadConfig } from './config.js';
import { createPostgresDataSource } from './postgres.js';
@ -22,6 +23,21 @@ const $db: Provider = {
inject: [DI.config],
};
const $meilisearch: Provider = {
provide: DI.meilisearch,
useFactory: (config) => {
if (config.meilisearch) {
return new MeiliSearch({
host: `http://${config.meilisearch.host}:${config.meilisearch.port}`,
apiKey: config.meilisearch.apiKey,
});
} else {
return null;
}
},
inject: [DI.config],
};
const $redis: Provider = {
provide: DI.redis,
useFactory: (config) => {
@ -73,8 +89,8 @@ const $redisForSub: Provider = {
@Global()
@Module({
imports: [RepositoryModule],
providers: [$config, $db, $redis, $redisForPub, $redisForSub],
exports: [$config, $db, $redis, $redisForPub, $redisForSub, RepositoryModule],
providers: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub],
exports: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, RepositoryModule],
})
export class GlobalModule implements OnApplicationShutdown {
constructor(

View file

@ -57,13 +57,10 @@ export type Source = {
db?: number;
prefix?: string;
};
elasticsearch: {
meilisearch?: {
host: string;
port: number;
ssl?: boolean;
user?: string;
pass?: string;
index?: string;
port: string;
apiKey: string;
};
proxy?: string;
@ -139,6 +136,7 @@ const path = process.env.MISSKEY_CONFIG_YML
: process.env.NODE_ENV === 'test'
? resolve(dir, 'test.yml')
: resolve(dir, 'default.yml');
export function loadConfig() {
const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../built/meta.json`, 'utf-8'));
const clientManifestExists = fs.existsSync(_dirname + '/../../../built/_vite_/manifest.json');

View file

@ -50,6 +50,7 @@ import { WebhookService } from './WebhookService.js';
import { ProxyAccountService } from './ProxyAccountService.js';
import { UtilityService } from './UtilityService.js';
import { FileInfoService } from './FileInfoService.js';
import { SearchService } from './SearchService.js';
import { ChartLoggerService } from './chart/ChartLoggerService.js';
import FederationChart from './chart/charts/federation.js';
import NotesChart from './chart/charts/notes.js';
@ -171,6 +172,8 @@ const $VideoProcessingService: Provider = { provide: 'VideoProcessingService', u
const $WebhookService: Provider = { provide: 'WebhookService', useExisting: WebhookService };
const $UtilityService: Provider = { provide: 'UtilityService', useExisting: UtilityService };
const $FileInfoService: Provider = { provide: 'FileInfoService', useExisting: FileInfoService };
const $SearchService: Provider = { provide: 'SearchService', useExisting: SearchService };
const $ChartLoggerService: Provider = { provide: 'ChartLoggerService', useExisting: ChartLoggerService };
const $FederationChart: Provider = { provide: 'FederationChart', useExisting: FederationChart };
const $NotesChart: Provider = { provide: 'NotesChart', useExisting: NotesChart };
@ -295,6 +298,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
WebhookService,
UtilityService,
FileInfoService,
SearchService,
ChartLoggerService,
FederationChart,
NotesChart,
@ -413,6 +417,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
$WebhookService,
$UtilityService,
$FileInfoService,
$SearchService,
$ChartLoggerService,
$FederationChart,
$NotesChart,
@ -532,6 +537,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
WebhookService,
UtilityService,
FileInfoService,
SearchService,
FederationChart,
NotesChart,
UsersChart,
@ -649,6 +655,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
$WebhookService,
$UtilityService,
$FileInfoService,
$SearchService,
$FederationChart,
$NotesChart,
$UsersChart,

View file

@ -46,6 +46,7 @@ import { bindThis } from '@/decorators.js';
import { DB_MAX_NOTE_TEXT_LENGTH } from '@/const.js';
import { RoleService } from '@/core/RoleService.js';
import { MetaService } from '@/core/MetaService.js';
import { SearchService } from '@/core/SearchService.js';
const mutedWordsCache = new MemorySingleCache<{ userId: UserProfile['userId']; mutedWords: UserProfile['mutedWords']; }[]>(1000 * 60 * 5);
@ -198,6 +199,7 @@ export class NoteCreateService implements OnApplicationShutdown {
private apRendererService: ApRendererService,
private roleService: RoleService,
private metaService: MetaService,
private searchService: SearchService,
private notesChart: NotesChart,
private perUserNotesChart: PerUserNotesChart,
private activeUsersChart: ActiveUsersChart,
@ -728,17 +730,9 @@ export class NoteCreateService implements OnApplicationShutdown {
@bindThis
private index(note: Note) {
if (note.text == null || this.config.elasticsearch == null) return;
/*
es!.index({
index: this.config.elasticsearch.index ?? 'misskey_note',
id: note.id.toString(),
body: {
text: normalizeForSearch(note.text),
userId: note.userId,
userHost: note.userHost,
},
});*/
if (note.text == null && note.cw == null) return;
this.searchService.indexNote(note);
}
@bindThis

View file

@ -0,0 +1,166 @@
import { Inject, Injectable } from '@nestjs/common';
import { In } from 'typeorm';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { bindThis } from '@/decorators.js';
import { Note } from '@/models/entities/Note.js';
import { User } from '@/models/index.js';
import type { NotesRepository } from '@/models/index.js';
import { sqlLikeEscape } from '@/misc/sql-like-escape.js';
import { QueryService } from '@/core/QueryService.js';
import { IdService } from '@/core/IdService.js';
import type { Index, MeiliSearch } from 'meilisearch';
type K = string;
type V = string | number | boolean;
type Q =
{ op: '=', k: K, v: V } |
{ op: '!=', k: K, v: V } |
{ op: '>', k: K, v: number } |
{ op: '<', k: K, v: number } |
{ op: '>=', k: K, v: number } |
{ op: '<=', k: K, v: number } |
{ op: 'and', qs: Q[] } |
{ op: 'or', qs: Q[] } |
{ op: 'not', q: Q };
function compileValue(value: V): string {
if (typeof value === 'string') {
return `'${value}'`; // TODO: escape
} else if (typeof value === 'number') {
return value.toString();
} else if (typeof value === 'boolean') {
return value.toString();
}
throw new Error('unrecognized value');
}
function compileQuery(q: Q): string {
switch (q.op) {
case '=': return `(${q.k} = ${compileValue(q.v)})`;
case '!=': return `(${q.k} != ${compileValue(q.v)})`;
case '>': return `(${q.k} > ${compileValue(q.v)})`;
case '<': return `(${q.k} < ${compileValue(q.v)})`;
case '>=': return `(${q.k} >= ${compileValue(q.v)})`;
case '<=': return `(${q.k} <= ${compileValue(q.v)})`;
case 'and': return q.qs.length === 0 ? '' : `(${ q.qs.map(_q => compileQuery(_q)).join(' AND ') })`;
case 'or': return q.qs.length === 0 ? '' : `(${ q.qs.map(_q => compileQuery(_q)).join(' OR ') })`;
case 'not': return `(NOT ${compileQuery(q.q)})`;
default: throw new Error('unrecognized query operator');
}
}
@Injectable()
export class SearchService {
private meilisearchNoteIndex: Index | null = null;
constructor(
@Inject(DI.config)
private config: Config,
@Inject(DI.meilisearch)
private meilisearch: MeiliSearch | null,
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
private queryService: QueryService,
private idService: IdService,
) {
if (meilisearch) {
this.meilisearchNoteIndex = meilisearch.index('notes');
this.meilisearchNoteIndex.updateSettings({
searchableAttributes: [
'text',
'cw',
],
sortableAttributes: [
'createdAt',
],
filterableAttributes: [
'createdAt',
'userId',
'userHost',
'channelId',
],
typoTolerance: {
enabled: false,
},
pagination: {
maxTotalHits: 10000,
},
});
}
}
@bindThis
public async indexNote(note: Note): Promise<void> {
if (this.meilisearch) {
this.meilisearchNoteIndex!.addDocuments([{
id: note.id,
createdAt: note.createdAt.getTime(),
userId: note.userId,
userHost: note.userHost,
channelId: note.channelId,
cw: note.cw,
text: note.text,
}], {
primaryKey: 'id',
});
}
}
@bindThis
public async searchNote(q: string, me: User | null, opts: {
userId?: Note['userId'] | null;
channelId?: Note['channelId'] | null;
}, pagination: {
untilId?: Note['id'];
sinceId?: Note['id'];
limit?: number;
}): Promise<Note[]> {
if (this.meilisearch) {
const filter: Q = {
op: 'and',
qs: [],
};
if (pagination.untilId) filter.qs.push({ op: '<', k: 'createdAt', v: this.idService.parse(pagination.untilId).date.getTime() });
if (pagination.sinceId) filter.qs.push({ op: '>', k: 'createdAt', v: this.idService.parse(pagination.sinceId).date.getTime() });
if (opts.userId) filter.qs.push({ op: '=', k: 'userId', v: opts.userId });
if (opts.channelId) filter.qs.push({ op: '=', k: 'channelId', v: opts.channelId });
const res = await this.meilisearchNoteIndex!.search(q, {
sort: ['createdAt:desc'],
matchingStrategy: 'all',
attributesToRetrieve: ['id', 'createdAt'],
filter: compileQuery(filter),
limit: pagination.limit,
});
if (res.hits.length === 0) return [];
return await this.notesRepository.findBy({
id: In(res.hits.map(x => x.id)),
});
} else {
const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId);
if (opts.userId) {
query.andWhere('note.userId = :userId', { userId: opts.userId });
} else if (opts.channelId) {
query.andWhere('note.channelId = :channelId', { channelId: opts.channelId });
}
query
.andWhere('note.text ILIKE :q', { q: `%${ sqlLikeEscape(q) }%` })
.innerJoinAndSelect('note.user', 'user')
.leftJoinAndSelect('note.reply', 'reply')
.leftJoinAndSelect('note.renote', 'renote')
.leftJoinAndSelect('reply.user', 'replyUser')
.leftJoinAndSelect('renote.user', 'renoteUser');
this.queryService.generateVisibilityQuery(query, me);
if (me) this.queryService.generateMutedUserQuery(query, me);
if (me) this.queryService.generateBlockedUserQuery(query, me);
return await query.take(pagination.limit).getMany();
}
}
}

View file

@ -1,6 +1,7 @@
export const DI = {
config: Symbol('config'),
db: Symbol('db'),
meilisearch: Symbol('meilisearch'),
redis: Symbol('redis'),
redisForPub: Symbol('redisForPub'),
redisForSub: Symbol('redisForSub'),

View file

@ -201,10 +201,6 @@ export const meta = {
type: 'boolean',
optional: false, nullable: false,
},
elasticsearch: {
type: 'boolean',
optional: false, nullable: false,
},
hcaptcha: {
type: 'boolean',
optional: false, nullable: false,
@ -331,7 +327,6 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
response.features = {
registration: !instance.disableRegistration,
emailRequiredForSignup: instance.emailRequiredForSignup,
elasticsearch: this.config.elasticsearch ? true : false,
hcaptcha: instance.enableHcaptcha,
recaptcha: instance.enableRecaptcha,
turnstile: instance.enableTurnstile,

View file

@ -1,11 +1,10 @@
import { Inject, Injectable } from '@nestjs/common';
import type { NotesRepository } from '@/models/index.js';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { QueryService } from '@/core/QueryService.js';
import { SearchService } from '@/core/SearchService.js';
import { NoteEntityService } from '@/core/entities/NoteEntityService.js';
import type { Config } from '@/config.js';
import { DI } from '@/di-symbols.js';
import { sqlLikeEscape } from '@/misc/sql-like-escape.js';
import { RoleService } from '@/core/RoleService.js';
import { ApiError } from '../../error.js';
@ -61,11 +60,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
@Inject(DI.config)
private config: Config,
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
private noteEntityService: NoteEntityService,
private queryService: QueryService,
private searchService: SearchService,
private roleService: RoleService,
) {
super(meta, paramDef, async (ps, me) => {
@ -74,27 +70,14 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
throw new ApiError(meta.errors.unavailable);
}
const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId);
if (ps.userId) {
query.andWhere('note.userId = :userId', { userId: ps.userId });
} else if (ps.channelId) {
query.andWhere('note.channelId = :channelId', { channelId: ps.channelId });
}
query
.andWhere('note.text ILIKE :q', { q: `%${ sqlLikeEscape(ps.query) }%` })
.innerJoinAndSelect('note.user', 'user')
.leftJoinAndSelect('note.reply', 'reply')
.leftJoinAndSelect('note.renote', 'renote')
.leftJoinAndSelect('reply.user', 'replyUser')
.leftJoinAndSelect('renote.user', 'renoteUser');
this.queryService.generateVisibilityQuery(query, me);
if (me) this.queryService.generateMutedUserQuery(query, me);
if (me) this.queryService.generateBlockedUserQuery(query, me);
const notes = await query.take(ps.limit).getMany();
const notes = await this.searchService.searchNote(ps.query, me, {
userId: ps.userId,
channelId: ps.channelId,
}, {
untilId: ps.untilId,
sinceId: ps.sinceId,
limit: ps.limit,
});
return await this.noteEntityService.packMany(notes, me);
});

View file

@ -229,6 +229,9 @@ importers:
jsrsasign:
specifier: 10.8.6
version: 10.8.6
meilisearch:
specifier: 0.32.3
version: 0.32.3
mfm-js:
specifier: 0.23.3
version: 0.23.3
@ -9582,7 +9585,6 @@ packages:
node-fetch: 2.6.7
transitivePeerDependencies:
- encoding
dev: true
/cross-spawn@5.1.0:
resolution: {integrity: sha512-pTgQJ5KC0d2hcY8eyL1IzlBPYjTkyH72XRZPnLyKus2mBfNjQs3klqbJU2VILqZryAZUt9JOb3h/mWMy23/f5A==}
@ -14496,6 +14498,14 @@ packages:
engines: {node: '>= 0.6'}
dev: true
/meilisearch@0.32.3:
resolution: {integrity: sha512-EOgfBuRE5SiIPIpEDYe2HO0D7a4z5bexIgaAdJFma/dH5hx1kwO+u/qb2g3qKyjG+iA3l8MlmTj/Xd72uahaAw==}
dependencies:
cross-fetch: 3.1.5
transitivePeerDependencies:
- encoding
dev: false
/memoizerific@1.11.3:
resolution: {integrity: sha512-/EuHYwAPdLtXwAwSZkh/Gutery6pD2KYd44oQLhAvQp/50mpyduZh8Q7PYHXTCJ+wuXxt7oij2LXyIJOOYFPog==}
dependencies:
@ -14657,6 +14667,7 @@ packages:
/minimist@1.2.7:
resolution: {integrity: sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==}
dev: false
/minimist@1.2.8:
resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==}
@ -19700,7 +19711,7 @@ packages:
axios: 0.27.2(debug@4.3.4)
joi: 17.7.0
lodash: 4.17.21
minimist: 1.2.7
minimist: 1.2.8
rxjs: 7.8.1
transitivePeerDependencies:
- debug