This commit is contained in:
syuilo 2021-02-07 10:43:34 +09:00
parent 823a0c86d3
commit 0a64d121d9
10 changed files with 27 additions and 12 deletions

View File

@ -0,0 +1,6 @@
export function normalizeForSearch(tag: string): string {
// ref.
// - https://analytics-note.xyz/programming/unicode-normalization-forms/
// - https://maku77.github.io/js/string/normalize.html
return tag.normalize('NFKC').toLowerCase();
}

View File

@ -27,6 +27,7 @@ import { getConnection } from 'typeorm';
import { ensure } from '../../../prelude/ensure'; import { ensure } from '../../../prelude/ensure';
import { toArray } from '../../../prelude/array'; import { toArray } from '../../../prelude/array';
import { fetchInstanceMetadata } from '../../../services/fetch-instance-metadata'; import { fetchInstanceMetadata } from '../../../services/fetch-instance-metadata';
import { normalizeForSearch } from '../../../misc/normalize-for-search';
const logger = apLogger; const logger = apLogger;
@ -134,7 +135,7 @@ export async function createPerson(uri: string, resolver?: Resolver): Promise<Us
const { fields } = analyzeAttachments(person.attachment || []); const { fields } = analyzeAttachments(person.attachment || []);
const tags = extractApHashtags(person.tag).map(tag => tag.toLowerCase()).splice(0, 32); const tags = extractApHashtags(person.tag).map(tag => normalizeForSearch(tag)).splice(0, 32);
const isBot = object.type === 'Service'; const isBot = object.type === 'Service';
@ -323,7 +324,7 @@ export async function updatePerson(uri: string, resolver?: Resolver | null, hint
const { fields } = analyzeAttachments(person.attachment || []); const { fields } = analyzeAttachments(person.attachment || []);
const tags = extractApHashtags(person.tag).map(tag => tag.toLowerCase()).splice(0, 32); const tags = extractApHashtags(person.tag).map(tag => normalizeForSearch(tag)).splice(0, 32);
const bday = person['vcard:bday']?.match(/^\d{4}-\d{2}-\d{2}/); const bday = person['vcard:bday']?.match(/^\d{4}-\d{2}-\d{2}/);

View File

@ -2,6 +2,7 @@ import $ from 'cafy';
import define from '../../define'; import define from '../../define';
import { ApiError } from '../../error'; import { ApiError } from '../../error';
import { Hashtags } from '../../../../models'; import { Hashtags } from '../../../../models';
import { normalizeForSearch } from '../../../../misc/normalize-for-search';
export const meta = { export const meta = {
desc: { desc: {
@ -38,7 +39,7 @@ export const meta = {
}; };
export default define(meta, async (ps, user) => { export default define(meta, async (ps, user) => {
const hashtag = await Hashtags.findOne({ name: ps.tag.toLowerCase() }); const hashtag = await Hashtags.findOne({ name: normalizeForSearch(ps.tag) });
if (hashtag == null) { if (hashtag == null) {
throw new ApiError(meta.errors.noSuchHashtag); throw new ApiError(meta.errors.noSuchHashtag);
} }

View File

@ -4,6 +4,7 @@ import { fetchMeta } from '../../../../misc/fetch-meta';
import { Notes } from '../../../../models'; import { Notes } from '../../../../models';
import { Note } from '../../../../models/entities/note'; import { Note } from '../../../../models/entities/note';
import { safeForSql } from '../../../../misc/safe-for-sql'; import { safeForSql } from '../../../../misc/safe-for-sql';
import { normalizeForSearch } from '../../../../misc/normalize-for-search';
/* /*
a分間のユニーク投稿数が今からa分前b分前の間のユニーク投稿数のn倍以上5 a分間のユニーク投稿数が今からa分前b分前の間のユニーク投稿数のn倍以上5
@ -54,7 +55,7 @@ export const meta = {
export default define(meta, async () => { export default define(meta, async () => {
const instance = await fetchMeta(true); const instance = await fetchMeta(true);
const hiddenTags = instance.hiddenTags.map(t => t.toLowerCase()); const hiddenTags = instance.hiddenTags.map(t => normalizeForSearch(t));
const now = new Date(); // 5分単位で丸めた現在日時 const now = new Date(); // 5分単位で丸めた現在日時
now.setMinutes(Math.round(now.getMinutes() / 5) * 5, 0, 0); now.setMinutes(Math.round(now.getMinutes() / 5) * 5, 0, 0);

View File

@ -1,6 +1,7 @@
import $ from 'cafy'; import $ from 'cafy';
import define from '../../define'; import define from '../../define';
import { Users } from '../../../../models'; import { Users } from '../../../../models';
import { normalizeForSearch } from '../../../../misc/normalize-for-search';
export const meta = { export const meta = {
requireCredential: false as const, requireCredential: false as const,
@ -59,7 +60,7 @@ export const meta = {
export default define(meta, async (ps, me) => { export default define(meta, async (ps, me) => {
const query = Users.createQueryBuilder('user') const query = Users.createQueryBuilder('user')
.where(':tag = ANY(user.tags)', { tag: ps.tag.toLowerCase() }); .where(':tag = ANY(user.tags)', { tag: normalizeForSearch(ps.tag) });
const recent = new Date(Date.now() - (1000 * 60 * 60 * 24 * 5)); const recent = new Date(Date.now() - (1000 * 60 * 60 * 24 * 5));

View File

@ -15,6 +15,7 @@ import { User } from '../../../../models/entities/user';
import { UserProfile } from '../../../../models/entities/user-profile'; import { UserProfile } from '../../../../models/entities/user-profile';
import { ensure } from '../../../../prelude/ensure'; import { ensure } from '../../../../prelude/ensure';
import { notificationTypes } from '../../../../types'; import { notificationTypes } from '../../../../types';
import { normalizeForSearch } from '../../../../misc/normalize-for-search';
export const meta = { export const meta = {
desc: { desc: {
@ -286,7 +287,7 @@ export default define(meta, async (ps, user, token) => {
if (newDescription != null) { if (newDescription != null) {
const tokens = parse(newDescription); const tokens = parse(newDescription);
emojis = emojis.concat(extractEmojis(tokens!)); emojis = emojis.concat(extractEmojis(tokens!));
tags = extractHashtags(tokens!).map(tag => tag.toLowerCase()).splice(0, 32); tags = extractHashtags(tokens!).map(tag => normalizeForSearch(tag)).splice(0, 32);
} }
updates.emojis = emojis; updates.emojis = emojis;

View File

@ -7,6 +7,7 @@ import { generateMutedUserQuery } from '../../common/generate-muted-user-query';
import { generateVisibilityQuery } from '../../common/generate-visibility-query'; import { generateVisibilityQuery } from '../../common/generate-visibility-query';
import { Brackets } from 'typeorm'; import { Brackets } from 'typeorm';
import { safeForSql } from '../../../../misc/safe-for-sql'; import { safeForSql } from '../../../../misc/safe-for-sql';
import { normalizeForSearch } from '../../../../misc/normalize-for-search';
export const meta = { export const meta = {
desc: { desc: {
@ -101,7 +102,7 @@ export default define(meta, async (ps, me) => {
if (ps.tag) { if (ps.tag) {
if (!safeForSql(ps.tag)) return; if (!safeForSql(ps.tag)) return;
query.andWhere(`'{"${ps.tag.toLowerCase()}"}' <@ note.tags`); query.andWhere(`'{"${normalizeForSearch(ps.tag)}"}' <@ note.tags`);
} else { } else {
let i = 0; let i = 0;
query.andWhere(new Brackets(qb => { query.andWhere(new Brackets(qb => {
@ -109,7 +110,7 @@ export default define(meta, async (ps, me) => {
qb.orWhere(new Brackets(qb => { qb.orWhere(new Brackets(qb => {
for (const tag of tags) { for (const tag of tags) {
if (!safeForSql(tag)) return; if (!safeForSql(tag)) return;
qb.andWhere(`'{"${tag.toLowerCase()}"}' <@ note.tags`); qb.andWhere(`'{"${normalizeForSearch(ps.tag)}"}' <@ note.tags`);
i++; i++;
} }
})); }));

View File

@ -3,6 +3,7 @@ import { isMutedUserRelated } from '../../../../misc/is-muted-user-related';
import Channel from '../channel'; import Channel from '../channel';
import { Notes } from '../../../../models'; import { Notes } from '../../../../models';
import { PackedNote } from '../../../../models/repositories/note'; import { PackedNote } from '../../../../models/repositories/note';
import { normalizeForSearch } from '../../../../misc/normalize-for-search';
export default class extends Channel { export default class extends Channel {
public readonly chName = 'hashtag'; public readonly chName = 'hashtag';
@ -23,7 +24,7 @@ export default class extends Channel {
@autobind @autobind
private async onNote(note: PackedNote) { private async onNote(note: PackedNote) {
const noteTags = note.tags ? note.tags.map((t: string) => t.toLowerCase()) : []; const noteTags = note.tags ? note.tags.map((t: string) => t.toLowerCase()) : [];
const matched = this.q.some(tags => tags.every(tag => noteTags.includes(tag.toLowerCase()))); const matched = this.q.some(tags => tags.every(tag => noteTags.includes(normalizeForSearch(tag))));
if (!matched) return; if (!matched) return;
// Renoteなら再pack // Renoteなら再pack

View File

@ -33,6 +33,7 @@ import { addNoteToAntenna } from '../add-note-to-antenna';
import { countSameRenotes } from '../../misc/count-same-renotes'; import { countSameRenotes } from '../../misc/count-same-renotes';
import { deliverToRelays } from '../relay'; import { deliverToRelays } from '../relay';
import { Channel } from '../../models/entities/channel'; import { Channel } from '../../models/entities/channel';
import { normalizeForSearch } from '../../misc/normalize-for-search';
type NotificationType = 'reply' | 'renote' | 'quote' | 'mention'; type NotificationType = 'reply' | 'renote' | 'quote' | 'mention';
@ -460,7 +461,7 @@ async function insertNote(user: User, data: Option, tags: string[], emojis: stri
text: data.text, text: data.text,
hasPoll: data.poll != null, hasPoll: data.poll != null,
cw: data.cw == null ? null : data.cw, cw: data.cw == null ? null : data.cw,
tags: tags.map(tag => tag.toLowerCase()), tags: tags.map(tag => normalizeForSearch(tag)),
emojis, emojis,
userId: user.id, userId: user.id,
viaMobile: data.viaMobile!, viaMobile: data.viaMobile!,
@ -547,7 +548,7 @@ function index(note: Note) {
index: config.elasticsearch.index || 'misskey_note', index: config.elasticsearch.index || 'misskey_note',
id: note.id.toString(), id: note.id.toString(),
body: { body: {
text: note.text.toLowerCase(), text: normalizeForSearch(note.text),
userId: note.userId, userId: note.userId,
userHost: note.userHost userHost: note.userHost
} }

View File

@ -3,6 +3,7 @@ import { Hashtags, Users } from '../models';
import { hashtagChart } from './chart'; import { hashtagChart } from './chart';
import { genId } from '../misc/gen-id'; import { genId } from '../misc/gen-id';
import { Hashtag } from '../models/entities/hashtag'; import { Hashtag } from '../models/entities/hashtag';
import { normalizeForSearch } from '../misc/normalize-for-search';
export async function updateHashtags(user: User, tags: string[]) { export async function updateHashtags(user: User, tags: string[]) {
for (const tag of tags) { for (const tag of tags) {
@ -21,7 +22,7 @@ export async function updateUsertags(user: User, tags: string[]) {
} }
export async function updateHashtag(user: User, tag: string, isUserAttached = false, inc = true) { export async function updateHashtag(user: User, tag: string, isUserAttached = false, inc = true) {
tag = tag.toLowerCase(); tag = normalizeForSearch(tag);
const index = await Hashtags.findOne({ name: tag }); const index = await Hashtags.findOne({ name: tag });