Merge pull request #1368 from akihikodaki/misc

Distribute posts from remote
This commit is contained in:
syuilo 2018-04-02 17:41:51 +09:00 committed by GitHub
commit 81af33f8a1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 417 additions and 353 deletions

50
src/post/create.ts Normal file
View file

@ -0,0 +1,50 @@
import parseAcct from '../acct/parse';
import Post from '../models/post';
import User from '../models/user';
export default async (post, reply, repost, atMentions) => {
post.mentions = [];
function addMention(mentionee) {
// Reject if already added
if (post.mentions.some(x => x.equals(mentionee))) return;
// Add mention
post.mentions.push(mentionee);
}
if (reply) {
// Add mention
addMention(reply.userId);
post.replyId = reply._id;
post._reply = { userId: reply.userId };
} else {
post.replyId = null;
post._reply = null;
}
if (repost) {
if (post.text) {
// Add mention
addMention(repost.userId);
}
post.repostId = repost._id;
post._repost = { userId: repost.userId };
} else {
post.repostId = null;
post._repost = null;
}
await Promise.all(atMentions.map(async mention => {
// Fetch mentioned user
// SELECT _id
const { _id } = await User
.findOne(parseAcct(mention), { _id: true });
// Add mention
addMention(_id);
}));
return Post.insert(post);
};

216
src/post/distribute.ts Normal file
View file

@ -0,0 +1,216 @@
import Channel from '../models/channel';
import Mute from '../models/mute';
import Following from '../models/following';
import Post from '../models/post';
import Watching from '../models/post-watching';
import ChannelWatching from '../models/channel-watching';
import User from '../models/user';
import stream, { publishChannelStream } from '../publishers/stream';
import notify from '../publishers/notify';
import pushSw from '../publishers/push-sw';
import watch from './watch';
export default async (user, mentions, post) => {
const promises = [
User.update({ _id: user._id }, {
// Increment my posts count
$inc: {
postsCount: 1
},
$set: {
latestPost: post._id
}
}),
] as Array<Promise<any>>;
function addMention(mentionee, reason) {
// Publish event
if (!user._id.equals(mentionee)) {
promises.push(Mute.find({
muterId: mentionee,
deletedAt: { $exists: false }
}).then(mentioneeMutes => {
const mentioneesMutedUserIds = mentioneeMutes.map(m => m.muteeId.toString());
if (mentioneesMutedUserIds.indexOf(user._id.toString()) == -1) {
stream(mentionee, reason, post);
pushSw(mentionee, reason, post);
}
}));
}
}
// タイムラインへの投稿
if (!post.channelId) {
// Publish event to myself's stream
stream(user._id, 'post', post);
// Fetch all followers
const followers = await Following
.find({
followeeId: user._id,
// 削除されたドキュメントは除く
deletedAt: { $exists: false }
}, {
followerId: true,
_id: false
});
// Publish event to followers stream
followers.forEach(following =>
stream(following.followerId, 'post', post));
}
// チャンネルへの投稿
if (post.channelId) {
// Increment channel index(posts count)
promises.push(Channel.update({ _id: post.channelId }, {
$inc: {
index: 1
}
}));
// Publish event to channel
publishChannelStream(post.channelId, 'post', post);
// Get channel watchers
const watches = await ChannelWatching.find({
channelId: post.channelId,
// 削除されたドキュメントは除く
deletedAt: { $exists: false }
});
// チャンネルの視聴者(のタイムライン)に配信
watches.forEach(w => {
stream(w.userId, 'post', post);
});
}
// If has in reply to post
if (post.replyId) {
promises.push(
// Increment replies count
Post.update({ _id: post.replyId }, {
$inc: {
repliesCount: 1
}
}),
// 自分自身へのリプライでない限りは通知を作成
notify(post.reply.userId, user._id, 'reply', {
postId: post._id
}),
// Fetch watchers
Watching
.find({
postId: post.replyId,
userId: { $ne: user._id },
// 削除されたドキュメントは除く
deletedAt: { $exists: false }
}, {
fields: {
userId: true
}
})
.then(watchers => {
watchers.forEach(watcher => {
notify(watcher.userId, user._id, 'reply', {
postId: post._id
});
});
})
);
// Add mention
addMention(post.reply.userId, 'reply');
// この投稿をWatchする
if (user.account.settings.autoWatch !== false) {
promises.push(watch(user._id, post.reply));
}
}
// If it is repost
if (post.repostId) {
const type = post.text ? 'quote' : 'repost';
promises.push(
// Notify
notify(post.repost.userId, user._id, type, {
postId: post._id
}),
// Fetch watchers
Watching
.find({
postId: post.repostId,
userId: { $ne: user._id },
// 削除されたドキュメントは除く
deletedAt: { $exists: false }
}, {
fields: {
userId: true
}
})
.then(watchers => {
watchers.forEach(watcher => {
notify(watcher.userId, user._id, type, {
postId: post._id
});
});
}),
// この投稿をWatchする
// TODO: ユーザーが「Repostしたときに自動でWatchする」設定を
// オフにしていた場合はしない
watch(user._id, post.repost)
);
// If it is quote repost
if (post.text) {
// Add mention
addMention(post.repost.userId, 'quote');
} else {
// Publish event
if (!user._id.equals(post.repost.userId)) {
stream(post.repost.userId, 'repost', post);
}
}
// 今までで同じ投稿をRepostしているか
const existRepost = await Post.findOne({
userId: user._id,
repostId: post.repostId,
_id: {
$ne: post._id
}
});
if (!existRepost) {
// Update repostee status
promises.push(Post.update({ _id: post.repostId }, {
$inc: {
repostCount: 1
}
}));
}
}
// Resolve all mentions
await Promise.all(mentions.map(async mention => {
// 既に言及されたユーザーに対する返信や引用repostの場合も無視
if (post.reply && post.reply.userId.equals(mention)) return;
if (post.repost && post.repost.userId.equals(mention)) return;
// Add mention
addMention(mention, 'mention');
// Create notification
await notify(mention, user._id, 'mention', {
postId: post._id
});
}));
return Promise.all(promises);
};

View file

@ -1,5 +1,5 @@
import * as mongodb from 'mongodb'; import * as mongodb from 'mongodb';
import Watching from '../../../models/post-watching'; import Watching from '../models/post-watching';
export default async (me: mongodb.ObjectID, post: object) => { export default async (me: mongodb.ObjectID, post: object) => {
// 自分の投稿はwatchできない // 自分の投稿はwatchできない

View file

@ -2,5 +2,5 @@ import User from '../../models/user';
import act from '../../remote/activitypub/act'; import act from '../../remote/activitypub/act';
export default ({ data }, done) => User.findOne({ _id: data.actor }) export default ({ data }, done) => User.findOne({ _id: data.actor })
.then(actor => act(actor, data.outbox)) .then(actor => act(actor, data.outbox, data.distribute))
.then(() => done(), done); .then(() => done(), done);

View file

@ -1,9 +1,9 @@
import create from '../create'; import create from '../create';
export default (resolver, actor, activity) => { export default (resolver, actor, activity, distribute) => {
if ('actor' in activity && actor.account.uri !== activity.actor) { if ('actor' in activity && actor.account.uri !== activity.actor) {
throw new Error(); throw new Error();
} }
return create(resolver, actor, activity.object); return create(resolver, actor, activity.object, distribute);
}; };

View file

@ -2,10 +2,10 @@ import create from './create';
import createObject from '../create'; import createObject from '../create';
import Resolver from '../resolver'; import Resolver from '../resolver';
export default (actor, value) => { export default (actor, value, distribute) => {
return new Resolver().resolve(value).then(resolved => Promise.all(resolved.map(async promisedResult => { return new Resolver().resolve(value).then(resolved => Promise.all(resolved.map(async promisedResult => {
const { resolver, object } = await promisedResult; const { resolver, object } = await promisedResult;
const created = await (await createObject(resolver, actor, [object]))[0]; const created = await (await createObject(resolver, actor, [object], distribute))[0];
if (created !== null) { if (created !== null) {
return created; return created;
@ -13,7 +13,7 @@ export default (actor, value) => {
switch (object.type) { switch (object.type) {
case 'Create': case 'Create':
return create(resolver, actor, object); return create(resolver, actor, object, distribute);
default: default:
return null; return null;

View file

@ -1,8 +1,11 @@
import { JSDOM } from 'jsdom'; import { JSDOM } from 'jsdom';
import config from '../../config'; import config from '../../config';
import Post from '../../models/post'; import { pack as packPost } from '../../models/post';
import RemoteUserObject, { IRemoteUserObject } from '../../models/remote-user-object'; import RemoteUserObject, { IRemoteUserObject } from '../../models/remote-user-object';
import { IRemoteUser } from '../../models/user';
import uploadFromUrl from '../../drive/upload-from-url'; import uploadFromUrl from '../../drive/upload-from-url';
import createPost from '../../post/create';
import distributePost from '../../post/distribute';
import Resolver from './resolver'; import Resolver from './resolver';
const createDOMPurify = require('dompurify'); const createDOMPurify = require('dompurify');
@ -16,72 +19,98 @@ function createRemoteUserObject($ref, $id, { id }) {
return RemoteUserObject.insert({ uri: id, object }); return RemoteUserObject.insert({ uri: id, object });
} }
async function createImage(actor, object) { class Creator {
if ('attributedTo' in object && actor.account.uri !== object.attributedTo) { private actor: IRemoteUser;
throw new Error(); private distribute: boolean;
constructor(actor, distribute) {
this.actor = actor;
this.distribute = distribute;
} }
const { _id } = await uploadFromUrl(object.url, actor); private async createImage(object) {
return createRemoteUserObject('driveFiles.files', _id, object); if ('attributedTo' in object && this.actor.account.uri !== object.attributedTo) {
} throw new Error();
async function createNote(resolver, actor, object) {
if ('attributedTo' in object && actor.account.uri !== object.attributedTo) {
throw new Error();
}
const mediaIds = 'attachment' in object &&
(await Promise.all(await create(resolver, actor, object.attachment)))
.filter(media => media !== null && media.object.$ref === 'driveFiles.files')
.map(({ object }) => object.$id);
const { window } = new JSDOM(object.content);
const { _id } = await Post.insert({
channelId: undefined,
index: undefined,
createdAt: new Date(object.published),
mediaIds,
replyId: undefined,
repostId: undefined,
poll: undefined,
text: window.document.body.textContent,
textHtml: object.content && createDOMPurify(window).sanitize(object.content),
userId: actor._id,
appId: null,
viaMobile: false,
geo: undefined
});
// Register to search database
if (object.content && config.elasticsearch.enable) {
const es = require('../../db/elasticsearch');
es.index({
index: 'misskey',
type: 'post',
id: _id.toString(),
body: {
text: window.document.body.textContent
}
});
}
return createRemoteUserObject('posts', _id, object);
}
export default async function create(parentResolver: Resolver, actor, value): Promise<Array<Promise<IRemoteUserObject>>> {
const results = await parentResolver.resolveRemoteUserObjects(value);
return results.map(promisedResult => promisedResult.then(({ resolver, object }) => {
switch (object.type) {
case 'Image':
return createImage(actor, object);
case 'Note':
return createNote(resolver, actor, object);
} }
return null; const { _id } = await uploadFromUrl(object.url, this.actor);
})); return createRemoteUserObject('driveFiles.files', _id, object);
}
private async createNote(resolver, object) {
if ('attributedTo' in object && this.actor.account.uri !== object.attributedTo) {
throw new Error();
}
const mediaIds = 'attachment' in object &&
(await Promise.all(await this.create(resolver, object.attachment)))
.filter(media => media !== null && media.object.$ref === 'driveFiles.files')
.map(({ object }) => object.$id);
const { window } = new JSDOM(object.content);
const inserted = await createPost({
channelId: undefined,
index: undefined,
createdAt: new Date(object.published),
mediaIds,
replyId: undefined,
repostId: undefined,
poll: undefined,
text: window.document.body.textContent,
textHtml: object.content && createDOMPurify(window).sanitize(object.content),
userId: this.actor._id,
appId: null,
viaMobile: false,
geo: undefined
}, null, null, []);
const promisedRemoteUserObject = createRemoteUserObject('posts', inserted._id, object);
const promises = [];
if (this.distribute) {
promises.push(distributePost(this.actor, inserted.mentions, packPost(inserted)));
}
// Register to search database
if (object.content && config.elasticsearch.enable) {
const es = require('../../db/elasticsearch');
promises.push(new Promise((resolve, reject) => {
es.index({
index: 'misskey',
type: 'post',
id: inserted._id.toString(),
body: {
text: window.document.body.textContent
}
}, resolve);
}));
}
await Promise.all(promises);
return promisedRemoteUserObject;
}
public async create(parentResolver, value): Promise<Array<Promise<IRemoteUserObject>>> {
const results = await parentResolver.resolveRemoteUserObjects(value);
return results.map(promisedResult => promisedResult.then(({ resolver, object }) => {
switch (object.type) {
case 'Image':
return this.createImage(object);
case 'Note':
return this.createNote(resolver, object);
}
return null;
}));
}
} }
export default (resolver: Resolver, actor, value, distribute?: boolean) => {
const creator = new Creator(actor, distribute);
return creator.create(resolver, value);
};

View file

@ -52,10 +52,10 @@ export default async (value, usernameLower, hostLower, acctLower) => {
bannerId: null, bannerId: null,
createdAt: Date.parse(object.published), createdAt: Date.parse(object.published),
description: summaryDOM.textContent, description: summaryDOM.textContent,
followersCount: followers.totalItem, followersCount: followers ? followers.totalItem || 0 : 0,
followingCount: following.totalItem, followingCount: following ? following.totalItem || 0 : 0,
name: object.name, name: object.name,
postsCount: outbox.totalItem, postsCount: outbox ? outbox.totalItem || 0 : 0,
driveCapacity: 1024 * 1024 * 8, // 8MiB driveCapacity: 1024 * 1024 * 8, // 8MiB
username: object.preferredUsername, username: object.preferredUsername,
usernameLower, usernameLower,

View file

@ -6,10 +6,14 @@ import queue from '../../queue';
import parseAcct from '../../acct/parse'; import parseAcct from '../../acct/parse';
const app = express(); const app = express();
app.disable('x-powered-by');
app.use(bodyParser.json());
app.post('/@:user/inbox', async (req, res) => { app.disable('x-powered-by');
app.post('/@:user/inbox', bodyParser.json({
type() {
return true;
}
}), async (req, res) => {
let parsed; let parsed;
req.headers.authorization = 'Signature ' + req.headers.signature; req.headers.authorization = 'Signature ' + req.headers.signature;
@ -51,6 +55,7 @@ app.post('/@:user/inbox', async (req, res) => {
type: 'performActivityPub', type: 'performActivityPub',
actor: user._id, actor: user._id,
outbox: req.body, outbox: req.body,
distribute: true,
}).save(); }).save();
return res.status(202).end(); return res.status(202).end();

View file

@ -3,24 +3,16 @@
*/ */
import $ from 'cafy'; import $ from 'cafy';
import deepEqual = require('deep-equal'); import deepEqual = require('deep-equal');
import renderAcct from '../../../../acct/render';
import config from '../../../../config';
import html from '../../../../text/html'; import html from '../../../../text/html';
import parse from '../../../../text/parse'; import parse from '../../../../text/parse';
import Post, { IPost, isValidText, isValidCw } from '../../../../models/post'; import Post, { IPost, isValidText, isValidCw, pack } from '../../../../models/post';
import User, { ILocalUser } from '../../../../models/user'; import { ILocalUser } from '../../../../models/user';
import Channel, { IChannel } from '../../../../models/channel'; import Channel, { IChannel } from '../../../../models/channel';
import Following from '../../../../models/following';
import Mute from '../../../../models/mute';
import DriveFile from '../../../../models/drive-file'; import DriveFile from '../../../../models/drive-file';
import Watching from '../../../../models/post-watching'; import create from '../../../../post/create';
import ChannelWatching from '../../../../models/channel-watching'; import distribute from '../../../../post/distribute';
import { pack } from '../../../../models/post';
import watch from '../../common/watch-post';
import stream, { publishChannelStream } from '../../../../publishers/stream';
import notify from '../../../../publishers/notify';
import pushSw from '../../../../publishers/push-sw';
import getAcct from '../../../../acct/render';
import parseAcct from '../../../../acct/parse';
import config from '../../../../config';
/** /**
* Create a post * Create a post
@ -251,226 +243,7 @@ module.exports = (params, user: ILocalUser, app) => new Promise(async (res, rej)
}); });
} }
// 投稿を作成 let atMentions = [];
const post = await Post.insert({
createdAt: new Date(),
channelId: channel ? channel._id : undefined,
index: channel ? channel.index + 1 : undefined,
mediaIds: files ? files.map(file => file._id) : [],
replyId: reply ? reply._id : undefined,
repostId: repost ? repost._id : undefined,
poll: poll,
text: text,
textHtml: tokens === null ? null : html(tokens),
cw: cw,
tags: tags,
userId: user._id,
appId: app ? app._id : null,
viaMobile: viaMobile,
geo,
// 以下非正規化データ
_reply: reply ? { userId: reply.userId } : undefined,
_repost: repost ? { userId: repost.userId } : undefined,
});
// Serialize
const postObj = await pack(post);
// Reponse
res({
createdPost: postObj
});
//#region Post processes
User.update({ _id: user._id }, {
$set: {
latestPost: post
}
});
const mentions = [];
async function addMention(mentionee, reason) {
// Reject if already added
if (mentions.some(x => x.equals(mentionee))) return;
// Add mention
mentions.push(mentionee);
// Publish event
if (!user._id.equals(mentionee)) {
const mentioneeMutes = await Mute.find({
muterId: mentionee,
deletedAt: { $exists: false }
});
const mentioneesMutedUserIds = mentioneeMutes.map(m => m.muteeId.toString());
if (mentioneesMutedUserIds.indexOf(user._id.toString()) == -1) {
stream(mentionee, reason, postObj);
pushSw(mentionee, reason, postObj);
}
}
}
// タイムラインへの投稿
if (!channel) {
// Publish event to myself's stream
stream(user._id, 'post', postObj);
// Fetch all followers
const followers = await Following
.find({
followeeId: user._id,
// 削除されたドキュメントは除く
deletedAt: { $exists: false }
}, {
followerId: true,
_id: false
});
// Publish event to followers stream
followers.forEach(following =>
stream(following.followerId, 'post', postObj));
}
// チャンネルへの投稿
if (channel) {
// Increment channel index(posts count)
Channel.update({ _id: channel._id }, {
$inc: {
index: 1
}
});
// Publish event to channel
publishChannelStream(channel._id, 'post', postObj);
// Get channel watchers
const watches = await ChannelWatching.find({
channelId: channel._id,
// 削除されたドキュメントは除く
deletedAt: { $exists: false }
});
// チャンネルの視聴者(のタイムライン)に配信
watches.forEach(w => {
stream(w.userId, 'post', postObj);
});
}
// Increment my posts count
User.update({ _id: user._id }, {
$inc: {
postsCount: 1
}
});
// If has in reply to post
if (reply) {
// Increment replies count
Post.update({ _id: reply._id }, {
$inc: {
repliesCount: 1
}
});
// 自分自身へのリプライでない限りは通知を作成
notify(reply.userId, user._id, 'reply', {
postId: post._id
});
// Fetch watchers
Watching
.find({
postId: reply._id,
userId: { $ne: user._id },
// 削除されたドキュメントは除く
deletedAt: { $exists: false }
}, {
fields: {
userId: true
}
})
.then(watchers => {
watchers.forEach(watcher => {
notify(watcher.userId, user._id, 'reply', {
postId: post._id
});
});
});
// この投稿をWatchする
if (user.account.settings.autoWatch !== false) {
watch(user._id, reply);
}
// Add mention
addMention(reply.userId, 'reply');
}
// If it is repost
if (repost) {
// Notify
const type = text ? 'quote' : 'repost';
notify(repost.userId, user._id, type, {
postId: post._id
});
// Fetch watchers
Watching
.find({
postId: repost._id,
userId: { $ne: user._id },
// 削除されたドキュメントは除く
deletedAt: { $exists: false }
}, {
fields: {
userId: true
}
})
.then(watchers => {
watchers.forEach(watcher => {
notify(watcher.userId, user._id, type, {
postId: post._id
});
});
});
// この投稿をWatchする
// TODO: ユーザーが「Repostしたときに自動でWatchする」設定を
// オフにしていた場合はしない
watch(user._id, repost);
// If it is quote repost
if (text) {
// Add mention
addMention(repost.userId, 'quote');
} else {
// Publish event
if (!user._id.equals(repost.userId)) {
stream(repost.userId, 'repost', postObj);
}
}
// 今までで同じ投稿をRepostしているか
const existRepost = await Post.findOne({
userId: user._id,
repostId: repost._id,
_id: {
$ne: post._id
}
});
if (!existRepost) {
// Update repostee status
Post.update({ _id: repost._id }, {
$inc: {
repostCount: 1
}
});
}
}
// If has text content // If has text content
if (text) { if (text) {
@ -486,40 +259,42 @@ module.exports = (params, user: ILocalUser, app) => new Promise(async (res, rej)
registerHashtags(user, hashtags); registerHashtags(user, hashtags);
*/ */
// Extract an '@' mentions // Extract an '@' mentions
const atMentions = tokens atMentions = tokens
.filter(t => t.type == 'mention') .filter(t => t.type == 'mention')
.map(getAcct) .map(renderAcct)
// Drop dupulicates // Drop dupulicates
.filter((v, i, s) => s.indexOf(v) == i); .filter((v, i, s) => s.indexOf(v) == i);
// Resolve all mentions
await Promise.all(atMentions.map(async (mention) => {
// Fetch mentioned user
// SELECT _id
const mentionee = await User
.findOne(parseAcct(mention), { _id: true });
// When mentioned user not found
if (mentionee == null) return;
// 既に言及されたユーザーに対する返信や引用repostの場合も無視
if (reply && reply.userId.equals(mentionee._id)) return;
if (repost && repost.userId.equals(mentionee._id)) return;
// Add mention
addMention(mentionee._id, 'mention');
// Create notification
notify(mentionee._id, user._id, 'mention', {
postId: post._id
});
return;
}));
} }
// 投稿を作成
const post = await create({
createdAt: new Date(),
channelId: channel ? channel._id : undefined,
index: channel ? channel.index + 1 : undefined,
mediaIds: files ? files.map(file => file._id) : [],
poll: poll,
text: text,
textHtml: tokens === null ? null : html(tokens),
cw: cw,
tags: tags,
userId: user._id,
appId: app ? app._id : null,
viaMobile: viaMobile,
geo
}, reply, repost, atMentions);
// Serialize
const postObj = await pack(post);
// Reponse
res({
createdPost: postObj
});
distribute(user, post.mentions, postObj);
// Register to search database // Register to search database
if (text && config.elasticsearch.enable) { if (post.text && config.elasticsearch.enable) {
const es = require('../../../db/elasticsearch'); const es = require('../../../db/elasticsearch');
es.index({ es.index({
@ -531,15 +306,4 @@ module.exports = (params, user: ILocalUser, app) => new Promise(async (res, rej)
} }
}); });
} }
// Append mentions data
if (mentions.length > 0) {
Post.update({ _id: post._id }, {
$set: {
mentions: mentions
}
});
}
//#endregion
}); });

View file

@ -5,7 +5,7 @@ import $ from 'cafy';
import Vote from '../../../../../models/poll-vote'; import Vote from '../../../../../models/poll-vote';
import Post from '../../../../../models/post'; import Post from '../../../../../models/post';
import Watching from '../../../../../models/post-watching'; import Watching from '../../../../../models/post-watching';
import watch from '../../../common/watch-post'; import watch from '../../../../../post/watch';
import { publishPostStream } from '../../../../../publishers/stream'; import { publishPostStream } from '../../../../../publishers/stream';
import notify from '../../../../../publishers/notify'; import notify from '../../../../../publishers/notify';

View file

@ -6,7 +6,7 @@ import Reaction from '../../../../../models/post-reaction';
import Post, { pack as packPost } from '../../../../../models/post'; import Post, { pack as packPost } from '../../../../../models/post';
import { pack as packUser } from '../../../../../models/user'; import { pack as packUser } from '../../../../../models/user';
import Watching from '../../../../../models/post-watching'; import Watching from '../../../../../models/post-watching';
import watch from '../../../common/watch-post'; import watch from '../../../../../post/watch';
import { publishPostStream } from '../../../../../publishers/stream'; import { publishPostStream } from '../../../../../publishers/stream';
import notify from '../../../../../publishers/notify'; import notify from '../../../../../publishers/notify';
import pushSw from '../../../../../publishers/push-sw'; import pushSw from '../../../../../publishers/push-sw';