diff --git a/.gitignore b/.gitignore index 42b1bde94..2ae0f98c5 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ /node_modules /built /uploads +/data npm-debug.log *.pem run.bat diff --git a/locales/en.yml b/locales/en.yml index d40896212..3b87ea758 100644 --- a/locales/en.yml +++ b/locales/en.yml @@ -22,6 +22,14 @@ common: confused: "Confused" pudding: "Pudding" + post_categories: + music: "Music" + game: "Video Game" + anime: "Anime" + it: "IT" + gadgets: "Gadgets" + photography: "Photography" + input-message-here: "Enter message here" send: "Send" delete: "Delete" @@ -80,6 +88,9 @@ common: mk-post-menu: pin: "Pin" pinned: "Pinned" + select: "Select category" + categorize: "Accept" + categorized: "Category reported. Thank you!" mk-reaction-picker: choose-reaction: "Pick your reaction" @@ -375,6 +386,7 @@ mobile: twitter-integration: "Twitter integration" signin-history: "Sign in history" api: "API" + link: "MisskeyLink" settings: "Settings" signout: "Sign out" diff --git a/locales/ja.yml b/locales/ja.yml index b8e5cff41..13d451b6d 100644 --- a/locales/ja.yml +++ b/locales/ja.yml @@ -22,6 +22,14 @@ common: confused: "こまこまのこまり" pudding: "Pudding" + post_categories: + music: "音楽" + game: "ゲーム" + anime: "アニメ" + it: "IT" + gadgets: "ガジェット" + photography: "写真" + input-message-here: "ここにメッセージを入力" send: "送信" delete: "削除" @@ -80,6 +88,9 @@ common: mk-post-menu: pin: "ピン留め" pinned: "ピン留めしました" + select: "カテゴリを選択" + categorize: "決定" + categorized: "カテゴリを報告しました。これによりMisskeyが賢くなり、投稿の自動カテゴライズに役立てられます。ご協力ありがとうございました。" mk-reaction-picker: choose-reaction: "リアクションを選択" @@ -375,6 +386,7 @@ mobile: twitter-integration: "Twitter連携" signin-history: "ログイン履歴" api: "API" + link: "Misskeyリンク" settings: "設定" signout: "サインアウト" diff --git a/package.json b/package.json index a2896f4c7..31cf7a02c 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "@types/webpack": "3.0.10", "@types/webpack-stream": "3.2.7", "@types/websocket": "0.0.34", + "@types/msgpack-lite": "^0.1.5", "chai": "4.1.2", "chai-http": "3.0.0", "css-loader": "0.28.7", @@ -120,10 +121,12 @@ "is-root": "1.0.0", "is-url": "1.2.2", "js-yaml": "3.9.1", + "mecab-async": "^0.1.0", "mongodb": "2.2.31", "monk": "6.0.3", "morgan": "1.8.2", "ms": "2.0.0", + "msgpack-lite": "^0.1.26", "multer": "1.3.0", "nprogress": "0.2.0", "os-utils": "0.0.14", diff --git a/src/api/endpoints.ts b/src/api/endpoints.ts index e5be68c09..97b98895b 100644 --- a/src/api/endpoints.ts +++ b/src/api/endpoints.ts @@ -394,6 +394,10 @@ const endpoints: Endpoint[] = [ name: 'posts/trend', withCredential: true }, + { + name: 'posts/categorize', + withCredential: true + }, { name: 'posts/reactions', withCredential: true diff --git a/src/api/endpoints/posts/categorize.ts b/src/api/endpoints/posts/categorize.ts new file mode 100644 index 000000000..3530ba6bc --- /dev/null +++ b/src/api/endpoints/posts/categorize.ts @@ -0,0 +1,52 @@ +/** + * Module dependencies + */ +import $ from 'cafy'; +import Post from '../../models/post'; + +/** + * Categorize a post + * + * @param {any} params + * @param {any} user + * @return {Promise} + */ +module.exports = (params, user) => new Promise(async (res, rej) => { + if (!user.is_pro) { + return rej('This endpoint is available only from a Pro account'); + } + + // Get 'post_id' parameter + const [postId, postIdErr] = $(params.post_id).id().$; + if (postIdErr) return rej('invalid post_id param'); + + // Get categorizee + const post = await Post.findOne({ + _id: postId + }); + + if (post === null) { + return rej('post not found'); + } + + if (post.is_category_verified) { + return rej('This post already has the verified category'); + } + + // Get 'category' parameter + const [category, categoryErr] = $(params.category).string().or([ + 'music', 'game', 'anime', 'it', 'gadgets', 'photography' + ]).$; + if (categoryErr) return rej('invalid category param'); + + // Set category + Post.update({ _id: post._id }, { + $set: { + category: category, + is_category_verified: true + } + }); + + // Send response + res(); +}); diff --git a/src/config.ts b/src/config.ts index 8f4ada5af..f333a1f5a 100644 --- a/src/config.ts +++ b/src/config.ts @@ -68,6 +68,9 @@ type Source = { hook_secret: string; username: string; }; + categorizer?: { + mecab_command?: string; + }; }; /** diff --git a/src/tools/ai/naive-bayes.js b/src/tools/ai/naive-bayes.js new file mode 100644 index 000000000..78f07153c --- /dev/null +++ b/src/tools/ai/naive-bayes.js @@ -0,0 +1,302 @@ +// Original source code: https://github.com/ttezel/bayes/blob/master/lib/naive_bayes.js (commit: 2c20d3066e4fc786400aaedcf3e42987e52abe3c) +// CUSTOMIZED BY SYUILO + +/* + Expose our naive-bayes generator function +*/ +module.exports = function (options) { + return new Naivebayes(options) +} + +// keys we use to serialize a classifier's state +var STATE_KEYS = module.exports.STATE_KEYS = [ + 'categories', 'docCount', 'totalDocuments', 'vocabulary', 'vocabularySize', + 'wordCount', 'wordFrequencyCount', 'options' +]; + +/** + * Initializes a NaiveBayes instance from a JSON state representation. + * Use this with classifier.toJson(). + * + * @param {String} jsonStr state representation obtained by classifier.toJson() + * @return {NaiveBayes} Classifier + */ +module.exports.fromJson = function (jsonStr) { + var parsed; + try { + parsed = JSON.parse(jsonStr) + } catch (e) { + throw new Error('Naivebayes.fromJson expects a valid JSON string.') + } + // init a new classifier + var classifier = new Naivebayes(parsed.options) + + // override the classifier's state + STATE_KEYS.forEach(function (k) { + if (!parsed[k]) { + throw new Error('Naivebayes.fromJson: JSON string is missing an expected property: `'+k+'`.') + } + classifier[k] = parsed[k] + }) + + return classifier +} + +/** + * Given an input string, tokenize it into an array of word tokens. + * This is the default tokenization function used if user does not provide one in `options`. + * + * @param {String} text + * @return {Array} + */ +var defaultTokenizer = function (text) { + //remove punctuation from text - remove anything that isn't a word char or a space + var rgxPunctuation = /[^(a-zA-ZA-Яa-я0-9_)+\s]/g + + var sanitized = text.replace(rgxPunctuation, ' ') + + return sanitized.split(/\s+/) +} + +/** + * Naive-Bayes Classifier + * + * This is a naive-bayes classifier that uses Laplace Smoothing. + * + * Takes an (optional) options object containing: + * - `tokenizer` => custom tokenization function + * + */ +function Naivebayes (options) { + // set options object + this.options = {} + if (typeof options !== 'undefined') { + if (!options || typeof options !== 'object' || Array.isArray(options)) { + throw TypeError('NaiveBayes got invalid `options`: `' + options + '`. Pass in an object.') + } + this.options = options + } + + this.tokenizer = this.options.tokenizer || defaultTokenizer + + //initialize our vocabulary and its size + this.vocabulary = {} + this.vocabularySize = 0 + + //number of documents we have learned from + this.totalDocuments = 0 + + //document frequency table for each of our categories + //=> for each category, how often were documents mapped to it + this.docCount = {} + + //for each category, how many words total were mapped to it + this.wordCount = {} + + //word frequency table for each category + //=> for each category, how frequent was a given word mapped to it + this.wordFrequencyCount = {} + + //hashmap of our category names + this.categories = {} +} + +/** + * Initialize each of our data structure entries for this new category + * + * @param {String} categoryName + */ +Naivebayes.prototype.initializeCategory = function (categoryName) { + if (!this.categories[categoryName]) { + this.docCount[categoryName] = 0 + this.wordCount[categoryName] = 0 + this.wordFrequencyCount[categoryName] = {} + this.categories[categoryName] = true + } + return this +} + +/** + * train our naive-bayes classifier by telling it what `category` + * the `text` corresponds to. + * + * @param {String} text + * @param {String} class + */ +Naivebayes.prototype.learn = function (text, category) { + var self = this + + //initialize category data structures if we've never seen this category + self.initializeCategory(category) + + //update our count of how many documents mapped to this category + self.docCount[category]++ + + //update the total number of documents we have learned from + self.totalDocuments++ + + //normalize the text into a word array + var tokens = self.tokenizer(text) + + //get a frequency count for each token in the text + var frequencyTable = self.frequencyTable(tokens) + + /* + Update our vocabulary and our word frequency count for this category + */ + + Object + .keys(frequencyTable) + .forEach(function (token) { + //add this word to our vocabulary if not already existing + if (!self.vocabulary[token]) { + self.vocabulary[token] = true + self.vocabularySize++ + } + + var frequencyInText = frequencyTable[token] + + //update the frequency information for this word in this category + if (!self.wordFrequencyCount[category][token]) + self.wordFrequencyCount[category][token] = frequencyInText + else + self.wordFrequencyCount[category][token] += frequencyInText + + //update the count of all words we have seen mapped to this category + self.wordCount[category] += frequencyInText + }) + + return self +} + +/** + * Determine what category `text` belongs to. + * + * @param {String} text + * @return {String} category + */ +Naivebayes.prototype.categorize = function (text) { + var self = this + , maxProbability = -Infinity + , chosenCategory = null + + var tokens = self.tokenizer(text) + var frequencyTable = self.frequencyTable(tokens) + + //iterate thru our categories to find the one with max probability for this text + Object + .keys(self.categories) + .forEach(function (category) { + + //start by calculating the overall probability of this category + //=> out of all documents we've ever looked at, how many were + // mapped to this category + var categoryProbability = self.docCount[category] / self.totalDocuments + + //take the log to avoid underflow + var logProbability = Math.log(categoryProbability) + + //now determine P( w | c ) for each word `w` in the text + Object + .keys(frequencyTable) + .forEach(function (token) { + var frequencyInText = frequencyTable[token] + var tokenProbability = self.tokenProbability(token, category) + + // console.log('token: %s category: `%s` tokenProbability: %d', token, category, tokenProbability) + + //determine the log of the P( w | c ) for this word + logProbability += frequencyInText * Math.log(tokenProbability) + }) + + if (logProbability > maxProbability) { + maxProbability = logProbability + chosenCategory = category + } + }) + + return chosenCategory +} + +/** + * Calculate probability that a `token` belongs to a `category` + * + * @param {String} token + * @param {String} category + * @return {Number} probability + */ +Naivebayes.prototype.tokenProbability = function (token, category) { + //how many times this word has occurred in documents mapped to this category + var wordFrequencyCount = this.wordFrequencyCount[category][token] || 0 + + //what is the count of all words that have ever been mapped to this category + var wordCount = this.wordCount[category] + + //use laplace Add-1 Smoothing equation + return ( wordFrequencyCount + 1 ) / ( wordCount + this.vocabularySize ) +} + +/** + * Build a frequency hashmap where + * - the keys are the entries in `tokens` + * - the values are the frequency of each entry in `tokens` + * + * @param {Array} tokens Normalized word array + * @return {Object} + */ +Naivebayes.prototype.frequencyTable = function (tokens) { + var frequencyTable = Object.create(null) + + tokens.forEach(function (token) { + if (!frequencyTable[token]) + frequencyTable[token] = 1 + else + frequencyTable[token]++ + }) + + return frequencyTable +} + +/** + * Dump the classifier's state as a JSON string. + * @return {String} Representation of the classifier. + */ +Naivebayes.prototype.toJson = function () { + var state = {} + var self = this + STATE_KEYS.forEach(function (k) { + state[k] = self[k] + }) + + var jsonStr = JSON.stringify(state) + + return jsonStr +} + +// (original method) +Naivebayes.prototype.export = function () { + var state = {} + var self = this + STATE_KEYS.forEach(function (k) { + state[k] = self[k] + }) + + return state +} + +module.exports.import = function (data) { + var parsed = data + + // init a new classifier + var classifier = new Naivebayes() + + // override the classifier's state + STATE_KEYS.forEach(function (k) { + if (!parsed[k]) { + throw new Error('Naivebayes.import: data is missing an expected property: `'+k+'`.') + } + classifier[k] = parsed[k] + }) + + return classifier +} diff --git a/src/tools/ai/predict-all-post-category.ts b/src/tools/ai/predict-all-post-category.ts new file mode 100644 index 000000000..87e198b39 --- /dev/null +++ b/src/tools/ai/predict-all-post-category.ts @@ -0,0 +1,57 @@ +const bayes = require('./naive-bayes.js'); +const MeCab = require('mecab-async'); + +import Post from '../../api/models/post'; +import config from '../../conf'; + +const classifier = bayes({ + tokenizer: this.tokenizer +}); + +const mecab = new MeCab(); +if (config.categorizer.mecab_command) mecab.command = config.categorizer.mecab_command; + +// 訓練データ取得 +Post.find({ + is_category_verified: true +}, { + fields: { + _id: false, + text: true, + category: true + } +}).then(verifiedPosts => { + // 学習 + verifiedPosts.forEach(post => { + classifier.learn(post.text, post.category); + }); + + // 全ての(人間によって証明されていない)投稿を取得 + Post.find({ + text: { + $exists: true + }, + is_category_verified: { + $ne: true + } + }, { + sort: { + _id: -1 + }, + fields: { + _id: true, + text: true + } + }).then(posts => { + posts.forEach(post => { + console.log(`predicting... ${post._id}`); + const category = classifier.categorize(post.text); + + Post.update({ _id: post._id }, { + $set: { + category: category + } + }); + }); + }); +}); diff --git a/src/tools/ai/predict-user-interst.ts b/src/tools/ai/predict-user-interst.ts new file mode 100644 index 000000000..99bdfa420 --- /dev/null +++ b/src/tools/ai/predict-user-interst.ts @@ -0,0 +1,45 @@ +import Post from '../../api/models/post'; +import User from '../../api/models/user'; + +export async function predictOne(id) { + console.log(`predict interest of ${id} ...`); + + // TODO: repostなども含める + const recentPosts = await Post.find({ + user_id: id, + category: { + $exists: true + } + }, { + sort: { + _id: -1 + }, + limit: 1000, + fields: { + _id: false, + category: true + } + }); + + const categories = {}; + + recentPosts.forEach(post => { + if (categories[post.category]) { + categories[post.category]++; + } else { + categories[post.category] = 1; + } + }); +} + +export async function predictAll() { + const allUsers = await User.find({}, { + fields: { + _id: true + } + }); + + allUsers.forEach(user => { + predictOne(user._id); + }); +} diff --git a/src/web/app/common/tags/post-menu.tag b/src/web/app/common/tags/post-menu.tag index 33895212b..be4468a21 100644 --- a/src/web/app/common/tags/post-menu.tag +++ b/src/web/app/common/tags/post-menu.tag @@ -2,6 +2,18 @@
+
+ + +