Introduce processor

This commit is contained in:
Akihiko Odaki 2018-03-29 01:20:40 +09:00
parent 68ce6d5748
commit 90f8fe7e53
582 changed files with 246 additions and 188 deletions

View File

@ -22,7 +22,7 @@ import * as replace from 'gulp-replace';
import * as htmlmin from 'gulp-htmlmin'; import * as htmlmin from 'gulp-htmlmin';
const uglifyes = require('uglify-es'); const uglifyes = require('uglify-es');
import { fa } from './src/common/build/fa'; import { fa } from './src/build/fa';
import version from './src/version'; import version from './src/version';
import config from './src/conf'; import config from './src/conf';
@ -39,7 +39,7 @@ if (isDebug) {
const constants = require('./src/const.json'); const constants = require('./src/const.json');
require('./src/web/docs/gulpfile.ts'); require('./src/server/web/docs/gulpfile.ts');
gulp.task('build', [ gulp.task('build', [
'build:js', 'build:js',
@ -52,7 +52,7 @@ gulp.task('build', [
gulp.task('rebuild', ['clean', 'build']); gulp.task('rebuild', ['clean', 'build']);
gulp.task('build:js', () => gulp.task('build:js', () =>
gulp.src(['./src/**/*.js', '!./src/web/**/*.js']) gulp.src(['./src/**/*.js', '!./src/server/web/**/*.js'])
.pipe(gulp.dest('./built/')) .pipe(gulp.dest('./built/'))
); );
@ -71,7 +71,7 @@ gulp.task('build:copy', () =>
gulp.src([ gulp.src([
'./build/Release/crypto_key.node', './build/Release/crypto_key.node',
'./src/**/assets/**/*', './src/**/assets/**/*',
'!./src/web/app/**/assets/**/*' '!./src/server/web/app/**/assets/**/*'
]).pipe(gulp.dest('./built/')) ]).pipe(gulp.dest('./built/'))
); );
@ -121,7 +121,7 @@ gulp.task('build:client', [
]); ]);
gulp.task('build:client:script', () => gulp.task('build:client:script', () =>
gulp.src(['./src/web/app/boot.js', './src/web/app/safe.js']) gulp.src(['./src/server/web/app/boot.js', './src/server/web/app/safe.js'])
.pipe(replace('VERSION', JSON.stringify(version))) .pipe(replace('VERSION', JSON.stringify(version)))
.pipe(replace('API', JSON.stringify(config.api_url))) .pipe(replace('API', JSON.stringify(config.api_url)))
.pipe(replace('ENV', JSON.stringify(env))) .pipe(replace('ENV', JSON.stringify(env)))
@ -129,15 +129,15 @@ gulp.task('build:client:script', () =>
.pipe(isProduction ? uglify({ .pipe(isProduction ? uglify({
toplevel: true toplevel: true
} as any) : gutil.noop()) } as any) : gutil.noop())
.pipe(gulp.dest('./built/web/assets/')) as any .pipe(gulp.dest('./built/server/web/assets/')) as any
); );
gulp.task('build:client:styles', () => gulp.task('build:client:styles', () =>
gulp.src('./src/web/app/init.css') gulp.src('./src/server/web/app/init.css')
.pipe(isProduction .pipe(isProduction
? (cssnano as any)() ? (cssnano as any)()
: gutil.noop()) : gutil.noop())
.pipe(gulp.dest('./built/web/assets/')) .pipe(gulp.dest('./built/server/web/assets/'))
); );
gulp.task('copy:client', [ gulp.task('copy:client', [
@ -145,14 +145,14 @@ gulp.task('copy:client', [
], () => ], () =>
gulp.src([ gulp.src([
'./assets/**/*', './assets/**/*',
'./src/web/assets/**/*', './src/server/web/assets/**/*',
'./src/web/app/*/assets/**/*' './src/server/web/app/*/assets/**/*'
]) ])
.pipe(isProduction ? (imagemin as any)() : gutil.noop()) .pipe(isProduction ? (imagemin as any)() : gutil.noop())
.pipe(rename(path => { .pipe(rename(path => {
path.dirname = path.dirname.replace('assets', '.'); path.dirname = path.dirname.replace('assets', '.');
})) }))
.pipe(gulp.dest('./built/web/assets/')) .pipe(gulp.dest('./built/server/web/assets/'))
); );
gulp.task('build:client:pug', [ gulp.task('build:client:pug', [
@ -160,13 +160,13 @@ gulp.task('build:client:pug', [
'build:client:script', 'build:client:script',
'build:client:styles' 'build:client:styles'
], () => ], () =>
gulp.src('./src/web/app/base.pug') gulp.src('./src/server/web/app/base.pug')
.pipe(pug({ .pipe(pug({
locals: { locals: {
themeColor: constants.themeColor, themeColor: constants.themeColor,
facss: fa.dom.css(), facss: fa.dom.css(),
//hljscss: fs.readFileSync('./node_modules/highlight.js/styles/default.css', 'utf8') //hljscss: fs.readFileSync('./node_modules/highlight.js/styles/default.css', 'utf8')
hljscss: fs.readFileSync('./src/web/assets/code-highlight.css', 'utf8') hljscss: fs.readFileSync('./src/server/web/assets/code-highlight.css', 'utf8')
} }
})) }))
.pipe(htmlmin({ .pipe(htmlmin({
@ -201,5 +201,5 @@ gulp.task('build:client:pug', [
// CSSも圧縮する // CSSも圧縮する
minifyCSS: true minifyCSS: true
})) }))
.pipe(gulp.dest('./built/web/app/')) .pipe(gulp.dest('./built/server/web/app/'))
); );

View File

@ -135,6 +135,7 @@
"is-url": "1.2.3", "is-url": "1.2.3",
"js-yaml": "3.11.0", "js-yaml": "3.11.0",
"jsdom": "^11.6.2", "jsdom": "^11.6.2",
"kue": "^0.11.6",
"license-checker": "18.0.0", "license-checker": "18.0.0",
"loader-utils": "1.1.0", "loader-utils": "1.1.0",
"mecab-async": "0.1.2", "mecab-async": "0.1.2",
@ -149,6 +150,7 @@
"nan": "^2.10.0", "nan": "^2.10.0",
"node-sass": "4.7.2", "node-sass": "4.7.2",
"node-sass-json-importer": "3.1.5", "node-sass-json-importer": "3.1.5",
"nopt": "^4.0.1",
"nprogress": "0.2.0", "nprogress": "0.2.0",
"object-assign-deep": "0.3.1", "object-assign-deep": "0.3.1",
"on-build-webpack": "0.1.0", "on-build-webpack": "0.1.0",

View File

@ -2,7 +2,7 @@
* Replace i18n texts * Replace i18n texts
*/ */
import locale from '../../../locales'; import locale from '../../locales';
export default class Replacer { export default class Replacer {
private lang: string; private lang: string;

View File

@ -1,6 +1,6 @@
import * as fs from 'fs'; import * as fs from 'fs';
const license = fs.readFileSync(__dirname + '/../../../LICENSE', 'utf-8'); const license = fs.readFileSync(__dirname + '/../../LICENSE', 'utf-8');
const licenseHtml = license const licenseHtml = license
.replace(/\r\n/g, '\n') .replace(/\r\n/g, '\n')

View File

@ -24,6 +24,8 @@ import stats from './utils/stats';
import { Config, path as configPath } from './config'; import { Config, path as configPath } from './config';
import loadConfig from './config'; import loadConfig from './config';
import parseOpt from './parse-opt';
const clusterLog = debug('misskey:cluster'); const clusterLog = debug('misskey:cluster');
const ev = new Xev(); const ev = new Xev();
@ -36,20 +38,22 @@ main();
* Init process * Init process
*/ */
function main() { function main() {
const opt = parseOpt(process.argv, 2);
if (cluster.isMaster) { if (cluster.isMaster) {
masterMain(); masterMain(opt);
ev.mount(); ev.mount();
stats(); stats();
} else { } else {
workerMain(); workerMain(opt);
} }
} }
/** /**
* Init master process * Init master process
*/ */
async function masterMain() { async function masterMain(opt) {
let config: Config; let config: Config;
try { try {
@ -69,19 +73,35 @@ async function masterMain() {
} }
spawnWorkers(() => { spawnWorkers(() => {
Logger.info(chalk.bold.green( if (!opt['only-processor']) {
`Now listening on port ${chalk.underline(config.port.toString())}`)); Logger.info(chalk.bold.green(
`Now listening on port ${chalk.underline(config.port.toString())}`));
Logger.info(chalk.bold.green(config.url)); Logger.info(chalk.bold.green(config.url));
}
if (!opt['only-server']) {
Logger.info(chalk.bold.green('Now processing jobs'));
}
}); });
} }
/** /**
* Init worker process * Init worker process
*/ */
function workerMain() { async function workerMain(opt) {
// start server if (!opt['only-processor']) {
require('./server'); // start server
await require('./server').default();
}
if (!opt['only-server']) {
// start processor
require('./processor').default();
}
// Send a 'ready' message to parent process
process.send('ready');
} }
/** /**

17
src/parse-opt.ts Normal file
View File

@ -0,0 +1,17 @@
const nopt = require('nopt');
export default (vector, index) => {
const parsed = nopt({
'only-processor': Boolean,
'only-server': Boolean
}, {
p: ['--only-processor'],
s: ['--only-server']
}, vector, index);
if (parsed['only-processor'] && parsed['only-server']) {
throw 'only-processor option and only-server option cannot be set at the same time';
}
return parsed;
};

4
src/processor/index.ts Normal file
View File

@ -0,0 +1,4 @@
import queue from '../queue';
import reportGitHubFailure from './report-github-failure';
export default () => queue.process('gitHubFailureReport', reportGitHubFailure);

View File

@ -0,0 +1,29 @@
import * as request from 'request';
import User from '../server/api/models/user';
const createPost = require('../server/api/endpoints/posts/create');
export default ({ data }, done) => {
const asyncBot = User.findOne({ _id: data.userId });
// Fetch parent status
request({
url: `${data.parentUrl}/statuses`,
headers: {
'User-Agent': 'misskey'
}
}, async (err, res, body) => {
if (err) {
console.error(err);
return;
}
const parentStatuses = JSON.parse(body);
const parentState = parentStatuses[0].state;
const stillFailed = parentState == 'failure' || parentState == 'error';
const text = stillFailed ?
`**⚠BUILD STILL FAILED⚠**: ?[${data.message}](${data.htmlUrl})` :
`**🚨BUILD FAILED🚨**: →→→?[${data.message}](${data.htmlUrl})←←←`;
createPost({ text }, await asyncBot);
done();
});
};

10
src/queue.ts Normal file
View File

@ -0,0 +1,10 @@
import { createQueue } from 'kue';
import config from './conf';
export default createQueue({
redis: {
port: config.redis.port,
host: config.redis.host,
auth: config.redis.pass
}
});

View File

@ -3,9 +3,9 @@ import * as express from 'express';
import * as request from 'request'; import * as request from 'request';
import * as crypto from 'crypto'; import * as crypto from 'crypto';
import User from '../../models/user'; import User from '../../models/user';
import config from '../../../conf'; import config from '../../../../conf';
import BotCore from '../core'; import BotCore from '../core';
import _redis from '../../../db/redis'; import _redis from '../../../../db/redis';
import prominence = require('prominence'); import prominence = require('prominence');
import getAcct from '../../../common/user/get-acct'; import getAcct from '../../../common/user/get-acct';
import parseAcct from '../../../common/user/parse-acct'; import parseAcct from '../../../common/user/parse-acct';

View File

@ -15,7 +15,7 @@ import DriveFolder from '../../models/drive-folder';
import { pack } from '../../models/drive-file'; import { pack } from '../../models/drive-file';
import event, { publishDriveStream } from '../../event'; import event, { publishDriveStream } from '../../event';
import getAcct from '../../../common/user/get-acct'; import getAcct from '../../../common/user/get-acct';
import config from '../../../conf'; import config from '../../../../conf';
const gm = _gm.subClass({ const gm = _gm.subClass({
imageMagick: true imageMagick: true

View File

@ -1,7 +1,7 @@
const push = require('web-push'); const push = require('web-push');
import * as mongo from 'mongodb'; import * as mongo from 'mongodb';
import Subscription from '../models/sw-subscription'; import Subscription from '../models/sw-subscription';
import config from '../../conf'; import config from '../../../conf';
if (config.sw) { if (config.sw) {
// アプリケーションの連絡先と、サーバーサイドの鍵ペアの情報を登録 // アプリケーションの連絡先と、サーバーサイドの鍵ペアの情報を登録

View File

@ -1,4 +1,4 @@
import config from '../../conf'; import config from '../../../conf';
export default function(res, user, redirect: boolean) { export default function(res, user, redirect: boolean) {
const expires = 1000 * 60 * 60 * 24 * 365; // One Year const expires = 1000 * 60 * 60 * 24 * 365; // One Year

View File

@ -5,7 +5,7 @@ import * as uuid from 'uuid';
import $ from 'cafy'; import $ from 'cafy';
import App from '../../../models/app'; import App from '../../../models/app';
import AuthSess from '../../../models/auth-session'; import AuthSess from '../../../models/auth-session';
import config from '../../../../conf'; import config from '../../../../../conf';
/** /**
* @swagger * @swagger

View File

@ -6,7 +6,7 @@ import * as bcrypt from 'bcryptjs';
import * as speakeasy from 'speakeasy'; import * as speakeasy from 'speakeasy';
import * as QRCode from 'qrcode'; import * as QRCode from 'qrcode';
import User from '../../../models/user'; import User from '../../../models/user';
import config from '../../../../conf'; import config from '../../../../../conf';
module.exports = async (params, user) => new Promise(async (res, rej) => { module.exports = async (params, user) => new Promise(async (res, rej) => {
// Get 'password' parameter // Get 'password' parameter

View File

@ -4,7 +4,7 @@
import $ from 'cafy'; import $ from 'cafy';
import User, { isValidName, isValidDescription, isValidLocation, isValidBirthday, pack } from '../../models/user'; import User, { isValidName, isValidDescription, isValidLocation, isValidBirthday, pack } from '../../models/user';
import event from '../../event'; import event from '../../event';
import config from '../../../conf'; import config from '../../../../conf';
/** /**
* Update myself * Update myself

View File

@ -11,7 +11,7 @@ import DriveFile from '../../../models/drive-file';
import { pack } from '../../../models/messaging-message'; import { pack } from '../../../models/messaging-message';
import publishUserStream from '../../../event'; import publishUserStream from '../../../event';
import { publishMessagingStream, publishMessagingIndexStream, pushSw } from '../../../event'; import { publishMessagingStream, publishMessagingIndexStream, pushSw } from '../../../event';
import config from '../../../../conf'; import config from '../../../../../conf';
/** /**
* Create a message * Create a message

View File

@ -2,8 +2,8 @@
* Module dependencies * Module dependencies
*/ */
import * as os from 'os'; import * as os from 'os';
import version from '../../version'; import version from '../../../version';
import config from '../../conf'; import config from '../../../conf';
import Meta from '../models/meta'; import Meta from '../models/meta';
/** /**

Some files were not shown because too many files have changed in this diff Show More