Merge branch 'develop' of https://github.com/misskey-dev/misskey into develop
This commit is contained in:
		
						commit
						23753ec75a
					
				
					 20 changed files with 311 additions and 207 deletions
				
			
		
							
								
								
									
										10
									
								
								.github/workflows/nodejs.yml
									
										
									
									
										vendored
									
									
								
							
							
						
						
									
										10
									
								
								.github/workflows/nodejs.yml
									
										
									
									
										vendored
									
									
								
							|  | @ -16,16 +16,16 @@ jobs: | ||||||
| 
 | 
 | ||||||
|     services: |     services: | ||||||
|       postgres: |       postgres: | ||||||
|         image: postgres:10-alpine |         image: postgres:12.2-alpine | ||||||
|         ports: |         ports: | ||||||
|           - 5432:5432 |           - 54312:5432 | ||||||
|         env: |         env: | ||||||
|           POSTGRES_DB: test-misskey |           POSTGRES_DB: test-misskey | ||||||
|           POSTGRES_HOST_AUTH_METHOD: trust |           POSTGRES_HOST_AUTH_METHOD: trust | ||||||
|       redis: |       redis: | ||||||
|         image: redis:alpine |         image: redis:4.0-alpine | ||||||
|         ports: |         ports: | ||||||
|           - 6379:6379 |           - 56312:6379 | ||||||
| 
 | 
 | ||||||
|     steps: |     steps: | ||||||
|     - uses: actions/checkout@v2 |     - uses: actions/checkout@v2 | ||||||
|  | @ -40,7 +40,7 @@ jobs: | ||||||
|     - name: Check yarn.lock |     - name: Check yarn.lock | ||||||
|       run: git diff --exit-code yarn.lock |       run: git diff --exit-code yarn.lock | ||||||
|     - name: Copy Configure |     - name: Copy Configure | ||||||
|       run: cp .circleci/misskey/*.yml .config |       run: cp test/test.yml .config | ||||||
|     - name: Build |     - name: Build | ||||||
|       run: yarn build |       run: yarn build | ||||||
|     - name: Test |     - name: Test | ||||||
|  |  | ||||||
|  | @ -57,6 +57,17 @@ If your language is not listed in Crowdin, please open an issue. | ||||||
| - Test codes are located in [`/test`](/test). | - Test codes are located in [`/test`](/test). | ||||||
| 
 | 
 | ||||||
| ### Run test | ### Run test | ||||||
|  | Create a config file. | ||||||
|  | ``` | ||||||
|  | cp test/test.yml .config/ | ||||||
|  | ``` | ||||||
|  | Prepare DB/Redis for testing. | ||||||
|  | ``` | ||||||
|  | docker-compose -f test/docker-compose.yml up | ||||||
|  | ``` | ||||||
|  | Alternatively, prepare an empty (data can be erased) DB and edit `.config/test.yml`.  | ||||||
|  | 
 | ||||||
|  | Run all test. | ||||||
| ``` | ``` | ||||||
| npm run test | npm run test | ||||||
| ``` | ``` | ||||||
|  |  | ||||||
|  | @ -12,7 +12,6 @@ | ||||||
| <script lang="ts"> | <script lang="ts"> | ||||||
| import { defineComponent } from 'vue'; | import { defineComponent } from 'vue'; | ||||||
| import MkPagination from '@client/components/ui/pagination.vue'; | import MkPagination from '@client/components/ui/pagination.vue'; | ||||||
| import { userPage, acct } from '@client/filters/user'; |  | ||||||
| 
 | 
 | ||||||
| export default defineComponent({ | export default defineComponent({ | ||||||
| 	components: { | 	components: { | ||||||
|  | @ -43,12 +42,6 @@ export default defineComponent({ | ||||||
| 			this.$refs.list.reload(); | 			this.$refs.list.reload(); | ||||||
| 		} | 		} | ||||||
| 	}, | 	}, | ||||||
| 
 |  | ||||||
| 	methods: { |  | ||||||
| 		userPage, |  | ||||||
| 		 |  | ||||||
| 		acct |  | ||||||
| 	} |  | ||||||
| }); | }); | ||||||
| </script> | </script> | ||||||
| 
 | 
 | ||||||
|  |  | ||||||
|  | @ -12,7 +12,6 @@ | ||||||
| import { defineComponent } from 'vue'; | import { defineComponent } from 'vue'; | ||||||
| import MkUserInfo from '@client/components/user-info.vue'; | import MkUserInfo from '@client/components/user-info.vue'; | ||||||
| import MkPagination from '@client/components/ui/pagination.vue'; | import MkPagination from '@client/components/ui/pagination.vue'; | ||||||
| import { userPage, acct } from '@client/filters/user'; |  | ||||||
| 
 | 
 | ||||||
| export default defineComponent({ | export default defineComponent({ | ||||||
| 	components: { | 	components: { | ||||||
|  | @ -51,12 +50,6 @@ export default defineComponent({ | ||||||
| 		user() { | 		user() { | ||||||
| 			this.$refs.list.reload(); | 			this.$refs.list.reload(); | ||||||
| 		} | 		} | ||||||
| 	}, |  | ||||||
| 
 |  | ||||||
| 	methods: { |  | ||||||
| 		userPage, |  | ||||||
| 		 |  | ||||||
| 		acct |  | ||||||
| 	} | 	} | ||||||
| }); | }); | ||||||
| </script> | </script> | ||||||
|  |  | ||||||
|  | @ -12,7 +12,6 @@ | ||||||
| import { defineComponent } from 'vue'; | import { defineComponent } from 'vue'; | ||||||
| import MkGalleryPostPreview from '@client/components/gallery-post-preview.vue'; | import MkGalleryPostPreview from '@client/components/gallery-post-preview.vue'; | ||||||
| import MkPagination from '@client/components/ui/pagination.vue'; | import MkPagination from '@client/components/ui/pagination.vue'; | ||||||
| import { userPage, acct } from '@client/filters/user'; |  | ||||||
| 
 | 
 | ||||||
| export default defineComponent({ | export default defineComponent({ | ||||||
| 	components: { | 	components: { | ||||||
|  | @ -43,12 +42,6 @@ export default defineComponent({ | ||||||
| 		user() { | 		user() { | ||||||
| 			this.$refs.list.reload(); | 			this.$refs.list.reload(); | ||||||
| 		} | 		} | ||||||
| 	}, |  | ||||||
| 
 |  | ||||||
| 	methods: { |  | ||||||
| 		userPage, |  | ||||||
| 		 |  | ||||||
| 		acct |  | ||||||
| 	} | 	} | ||||||
| }); | }); | ||||||
| </script> | </script> | ||||||
|  |  | ||||||
|  | @ -10,7 +10,6 @@ | ||||||
| import { defineComponent } from 'vue'; | import { defineComponent } from 'vue'; | ||||||
| import MkPagePreview from '@client/components/page-preview.vue'; | import MkPagePreview from '@client/components/page-preview.vue'; | ||||||
| import MkPagination from '@client/components/ui/pagination.vue'; | import MkPagination from '@client/components/ui/pagination.vue'; | ||||||
| import { userPage, acct } from '@client/filters/user'; |  | ||||||
| 
 | 
 | ||||||
| export default defineComponent({ | export default defineComponent({ | ||||||
| 	components: { | 	components: { | ||||||
|  | @ -41,12 +40,6 @@ export default defineComponent({ | ||||||
| 		user() { | 		user() { | ||||||
| 			this.$refs.list.reload(); | 			this.$refs.list.reload(); | ||||||
| 		} | 		} | ||||||
| 	}, |  | ||||||
| 
 |  | ||||||
| 	methods: { |  | ||||||
| 		userPage, |  | ||||||
| 		 |  | ||||||
| 		acct |  | ||||||
| 	} | 	} | ||||||
| }); | }); | ||||||
| </script> | </script> | ||||||
|  |  | ||||||
|  | @ -2,7 +2,7 @@ import * as fs from 'fs'; | ||||||
| import * as stream from 'stream'; | import * as stream from 'stream'; | ||||||
| import * as util from 'util'; | import * as util from 'util'; | ||||||
| import got, * as Got from 'got'; | import got, * as Got from 'got'; | ||||||
| import { httpAgent, httpsAgent } from './fetch'; | import { httpAgent, httpsAgent, StatusError } from './fetch'; | ||||||
| import config from '@/config/index'; | import config from '@/config/index'; | ||||||
| import * as chalk from 'chalk'; | import * as chalk from 'chalk'; | ||||||
| import Logger from '@/services/logger'; | import Logger from '@/services/logger'; | ||||||
|  | @ -37,6 +37,7 @@ export async function downloadUrl(url: string, path: string) { | ||||||
| 			http: httpAgent, | 			http: httpAgent, | ||||||
| 			https: httpsAgent, | 			https: httpsAgent, | ||||||
| 		}, | 		}, | ||||||
|  | 		http2: false,	// default
 | ||||||
| 		retry: 0, | 		retry: 0, | ||||||
| 	}).on('response', (res: Got.Response) => { | 	}).on('response', (res: Got.Response) => { | ||||||
| 		if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !config.proxy && res.ip) { | 		if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !config.proxy && res.ip) { | ||||||
|  | @ -59,17 +60,17 @@ export async function downloadUrl(url: string, path: string) { | ||||||
| 			logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`); | 			logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`); | ||||||
| 			req.destroy(); | 			req.destroy(); | ||||||
| 		} | 		} | ||||||
| 	}).on('error', (e: any) => { |  | ||||||
| 		if (e.name === 'HTTPError') { |  | ||||||
| 			const statusCode = e.response?.statusCode; |  | ||||||
| 			const statusMessage = e.response?.statusMessage; |  | ||||||
| 			e.name = `StatusError`; |  | ||||||
| 			e.statusCode = statusCode; |  | ||||||
| 			e.message = `${statusCode} ${statusMessage}`; |  | ||||||
| 		} |  | ||||||
| 	}); | 	}); | ||||||
| 
 | 
 | ||||||
| 	await pipeline(req, fs.createWriteStream(path)); | 	try { | ||||||
|  | 		await pipeline(req, fs.createWriteStream(path)); | ||||||
|  | 	} catch (e) { | ||||||
|  | 		if (e instanceof Got.HTTPError) { | ||||||
|  | 			throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage); | ||||||
|  | 		} else { | ||||||
|  | 			throw e; | ||||||
|  | 		} | ||||||
|  | 	} | ||||||
| 
 | 
 | ||||||
| 	logger.succ(`Download finished: ${chalk.cyan(url)}`); | 	logger.succ(`Download finished: ${chalk.cyan(url)}`); | ||||||
| } | } | ||||||
|  |  | ||||||
|  | @ -1,51 +1,62 @@ | ||||||
| import * as http from 'http'; | import * as http from 'http'; | ||||||
| import * as https from 'https'; | import * as https from 'https'; | ||||||
| import CacheableLookup from 'cacheable-lookup'; | import CacheableLookup from 'cacheable-lookup'; | ||||||
| import fetch, { HeadersInit } from 'node-fetch'; | import fetch from 'node-fetch'; | ||||||
| import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'; | import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'; | ||||||
| import config from '@/config/index'; | import config from '@/config/index'; | ||||||
| import { URL } from 'url'; | import { URL } from 'url'; | ||||||
| 
 | 
 | ||||||
| export async function getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: HeadersInit) { | export async function getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record<string, string>) { | ||||||
| 	const res = await fetch(url, { | 	const res = await getResponse({ | ||||||
|  | 		url, | ||||||
|  | 		method: 'GET', | ||||||
| 		headers: Object.assign({ | 		headers: Object.assign({ | ||||||
| 			'User-Agent': config.userAgent, | 			'User-Agent': config.userAgent, | ||||||
| 			Accept: accept | 			Accept: accept | ||||||
| 		}, headers || {}), | 		}, headers || {}), | ||||||
| 		timeout, | 		timeout | ||||||
| 		agent: getAgentByUrl, |  | ||||||
| 	}); | 	}); | ||||||
| 
 | 
 | ||||||
| 	if (!res.ok) { |  | ||||||
| 		throw { |  | ||||||
| 			name: `StatusError`, |  | ||||||
| 			statusCode: res.status, |  | ||||||
| 			message: `${res.status} ${res.statusText}`, |  | ||||||
| 		}; |  | ||||||
| 	} |  | ||||||
| 
 |  | ||||||
| 	return await res.json(); | 	return await res.json(); | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| export async function getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: HeadersInit) { | export async function getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: Record<string, string>) { | ||||||
| 	const res = await fetch(url, { | 	const res = await getResponse({ | ||||||
|  | 		url, | ||||||
|  | 		method: 'GET', | ||||||
| 		headers: Object.assign({ | 		headers: Object.assign({ | ||||||
| 			'User-Agent': config.userAgent, | 			'User-Agent': config.userAgent, | ||||||
| 			Accept: accept | 			Accept: accept | ||||||
| 		}, headers || {}), | 		}, headers || {}), | ||||||
|  | 		timeout | ||||||
|  | 	}); | ||||||
|  | 
 | ||||||
|  | 	return await res.text(); | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | export async function getResponse(args: { url: string, method: string, body?: string, headers: Record<string, string>, timeout?: number, size?: number }) { | ||||||
|  | 	const timeout = args?.timeout || 10 * 1000; | ||||||
|  | 
 | ||||||
|  | 	const controller = new AbortController(); | ||||||
|  | 	setTimeout(() => { | ||||||
|  | 		controller.abort(); | ||||||
|  | 	}, timeout * 6); | ||||||
|  | 
 | ||||||
|  | 	const res = await fetch(args.url, { | ||||||
|  | 		method: args.method, | ||||||
|  | 		headers: args.headers, | ||||||
|  | 		body: args.body, | ||||||
| 		timeout, | 		timeout, | ||||||
|  | 		size: args?.size || 10 * 1024 * 1024, | ||||||
| 		agent: getAgentByUrl, | 		agent: getAgentByUrl, | ||||||
|  | 		signal: controller.signal, | ||||||
| 	}); | 	}); | ||||||
| 
 | 
 | ||||||
| 	if (!res.ok) { | 	if (!res.ok) { | ||||||
| 		throw { | 		throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText); | ||||||
| 			name: `StatusError`, |  | ||||||
| 			statusCode: res.status, |  | ||||||
| 			message: `${res.status} ${res.statusText}`, |  | ||||||
| 		}; |  | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
| 	return await res.text(); | 	return res; | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| const cache = new CacheableLookup({ | const cache = new CacheableLookup({ | ||||||
|  | @ -114,3 +125,17 @@ export function getAgentByUrl(url: URL, bypassProxy = false) { | ||||||
| 		return url.protocol == 'http:' ? httpAgent : httpsAgent; | 		return url.protocol == 'http:' ? httpAgent : httpsAgent; | ||||||
| 	} | 	} | ||||||
| } | } | ||||||
|  | 
 | ||||||
|  | export class StatusError extends Error { | ||||||
|  | 	public statusCode: number; | ||||||
|  | 	public statusMessage?: string; | ||||||
|  | 	public isClientError: boolean; | ||||||
|  | 
 | ||||||
|  | 	constructor(message: string, statusCode: number, statusMessage?: string) { | ||||||
|  | 		super(message); | ||||||
|  | 		this.name = 'StatusError'; | ||||||
|  | 		this.statusCode = statusCode; | ||||||
|  | 		this.statusMessage = statusMessage; | ||||||
|  | 		this.isClientError = typeof this.statusCode === 'number' && this.statusCode >= 400 && this.statusCode < 500; | ||||||
|  | 	} | ||||||
|  | } | ||||||
|  |  | ||||||
|  | @ -11,6 +11,7 @@ import { toPuny } from '@/misc/convert-host'; | ||||||
| import { Cache } from '@/misc/cache'; | import { Cache } from '@/misc/cache'; | ||||||
| import { Instance } from '@/models/entities/instance'; | import { Instance } from '@/models/entities/instance'; | ||||||
| import { DeliverJobData } from '../types'; | import { DeliverJobData } from '../types'; | ||||||
|  | import { StatusError } from '@/misc/fetch'; | ||||||
| 
 | 
 | ||||||
| const logger = new Logger('deliver'); | const logger = new Logger('deliver'); | ||||||
| 
 | 
 | ||||||
|  | @ -68,16 +69,16 @@ export default async (job: Bull.Job<DeliverJobData>) => { | ||||||
| 		registerOrFetchInstanceDoc(host).then(i => { | 		registerOrFetchInstanceDoc(host).then(i => { | ||||||
| 			Instances.update(i.id, { | 			Instances.update(i.id, { | ||||||
| 				latestRequestSentAt: new Date(), | 				latestRequestSentAt: new Date(), | ||||||
| 				latestStatus: res != null && res.hasOwnProperty('statusCode') ? res.statusCode : null, | 				latestStatus: res instanceof StatusError ? res.statusCode : null, | ||||||
| 				isNotResponding: true | 				isNotResponding: true | ||||||
| 			}); | 			}); | ||||||
| 
 | 
 | ||||||
| 			instanceChart.requestSent(i.host, false); | 			instanceChart.requestSent(i.host, false); | ||||||
| 		}); | 		}); | ||||||
| 
 | 
 | ||||||
| 		if (res != null && res.hasOwnProperty('statusCode')) { | 		if (res instanceof StatusError) { | ||||||
| 			// 4xx
 | 			// 4xx
 | ||||||
| 			if (res.statusCode >= 400 && res.statusCode < 500) { | 			if (res.isClientError) { | ||||||
| 				// HTTPステータスコード4xxはクライアントエラーであり、それはつまり
 | 				// HTTPステータスコード4xxはクライアントエラーであり、それはつまり
 | ||||||
| 				// 何回再送しても成功することはないということなのでエラーにはしないでおく
 | 				// 何回再送しても成功することはないということなのでエラーにはしないでおく
 | ||||||
| 				return `${res.statusCode} ${res.statusMessage}`; | 				return `${res.statusCode} ${res.statusMessage}`; | ||||||
|  |  | ||||||
|  | @ -14,6 +14,7 @@ import { InboxJobData } from '../types'; | ||||||
| import DbResolver from '@/remote/activitypub/db-resolver'; | import DbResolver from '@/remote/activitypub/db-resolver'; | ||||||
| import { resolvePerson } from '@/remote/activitypub/models/person'; | import { resolvePerson } from '@/remote/activitypub/models/person'; | ||||||
| import { LdSignature } from '@/remote/activitypub/misc/ld-signature'; | import { LdSignature } from '@/remote/activitypub/misc/ld-signature'; | ||||||
|  | import { StatusError } from '@/misc/fetch'; | ||||||
| 
 | 
 | ||||||
| const logger = new Logger('inbox'); | const logger = new Logger('inbox'); | ||||||
| 
 | 
 | ||||||
|  | @ -53,7 +54,7 @@ export default async (job: Bull.Job<InboxJobData>): Promise<string> => { | ||||||
| 			authUser = await dbResolver.getAuthUserFromApId(getApId(activity.actor)); | 			authUser = await dbResolver.getAuthUserFromApId(getApId(activity.actor)); | ||||||
| 		} catch (e) { | 		} catch (e) { | ||||||
| 			// 対象が4xxならスキップ
 | 			// 対象が4xxならスキップ
 | ||||||
| 			if (e.statusCode >= 400 && e.statusCode < 500) { | 			if (e instanceof StatusError && e.isClientError) { | ||||||
| 				return `skip: Ignored deleted actors on both ends ${activity.actor} - ${e.statusCode}`; | 				return `skip: Ignored deleted actors on both ends ${activity.actor} - ${e.statusCode}`; | ||||||
| 			} | 			} | ||||||
| 			throw `Error in actor ${activity.actor} - ${e.statusCode || e}`; | 			throw `Error in actor ${activity.actor} - ${e.statusCode || e}`; | ||||||
|  |  | ||||||
							
								
								
									
										104
									
								
								src/remote/activitypub/ap-request.ts
									
										
									
									
									
										Normal file
									
								
							
							
						
						
									
										104
									
								
								src/remote/activitypub/ap-request.ts
									
										
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,104 @@ | ||||||
|  | import * as crypto from 'crypto'; | ||||||
|  | import { URL } from 'url'; | ||||||
|  | 
 | ||||||
|  | type Request = { | ||||||
|  | 	url: string; | ||||||
|  | 	method: string; | ||||||
|  | 	headers: Record<string, string>; | ||||||
|  | }; | ||||||
|  | 
 | ||||||
|  | type PrivateKey = { | ||||||
|  | 	privateKeyPem: string; | ||||||
|  | 	keyId: string; | ||||||
|  | }; | ||||||
|  | 
 | ||||||
|  | export function createSignedPost(args: { key: PrivateKey, url: string, body: string, additionalHeaders: Record<string, string> }) { | ||||||
|  | 	const u = new URL(args.url); | ||||||
|  | 	const digestHeader = `SHA-256=${crypto.createHash('sha256').update(args.body).digest('base64')}`; | ||||||
|  | 
 | ||||||
|  | 	const request: Request = { | ||||||
|  | 		url: u.href, | ||||||
|  | 		method: 'POST', | ||||||
|  | 		headers:  objectAssignWithLcKey({ | ||||||
|  | 			'Date': new Date().toUTCString(), | ||||||
|  | 			'Host': u.hostname, | ||||||
|  | 			'Content-Type': 'application/activity+json', | ||||||
|  | 			'Digest': digestHeader, | ||||||
|  | 		}, args.additionalHeaders), | ||||||
|  | 	}; | ||||||
|  | 
 | ||||||
|  | 	const result = signToRequest(request, args.key, ['(request-target)', 'date', 'host', 'digest']); | ||||||
|  | 
 | ||||||
|  | 	return { | ||||||
|  | 		request, | ||||||
|  | 		signingString: result.signingString, | ||||||
|  | 		signature: result.signature, | ||||||
|  | 		signatureHeader: result.signatureHeader, | ||||||
|  | 	}; | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | export function createSignedGet(args: { key: PrivateKey, url: string, additionalHeaders: Record<string, string> }) { | ||||||
|  | 	const u = new URL(args.url); | ||||||
|  | 
 | ||||||
|  | 	const request: Request = { | ||||||
|  | 		url: u.href, | ||||||
|  | 		method: 'GET', | ||||||
|  | 		headers:  objectAssignWithLcKey({ | ||||||
|  | 			'Accept': 'application/activity+json, application/ld+json', | ||||||
|  | 			'Date': new Date().toUTCString(), | ||||||
|  | 			'Host': new URL(args.url).hostname, | ||||||
|  | 		}, args.additionalHeaders), | ||||||
|  | 	}; | ||||||
|  | 
 | ||||||
|  | 	const result = signToRequest(request, args.key, ['(request-target)', 'date', 'host', 'accept']); | ||||||
|  | 
 | ||||||
|  | 	return { | ||||||
|  | 		request, | ||||||
|  | 		signingString: result.signingString, | ||||||
|  | 		signature: result.signature, | ||||||
|  | 		signatureHeader: result.signatureHeader, | ||||||
|  | 	}; | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | function signToRequest(request: Request, key: PrivateKey, includeHeaders: string[]) { | ||||||
|  | 	const signingString = genSigningString(request, includeHeaders); | ||||||
|  | 	const signature = crypto.sign('sha256', Buffer.from(signingString), key.privateKeyPem).toString('base64'); | ||||||
|  | 	const signatureHeader = `keyId="${key.keyId}",algorithm="rsa-sha256",headers="${includeHeaders.join(' ')}",signature="${signature}"`; | ||||||
|  | 
 | ||||||
|  | 	request.headers = objectAssignWithLcKey(request.headers, { | ||||||
|  | 		Signature: signatureHeader | ||||||
|  | 	}); | ||||||
|  | 
 | ||||||
|  | 	return { | ||||||
|  | 		request, | ||||||
|  | 		signingString, | ||||||
|  | 		signature, | ||||||
|  | 		signatureHeader, | ||||||
|  | 	}; | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | function genSigningString(request: Request, includeHeaders: string[]) { | ||||||
|  | 	request.headers = lcObjectKey(request.headers); | ||||||
|  | 
 | ||||||
|  | 	const results: string[] = []; | ||||||
|  | 
 | ||||||
|  | 	for (const key of includeHeaders.map(x => x.toLowerCase())) { | ||||||
|  | 		if (key === '(request-target)') { | ||||||
|  | 			results.push(`(request-target): ${request.method.toLowerCase()} ${new URL(request.url).pathname}`); | ||||||
|  | 		} else { | ||||||
|  | 			results.push(`${key}: ${request.headers[key]}`); | ||||||
|  | 		} | ||||||
|  | 	} | ||||||
|  | 
 | ||||||
|  | 	return results.join('\n'); | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | function lcObjectKey(src: Record<string, string>) { | ||||||
|  | 	const dst: Record<string, string> = {}; | ||||||
|  | 	for (const key of Object.keys(src).filter(x => x != '__proto__' && typeof src[x] === 'string')) dst[key.toLowerCase()] = src[key]; | ||||||
|  | 	return dst; | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | function objectAssignWithLcKey(a: Record<string, string>, b: Record<string, string>) { | ||||||
|  | 	return Object.assign(lcObjectKey(a), lcObjectKey(b)); | ||||||
|  | } | ||||||
|  | @ -8,6 +8,7 @@ import { extractDbHost } from '@/misc/convert-host'; | ||||||
| import { fetchMeta } from '@/misc/fetch-meta'; | import { fetchMeta } from '@/misc/fetch-meta'; | ||||||
| import { getApLock } from '@/misc/app-lock'; | import { getApLock } from '@/misc/app-lock'; | ||||||
| import { parseAudience } from '../../audience'; | import { parseAudience } from '../../audience'; | ||||||
|  | import { StatusError } from '@/misc/fetch'; | ||||||
| 
 | 
 | ||||||
| const logger = apLogger; | const logger = apLogger; | ||||||
| 
 | 
 | ||||||
|  | @ -41,7 +42,7 @@ export default async function(resolver: Resolver, actor: IRemoteUser, activity: | ||||||
| 			renote = await resolveNote(targetUri); | 			renote = await resolveNote(targetUri); | ||||||
| 		} catch (e) { | 		} catch (e) { | ||||||
| 			// 対象が4xxならスキップ
 | 			// 対象が4xxならスキップ
 | ||||||
| 			if (e.statusCode >= 400 && e.statusCode < 500) { | 			if (e instanceof StatusError && e.isClientError) { | ||||||
| 				logger.warn(`Ignored announce target ${targetUri} - ${e.statusCode}`); | 				logger.warn(`Ignored announce target ${targetUri} - ${e.statusCode}`); | ||||||
| 				return; | 				return; | ||||||
| 			} | 			} | ||||||
|  |  | ||||||
|  | @ -4,6 +4,7 @@ import { createNote, fetchNote } from '../../models/note'; | ||||||
| import { getApId, IObject, ICreate } from '../../type'; | import { getApId, IObject, ICreate } from '../../type'; | ||||||
| import { getApLock } from '@/misc/app-lock'; | import { getApLock } from '@/misc/app-lock'; | ||||||
| import { extractDbHost } from '@/misc/convert-host'; | import { extractDbHost } from '@/misc/convert-host'; | ||||||
|  | import { StatusError } from '@/misc/fetch'; | ||||||
| 
 | 
 | ||||||
| /** | /** | ||||||
|  * 投稿作成アクティビティを捌きます |  * 投稿作成アクティビティを捌きます | ||||||
|  | @ -32,7 +33,7 @@ export default async function(resolver: Resolver, actor: IRemoteUser, note: IObj | ||||||
| 		await createNote(note, resolver, silent); | 		await createNote(note, resolver, silent); | ||||||
| 		return 'ok'; | 		return 'ok'; | ||||||
| 	} catch (e) { | 	} catch (e) { | ||||||
| 		if (e.statusCode >= 400 && e.statusCode < 500) { | 		if (e instanceof StatusError && e.isClientError) { | ||||||
| 			return `skip ${e.statusCode}`; | 			return `skip ${e.statusCode}`; | ||||||
| 		} else { | 		} else { | ||||||
| 			throw e; | 			throw e; | ||||||
|  |  | ||||||
|  | @ -26,6 +26,7 @@ import { createMessage } from '@/services/messages/create'; | ||||||
| import { parseAudience } from '../audience'; | import { parseAudience } from '../audience'; | ||||||
| import { extractApMentions } from './mention'; | import { extractApMentions } from './mention'; | ||||||
| import DbResolver from '../db-resolver'; | import DbResolver from '../db-resolver'; | ||||||
|  | import { StatusError } from '@/misc/fetch'; | ||||||
| 
 | 
 | ||||||
| const logger = apLogger; | const logger = apLogger; | ||||||
| 
 | 
 | ||||||
|  | @ -177,7 +178,7 @@ export async function createNote(value: string | IObject, resolver?: Resolver, s | ||||||
| 				} | 				} | ||||||
| 			} catch (e) { | 			} catch (e) { | ||||||
| 				return { | 				return { | ||||||
| 					status: e.statusCode >= 400 && e.statusCode < 500 ? 'permerror' : 'temperror' | 					status: (e instanceof StatusError && e.isClientError) ? 'permerror' : 'temperror' | ||||||
| 				}; | 				}; | ||||||
| 			} | 			} | ||||||
| 		}; | 		}; | ||||||
|  |  | ||||||
|  | @ -1,66 +1,31 @@ | ||||||
| import * as http from 'http'; |  | ||||||
| import * as https from 'https'; |  | ||||||
| import { sign } from 'http-signature'; |  | ||||||
| import * as crypto from 'crypto'; |  | ||||||
| 
 |  | ||||||
| import config from '@/config/index'; | import config from '@/config/index'; | ||||||
| import { User } from '@/models/entities/user'; |  | ||||||
| import { getAgentByUrl } from '@/misc/fetch'; |  | ||||||
| import { URL } from 'url'; |  | ||||||
| import got from 'got'; |  | ||||||
| import * as Got from 'got'; |  | ||||||
| import { getUserKeypair } from '@/misc/keypair-store'; | import { getUserKeypair } from '@/misc/keypair-store'; | ||||||
|  | import { User } from '@/models/entities/user'; | ||||||
|  | import { getResponse } from '../../misc/fetch'; | ||||||
|  | import { createSignedPost, createSignedGet } from './ap-request'; | ||||||
| 
 | 
 | ||||||
| export default async (user: { id: User['id'] }, url: string, object: any) => { | export default async (user: { id: User['id'] }, url: string, object: any) => { | ||||||
| 	const timeout = 10 * 1000; | 	const body = JSON.stringify(object); | ||||||
| 
 |  | ||||||
| 	const { protocol, hostname, port, pathname, search } = new URL(url); |  | ||||||
| 
 |  | ||||||
| 	const data = JSON.stringify(object); |  | ||||||
| 
 |  | ||||||
| 	const sha256 = crypto.createHash('sha256'); |  | ||||||
| 	sha256.update(data); |  | ||||||
| 	const hash = sha256.digest('base64'); |  | ||||||
| 
 | 
 | ||||||
| 	const keypair = await getUserKeypair(user.id); | 	const keypair = await getUserKeypair(user.id); | ||||||
| 
 | 
 | ||||||
| 	await new Promise<void>((resolve, reject) => { | 	const req = createSignedPost({ | ||||||
| 		const req = https.request({ | 		key: { | ||||||
| 			agent: getAgentByUrl(new URL(`https://example.net`)), | 			privateKeyPem: keypair.privateKey, | ||||||
| 			protocol, | 			keyId: `${config.url}/users/${user.id}#main-key` | ||||||
| 			hostname, | 		}, | ||||||
| 			port, | 		url, | ||||||
| 			method: 'POST', | 		body, | ||||||
| 			path: pathname + search, | 		additionalHeaders: { | ||||||
| 			timeout, | 			'User-Agent': config.userAgent, | ||||||
| 			headers: { | 		} | ||||||
| 				'User-Agent': config.userAgent, | 	}); | ||||||
| 				'Content-Type': 'application/activity+json', |  | ||||||
| 				'Digest': `SHA-256=${hash}` |  | ||||||
| 			} |  | ||||||
| 		}, res => { |  | ||||||
| 			if (res.statusCode! >= 400) { |  | ||||||
| 				reject(res); |  | ||||||
| 			} else { |  | ||||||
| 				resolve(); |  | ||||||
| 			} |  | ||||||
| 		}); |  | ||||||
| 
 | 
 | ||||||
| 		sign(req, { | 	await getResponse({ | ||||||
| 			authorizationHeaderName: 'Signature', | 		url, | ||||||
| 			key: keypair.privateKey, | 		method: req.request.method, | ||||||
| 			keyId: `${config.url}/users/${user.id}#main-key`, | 		headers: req.request.headers, | ||||||
| 			headers: ['(request-target)', 'date', 'host', 'digest'] | 		body, | ||||||
| 		}); |  | ||||||
| 
 |  | ||||||
| 		req.on('timeout', () => req.abort()); |  | ||||||
| 
 |  | ||||||
| 		req.on('error', e => { |  | ||||||
| 			if (req.aborted) reject('timeout'); |  | ||||||
| 			reject(e); |  | ||||||
| 		}); |  | ||||||
| 
 |  | ||||||
| 		req.end(data); |  | ||||||
| 	}); | 	}); | ||||||
| }; | }; | ||||||
| 
 | 
 | ||||||
|  | @ -70,87 +35,24 @@ export default async (user: { id: User['id'] }, url: string, object: any) => { | ||||||
|  * @param url URL to fetch |  * @param url URL to fetch | ||||||
|  */ |  */ | ||||||
| export async function signedGet(url: string, user: { id: User['id'] }) { | export async function signedGet(url: string, user: { id: User['id'] }) { | ||||||
| 	const timeout = 10 * 1000; |  | ||||||
| 
 |  | ||||||
| 	const keypair = await getUserKeypair(user.id); | 	const keypair = await getUserKeypair(user.id); | ||||||
| 
 | 
 | ||||||
| 	const req = got.get<any>(url, { | 	const req = createSignedGet({ | ||||||
| 		headers: { | 		key: { | ||||||
| 			'Accept': 'application/activity+json, application/ld+json', | 			privateKeyPem: keypair.privateKey, | ||||||
|  | 			keyId: `${config.url}/users/${user.id}#main-key` | ||||||
|  | 		}, | ||||||
|  | 		url, | ||||||
|  | 		additionalHeaders: { | ||||||
| 			'User-Agent': config.userAgent, | 			'User-Agent': config.userAgent, | ||||||
| 		}, | 		} | ||||||
| 		responseType: 'json', |  | ||||||
| 		timeout, |  | ||||||
| 		hooks: { |  | ||||||
| 			beforeRequest: [ |  | ||||||
| 				options => { |  | ||||||
| 					options.request = (url: URL, opt: http.RequestOptions, callback?: (response: any) => void) => { |  | ||||||
| 						// Select custom agent by URL
 |  | ||||||
| 						opt.agent = getAgentByUrl(url, false); |  | ||||||
| 
 |  | ||||||
| 						// Wrap original https?.request
 |  | ||||||
| 						const requestFunc = url.protocol === 'http:' ? http.request : https.request; |  | ||||||
| 						const clientRequest = requestFunc(url, opt, callback) as http.ClientRequest; |  | ||||||
| 
 |  | ||||||
| 						// HTTP-Signature
 |  | ||||||
| 						sign(clientRequest, { |  | ||||||
| 							authorizationHeaderName: 'Signature', |  | ||||||
| 							key: keypair.privateKey, |  | ||||||
| 							keyId: `${config.url}/users/${user.id}#main-key`, |  | ||||||
| 							headers: ['(request-target)', 'host', 'date', 'accept'] |  | ||||||
| 						}); |  | ||||||
| 
 |  | ||||||
| 						return clientRequest; |  | ||||||
| 					}; |  | ||||||
| 				}, |  | ||||||
| 			], |  | ||||||
| 		}, |  | ||||||
| 		retry: 0, |  | ||||||
| 	}); | 	}); | ||||||
| 
 | 
 | ||||||
| 	const res = await receiveResponce(req, 10 * 1024 * 1024); | 	const res = await getResponse({ | ||||||
|  | 		url, | ||||||
|  | 		method: req.request.method, | ||||||
|  | 		headers: req.request.headers | ||||||
|  | 	}); | ||||||
| 
 | 
 | ||||||
| 	return res.body; | 	return await res.json(); | ||||||
| } |  | ||||||
| 
 |  | ||||||
| /** |  | ||||||
|  * Receive response (with size limit) |  | ||||||
|  * @param req Request |  | ||||||
|  * @param maxSize size limit |  | ||||||
|  */ |  | ||||||
| export async function receiveResponce<T>(req: Got.CancelableRequest<Got.Response<T>>, maxSize: number) { |  | ||||||
| 	// 応答ヘッダでサイズチェック
 |  | ||||||
| 	req.on('response', (res: Got.Response) => { |  | ||||||
| 		const contentLength = res.headers['content-length']; |  | ||||||
| 		if (contentLength != null) { |  | ||||||
| 			const size = Number(contentLength); |  | ||||||
| 			if (size > maxSize) { |  | ||||||
| 				req.cancel(); |  | ||||||
| 			} |  | ||||||
| 		} |  | ||||||
| 	}); |  | ||||||
| 
 |  | ||||||
| 	// 受信中のデータでサイズチェック
 |  | ||||||
| 	req.on('downloadProgress', (progress: Got.Progress) => { |  | ||||||
| 		if (progress.transferred > maxSize) { |  | ||||||
| 			req.cancel(); |  | ||||||
| 		} |  | ||||||
| 	}); |  | ||||||
| 
 |  | ||||||
| 	// 応答取得 with ステータスコードエラーの整形
 |  | ||||||
| 	const res = await req.catch(e => { |  | ||||||
| 		if (e.name === 'HTTPError') { |  | ||||||
| 			const statusCode = (e as Got.HTTPError).response.statusCode; |  | ||||||
| 			const statusMessage = (e as Got.HTTPError).response.statusMessage; |  | ||||||
| 			throw { |  | ||||||
| 				name: `StatusError`, |  | ||||||
| 				statusCode, |  | ||||||
| 				message: `${statusCode} ${statusMessage}`, |  | ||||||
| 			}; |  | ||||||
| 		} else { |  | ||||||
| 			throw e; |  | ||||||
| 		} |  | ||||||
| 	}); |  | ||||||
| 
 |  | ||||||
| 	return res; |  | ||||||
| } | } | ||||||
|  |  | ||||||
|  | @ -13,6 +13,7 @@ import { downloadUrl } from '@/misc/download-url'; | ||||||
| import { detectType } from '@/misc/get-file-info'; | import { detectType } from '@/misc/get-file-info'; | ||||||
| import { convertToJpeg, convertToPngOrJpeg } from '@/services/drive/image-processor'; | import { convertToJpeg, convertToPngOrJpeg } from '@/services/drive/image-processor'; | ||||||
| import { GenerateVideoThumbnail } from '@/services/drive/generate-video-thumbnail'; | import { GenerateVideoThumbnail } from '@/services/drive/generate-video-thumbnail'; | ||||||
|  | import { StatusError } from '@/misc/fetch'; | ||||||
| 
 | 
 | ||||||
| //const _filename = fileURLToPath(import.meta.url);
 | //const _filename = fileURLToPath(import.meta.url);
 | ||||||
| const _filename = __filename; | const _filename = __filename; | ||||||
|  | @ -83,9 +84,9 @@ export default async function(ctx: Koa.Context) { | ||||||
| 				ctx.set('Content-Type', image.type); | 				ctx.set('Content-Type', image.type); | ||||||
| 				ctx.set('Cache-Control', 'max-age=31536000, immutable'); | 				ctx.set('Cache-Control', 'max-age=31536000, immutable'); | ||||||
| 			} catch (e) { | 			} catch (e) { | ||||||
| 				serverLogger.error(e.statusCode); | 				serverLogger.error(`${e}`); | ||||||
| 
 | 
 | ||||||
| 				if (typeof e.statusCode === 'number' && e.statusCode >= 400 && e.statusCode < 500) { | 				if (e instanceof StatusError && e.isClientError) { | ||||||
| 					ctx.status = e.statusCode; | 					ctx.status = e.statusCode; | ||||||
| 					ctx.set('Cache-Control', 'max-age=86400'); | 					ctx.set('Cache-Control', 'max-age=86400'); | ||||||
| 				} else { | 				} else { | ||||||
|  |  | ||||||
|  | @ -5,6 +5,7 @@ import { IImage, convertToPng, convertToJpeg } from '@/services/drive/image-proc | ||||||
| import { createTemp } from '@/misc/create-temp'; | import { createTemp } from '@/misc/create-temp'; | ||||||
| import { downloadUrl } from '@/misc/download-url'; | import { downloadUrl } from '@/misc/download-url'; | ||||||
| import { detectType } from '@/misc/get-file-info'; | import { detectType } from '@/misc/get-file-info'; | ||||||
|  | import { StatusError } from '@/misc/fetch'; | ||||||
| 
 | 
 | ||||||
| export async function proxyMedia(ctx: Koa.Context) { | export async function proxyMedia(ctx: Koa.Context) { | ||||||
| 	const url = 'url' in ctx.query ? ctx.query.url : 'https://' + ctx.params.url; | 	const url = 'url' in ctx.query ? ctx.query.url : 'https://' + ctx.params.url; | ||||||
|  | @ -37,9 +38,9 @@ export async function proxyMedia(ctx: Koa.Context) { | ||||||
| 		ctx.set('Cache-Control', 'max-age=31536000, immutable'); | 		ctx.set('Cache-Control', 'max-age=31536000, immutable'); | ||||||
| 		ctx.body = image.data; | 		ctx.body = image.data; | ||||||
| 	} catch (e) { | 	} catch (e) { | ||||||
| 		serverLogger.error(e); | 		serverLogger.error(`${e}`); | ||||||
| 
 | 
 | ||||||
| 		if (typeof e.statusCode === 'number' && e.statusCode >= 400 && e.statusCode < 500) { | 		if (e instanceof StatusError && e.isClientError) { | ||||||
| 			ctx.status = e.statusCode; | 			ctx.status = e.statusCode; | ||||||
| 		} else { | 		} else { | ||||||
| 			ctx.status = 500; | 			ctx.status = 500; | ||||||
|  |  | ||||||
							
								
								
									
										55
									
								
								test/ap-request.ts
									
										
									
									
									
										Normal file
									
								
							
							
						
						
									
										55
									
								
								test/ap-request.ts
									
										
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,55 @@ | ||||||
|  | import * as assert from 'assert'; | ||||||
|  | import { genRsaKeyPair } from '../src/misc/gen-key-pair'; | ||||||
|  | import { createSignedPost, createSignedGet } from '../src/remote/activitypub/ap-request'; | ||||||
|  | const httpSignature = require('http-signature'); | ||||||
|  | 
 | ||||||
|  | export const buildParsedSignature = (signingString: string, signature: string, algorithm: string) => { | ||||||
|  | 	return { | ||||||
|  | 		scheme: 'Signature', | ||||||
|  | 		params: { | ||||||
|  | 			keyId: 'KeyID',	// dummy, not used for verify
 | ||||||
|  | 			algorithm: algorithm, | ||||||
|  | 			headers: [ '(request-target)', 'date', 'host', 'digest' ],	// dummy, not used for verify
 | ||||||
|  | 			signature: signature, | ||||||
|  | 		}, | ||||||
|  | 		signingString: signingString, | ||||||
|  | 		algorithm: algorithm?.toUpperCase(), | ||||||
|  | 		keyId: 'KeyID',	// dummy, not used for verify
 | ||||||
|  | 	}; | ||||||
|  | }; | ||||||
|  | 
 | ||||||
|  | describe('ap-request', () => { | ||||||
|  | 	it('createSignedPost with verify', async () => { | ||||||
|  | 		const keypair = await genRsaKeyPair(); | ||||||
|  | 		const key = { keyId: 'x', 'privateKeyPem': keypair.privateKey }; | ||||||
|  | 		const url = 'https://example.com/inbox'; | ||||||
|  | 		const activity = { a: 1 }; | ||||||
|  | 		const body = JSON.stringify(activity); | ||||||
|  | 		const headers = { | ||||||
|  | 			'User-Agent': 'UA' | ||||||
|  | 		}; | ||||||
|  | 
 | ||||||
|  | 		const req = createSignedPost({ key, url, body, additionalHeaders: headers }); | ||||||
|  | 
 | ||||||
|  | 		const parsed = buildParsedSignature(req.signingString, req.signature, 'rsa-sha256'); | ||||||
|  | 
 | ||||||
|  | 		const result = httpSignature.verifySignature(parsed, keypair.publicKey); | ||||||
|  | 		assert.deepStrictEqual(result, true); | ||||||
|  | 	}); | ||||||
|  | 
 | ||||||
|  | 	it('createSignedGet with verify', async () => { | ||||||
|  | 		const keypair = await genRsaKeyPair(); | ||||||
|  | 		const key = { keyId: 'x', 'privateKeyPem': keypair.privateKey }; | ||||||
|  | 		const url = 'https://example.com/outbox'; | ||||||
|  | 		const headers = { | ||||||
|  | 			'User-Agent': 'UA' | ||||||
|  | 		}; | ||||||
|  | 
 | ||||||
|  | 		const req = createSignedGet({ key, url, additionalHeaders: headers }); | ||||||
|  | 
 | ||||||
|  | 		const parsed = buildParsedSignature(req.signingString, req.signature, 'rsa-sha256'); | ||||||
|  | 
 | ||||||
|  | 		const result = httpSignature.verifySignature(parsed, keypair.publicKey); | ||||||
|  | 		assert.deepStrictEqual(result, true); | ||||||
|  | 	}); | ||||||
|  | }); | ||||||
							
								
								
									
										15
									
								
								test/docker-compose.yml
									
										
									
									
									
										Normal file
									
								
							
							
						
						
									
										15
									
								
								test/docker-compose.yml
									
										
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,15 @@ | ||||||
|  | version: "3" | ||||||
|  | 
 | ||||||
|  | services: | ||||||
|  |   redistest: | ||||||
|  |     image: redis:4.0-alpine | ||||||
|  |     ports: | ||||||
|  |       - "127.0.0.1:56312:6379" | ||||||
|  | 
 | ||||||
|  |   dbtest: | ||||||
|  |     image: postgres:12.2-alpine | ||||||
|  |     ports: | ||||||
|  |       - "127.0.0.1:54312:5432" | ||||||
|  |     environment: | ||||||
|  |       POSTGRES_DB: "test-misskey" | ||||||
|  |       POSTGRES_HOST_AUTH_METHOD: trust | ||||||
							
								
								
									
										12
									
								
								test/test.yml
									
										
									
									
									
										Normal file
									
								
							
							
						
						
									
										12
									
								
								test/test.yml
									
										
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,12 @@ | ||||||
|  | url: 'http://misskey.local' | ||||||
|  | port: 61812 | ||||||
|  | db: | ||||||
|  |   host: localhost | ||||||
|  |   port: 54312 | ||||||
|  |   db: test-misskey | ||||||
|  |   user: postgres | ||||||
|  |   pass: '' | ||||||
|  | redis: | ||||||
|  |   host: localhost | ||||||
|  |   port: 56312 | ||||||
|  | id: aid | ||||||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue