diff --git a/src/index.ts b/src/index.ts index 27060006..de46fe38 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,13 +3,12 @@ * https://github.com/misskey-dev/summaly */ -import got from 'got'; import type { Agents as GotAgents } from 'got'; import type { FastifyInstance } from 'fastify'; import { SummalyResult as _SummalyResult } from '@/summary.js'; import { SummalyPlugin as _SummalyPlugin } from '@/iplugin.js'; import { general, type GeneralScrapingOptions } from '@/general.js'; -import { DEFAULT_BOT_UA, DEFAULT_OPERATION_TIMEOUT, DEFAULT_RESPONSE_TIMEOUT, agent, setAgent } from '@/utils/got.js'; +import { DEFAULT_BOT_UA, getResponse, setAgent } from '@/utils/got.js'; import { plugins as builtinPlugins } from '@/plugins/index.js'; export type SummalyResult = _SummalyResult; @@ -91,32 +90,22 @@ export const summaly = async (url: string, options?: SummalyOptions): Promise url)にすればいいけど、jestにtrace-redirectを食わせるのが面倒なのでtry-catch + // 各種チェックを通すため、getResponse経由でリダイレクトを解決する try { - const timeout = opts.responseTimeout ?? DEFAULT_RESPONSE_TIMEOUT; - const operationTimeout = opts.operationTimeout ?? DEFAULT_OPERATION_TIMEOUT; - actualUrl = await got - .head(url, { - headers: { - accept: 'text/html,application/xhtml+xml', - 'user-agent': opts.userAgent ?? DEFAULT_BOT_UA, - 'accept-language': opts.lang ?? undefined, - }, - timeout: { - lookup: timeout, - connect: timeout, - secureConnect: timeout, - socket: timeout, // read timeout - response: timeout, - send: timeout, - request: operationTimeout, // whole operation timeout - }, - agent, - http2: false, - retry: { - limit: 0, - }, - }) - .then(res => res.url); + const res = await getResponse({ + url, + method: 'HEAD', + headers: { + accept: 'text/html,application/xhtml+xml', + 'user-agent': opts.userAgent ?? DEFAULT_BOT_UA, + 'accept-language': opts.lang ?? undefined, + }, + followRedirects: true, + responseTimeout: opts.responseTimeout, + operationTimeout: opts.operationTimeout, + contentLengthLimit: opts.contentLengthLimit, + }); + actualUrl = res.url; } catch { actualUrl = url; } diff --git a/src/utils/got.ts b/src/utils/got.ts index d7b89004..2eb00269 100644 --- a/src/utils/got.ts +++ b/src/utils/got.ts @@ -1,3 +1,4 @@ +import type { ClientRequest, IncomingMessage } from 'node:http'; import got from 'got'; import { HTTPError as GotHTTPError } from 'got'; import type { @@ -125,27 +126,19 @@ export async function getResponse(args: GotOptions) { limit: 0, }, signal: abort.signal, + hooks: { + // 応答ヘッダ受信時点のチェックでabortしても、gotはリダイレクト先へのリクエストを発行してしまうため、ここで止める + beforeRedirect: [() => { + abort.signal.throwIfAborted(); + }], + }, }); const res = await receiveResponse({ req, opts: args, abort }); - // SUMMALY_ALLOW_PRIVATE_IPはテスト用 - // TODO: Try moving this to receiveResponse- ATM `got` doesn't provide a means - // to check the IP/response header data while streaming the response... - const allowPrivateIp = process.env.SUMMALY_ALLOW_PRIVATE_IP === 'true' || Object.keys(agent).length > 0; - if (!allowPrivateIp && res.ip != null) { - let ip: IPv4 | IPv6; - try { - ip = ipaddr.parse(res.ip); - } catch { - throw new StatusError(`Invalid IP ${res.ip}`, 500, 'Invalid IP'); - } - if (ip.kind() === 'ipv6' && (ip as IPv6).isIPv4MappedAddress()) { - ip = (ip as IPv6).toIPv4Address(); - } - if (ip.range() !== 'unicast') { - throw new StatusError(`Private IP rejected ${res.ip}`, 400, 'Private IP Rejected'); - } + // 念のため最終応答に対しても同じチェックを行う(通常はreceiveResponse内で弾かれている) + if (!isPrivateIpAllowed() && res.ip != null) { + assertPublicIp(res.ip); } // Check html @@ -171,6 +164,28 @@ export async function getResponse(args: GotOptions) { return res; } +function isPrivateIpAllowed() { + // SUMMALY_ALLOW_PRIVATE_IPはテスト用 + // agentが指定されている場合、接続先はagent(プロキシ等)になり実際の宛先IPを検査できないため、 + // 宛先の制限はagent側に任せる + return process.env.SUMMALY_ALLOW_PRIVATE_IP === 'true' || Object.keys(agent).length > 0; +} + +function assertPublicIp(rawIp: string) { + let ip: IPv4 | IPv6; + try { + ip = ipaddr.parse(rawIp); + } catch { + throw new StatusError(`Invalid IP ${rawIp}`, 500, 'Invalid IP'); + } + if (ip.kind() === 'ipv6' && (ip as IPv6).isIPv4MappedAddress()) { + ip = (ip as IPv6).toIPv4Address(); + } + if (ip.range() !== 'unicast') { + throw new StatusError(`Private IP rejected ${rawIp}`, 400, 'Private IP Rejected'); + } +} + async function receiveResponse(args: { req: GotRequestPromise>, opts: GotOptions, @@ -179,6 +194,30 @@ async function receiveResponse(args: { const req = args.req; const maxSize = args.opts.contentLengthLimit ?? DEFAULT_MAX_RESPONSE_SIZE; + const allowPrivateIp = isPrivateIpAllowed(); + + req.on('request', (clientRequest: ClientRequest) => { + clientRequest.prependOnceListener('response', (res: IncomingMessage) => { + try { + const ip = res.socket.remoteAddress; + if (!allowPrivateIp && ip != null) { + assertPublicIp(ip); + } + + // リダイレクトやエラー応答はここでは判定しない(ステータスコードによるエラーを優先する) + const statusCode = res.statusCode ?? 0; + if (statusCode < 200 || statusCode >= 300) return; + + const contentType = res.headers['content-type']; + if (args.opts.typeFilter && !contentType?.match(args.opts.typeFilter)) { + throw new Error(`Rejected by type filter ${contentType}`); + } + } catch (e) { + args.abort.abort(e); + } + }); + }); + // 受信中のデータでサイズチェック req.on('downloadProgress', (progress: GotProgress) => { if (progress.transferred > maxSize && progress.percent !== 1) { @@ -189,8 +228,13 @@ async function receiveResponse(args: { // 応答取得 with ステータスコードエラーの整形 const res = await req.catch(e => { const abortReason = args.abort.signal.reason; - if (args.abort.signal.aborted && typeof abortReason === 'string' && abortReason.length > 0) { - throw new Error(abortReason); + if (args.abort.signal.aborted) { + if (abortReason instanceof Error) { + throw abortReason; + } + if (typeof abortReason === 'string' && abortReason.length > 0) { + throw new Error(abortReason); + } } if (e instanceof GotHTTPError) { diff --git a/test/index.test.ts b/test/index.test.ts index 409746a0..3c2b54cb 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -233,6 +233,24 @@ describe('local tests', () => { } }); + test('リダイレクト解決のためのHEADリクエストでもprivate ipが弾かれ、リダイレクト先へ到達しないこと', async () => { + await app?.close(); + + let redirectedHeadCount = 0; + app = fastify({ exposeHeadRoutes: false }); + app.head('/', (request, reply) => reply.redirect('/redirected', 302)); + app.head('/redirected', (request, reply) => { + redirectedHeadCount++; + return reply.status(200).send(); + }); + app.get('*', (request, reply) => reply.status(404).send()); + await app.listen({ port }); + + const summary = await summaly(host).catch((e: StatusError) => e); + expect(summary).toBeInstanceOf(StatusError); + expect(redirectedHeadCount).toBe(0); + }); + afterEach(() => { process.env.SUMMALY_ALLOW_PRIVATE_IP = 'true'; }); @@ -711,6 +729,61 @@ describe('local tests', () => { }); }); + describe('content-type filter', () => { + test('HTML以外のコンテンツは本文を受信せずに即座にエラーになること', async () => { + app = fastify(); + app.get('/', (request, reply) => { + let closed = false; + reply.raw.on('close', () => { closed = true; }); + reply.header('content-type', 'application/pdf'); + // 終わらないストリームを返し、本文の受信を待たずに打ち切られることを確認する + return reply.send(Readable.from((async function* () { + yield Buffer.from('%PDF-1.7\n'); + while (!closed) { + await new Promise(resolve => setTimeout(resolve, 10)); + yield Buffer.alloc(1024, 'a'); + } + })())); + }); + await app.listen({ port }); + + await expect(summaly(host)).rejects.toThrow(/Rejected by type filter application\/pdf/); + }, 3000); + + test('content-typeが返されない場合はエラーになること', async () => { + app = fastify(); + app.get('/', (request, reply) => { + reply.removeHeader('content-type'); + return reply.send(Readable.from((async function* () { + yield Buffer.from('hello'); + })())); + }); + await app.listen({ port }); + + await expect(summaly(host)).rejects.toThrow(/Rejected by type filter/); + }); + + test('リダイレクト応答のcontent-typeでは弾かれないこと', async () => { + const content = fs.readFileSync(_dirname + '/htmls/og-title.html'); + + // HEADでの事前のリダイレクト解決をさせず、GET時にリダイレクトを辿らせる + app = fastify({ exposeHeadRoutes: false }); + app.get('/', (request, reply) => { + reply.header('content-type', 'application/octet-stream'); + return reply.redirect('/html', 302); + }); + app.get('/html', (request, reply) => { + reply.header('content-length', content.byteLength); + reply.header('content-type', 'text/html'); + return reply.send(content); + }); + await app.listen({ port }); + + const summary = await summaly(host); + expect(summary.title).toBe('Strawberry Pasta'); + }); + }); + describe('content-length limit', () => { test('content-lengthの上限以内であればエラーが起こらないこと', async () => { const content = fs.readFileSync(_dirname + '/htmls/basic.html');