From c2d1fd3faba7b5f75f48b04cb8cf81622ad8930a Mon Sep 17 00:00:00 2001 From: leolambo Date: Tue, 28 Apr 2026 11:11:55 -0400 Subject: [PATCH 01/19] Add streamJsonArray helper for route-layer piping Unify ExternalApiStream.onStream and Storage.apiStream into a single route-layer helper. CSPs can now return Readable streams while routes handle HTTP framing, error injection, and client disconnect cleanup. Supports JSON-array (default) and JSONL output. Pre-data errors reject so the caller can send a proper 5xx; mid-stream errors append an inline error marker. Detects mongo cursor-style streams via .close(). --- packages/bitcore-node/src/routes/apiUtils.ts | 112 ++++++++++++++++++- 1 file changed, 111 insertions(+), 1 deletion(-) diff --git a/packages/bitcore-node/src/routes/apiUtils.ts b/packages/bitcore-node/src/routes/apiUtils.ts index 5197557e616..abf9814d64a 100644 --- a/packages/bitcore-node/src/routes/apiUtils.ts +++ b/packages/bitcore-node/src/routes/apiUtils.ts @@ -1,4 +1,5 @@ -import { Response } from 'express'; +import { Readable } from 'stream'; +import { Request, Response } from 'express'; import { AdapterError, AdapterErrorCode, AllProvidersUnavailableError } from '../providers/chain-state/external/adapters/errors'; export function respondWithError(res: Response, err: any) { @@ -10,3 +11,112 @@ export function respondWithError(res: Response, err: any) { } return res.status(500).send(err.message || err); } + +export interface StreamJsonArrayOpts { + jsonl?: boolean; +} + +export interface StreamJsonArrayResult { + success: boolean; + error?: any; +} + +/** + * Pipe a Readable stream to an Express response as a JSON array (default) or JSONL. + * + * - Pre-data errors reject so the route can send a proper status code + * - Mid-stream errors append an inline error marker and end the response + * - Client/response disconnects destroy the stream (and call .close() if present, e.g. mongo cursor) + */ +export function streamJsonArray( + stream: Readable & { close?: () => void }, + req: Request, + res: Response, + opts: StreamJsonArrayOpts = {} +): Promise { + return new Promise((resolve, reject) => { + let closed = false; + let isFirst = true; + + const cleanup = () => { + closed = true; + if (typeof stream.close === 'function') { + try { stream.close(); } catch { /* noop */ } + } else if (typeof stream.destroy === 'function') { + try { stream.destroy(); } catch { /* noop */ } + } + }; + + req.on('close', () => { closed = true; if (typeof stream.close === 'function') stream.close(); }); + res.type('json'); + res.on('close', () => { closed = true; if (typeof stream.close === 'function') stream.close(); }); + + stream.on('error', (err: any) => { + if (closed) return; + if (err?.isAxiosError) { + err.log = { + url: err?.config?.url, + statusCode: err?.response?.status, + statusMsg: err?.response?.statusText, + data: err?.response?.data, + }; + } + if (err?.log?.data?.message?.includes('not supported')) { + closed = true; + res.write('[]'); + res.end(); + return resolve({ success: false, error: err }); + } + if (!isFirst) { + // Headers already sent — emit inline error marker, end response, log upstream + closed = true; + const errMsg = '{"error": "An error occurred during data stream"}'; + if (opts.jsonl) { + res.write(`${errMsg}`); + } else { + res.write(`,\n${errMsg}\n]`); + } + res.end(); + cleanup(); + return resolve({ success: false, error: err }); + } + // Pre-data — caller can send proper 5xx status + return reject(err); + }); + + stream.on('data', (data: any) => { + if (closed) { + cleanup(); + return; + } + if (!opts.jsonl) { + if (isFirst) { + res.write('[\n'); + } else { + res.write(',\n'); + } + } + if (isFirst) { + isFirst = false; + } + if (typeof data !== 'string' && !Buffer.isBuffer(data)) { + data = JSON.stringify(data); + } + res.write(data); + }); + + stream.on('end', () => { + if (closed) return; + closed = true; + if (!opts.jsonl) { + if (isFirst) { + res.write('[]'); + } else { + res.write('\n]'); + } + } + res.end(); + resolve({ success: true }); + }); + }); +} From 128c3a1c795a9624edee1d921d7ae86949afddd9 Mon Sep 17 00:00:00 2001 From: leolambo Date: Tue, 28 Apr 2026 14:32:02 -0400 Subject: [PATCH 02/19] Decouple chain-state providers from HTTP req/res CSP stream methods now return Readable; routes own the piping via streamJsonArray. The HTTP framing, mid-stream error injection, and client-disconnect cleanup that were duplicated across ExternalApiStream and Storage now live in one place at the route layer. Touches all six stream entry points (address, tx, block, wallet addresses/transactions/utxos) plus the gnosis multisig route, across internal, base EVM, Moralis, MultiProvider, SVM, and Ripple. Storage's apiStreamingFind drops req/res and returns the cursor stream directly. Stream params types lose req/res; integration tests pipe the returned stream through streamJsonArray. Auto-detects a jsonl flag on the returned stream so chain-specific formatting stays in the CSP without leaking through routes. --- .../src/modules/moralis/api/csp.ts | 14 +- .../src/modules/multiProvider/api/csp.ts | 8 +- .../src/modules/ripple/api/csp.ts | 5 +- .../src/providers/chain-state/evm/api/csp.ts | 103 ++++------ .../providers/chain-state/evm/api/gnosis.ts | 16 +- .../providers/chain-state/evm/api/routes.ts | 14 +- .../chain-state/internal/internal.ts | 29 +-- .../src/providers/chain-state/svm/api/csp.ts | 186 +++++++----------- .../bitcore-node/src/routes/api/address.ts | 10 +- packages/bitcore-node/src/routes/api/block.ts | 14 +- packages/bitcore-node/src/routes/api/tx.ts | 11 +- .../bitcore-node/src/routes/api/wallet.ts | 36 ++-- packages/bitcore-node/src/services/storage.ts | 94 +-------- .../types/namespaces/ChainStateProvider.ts | 12 -- .../test/integration/ethereum/csp.test.ts | 16 +- .../test/integration/matic/csp.test.ts | 13 +- .../test/integration/solana/csp.test.ts | 6 +- 17 files changed, 220 insertions(+), 367 deletions(-) diff --git a/packages/bitcore-node/src/modules/moralis/api/csp.ts b/packages/bitcore-node/src/modules/moralis/api/csp.ts index b949d9030ad..b39e71021f1 100644 --- a/packages/bitcore-node/src/modules/moralis/api/csp.ts +++ b/packages/bitcore-node/src/modules/moralis/api/csp.ts @@ -108,7 +108,7 @@ export class MoralisStateProvider extends BaseEVMStateProvider { // @override async streamBlocks(params: StreamBlocksParams) { - const { chain, network, req, res } = params; + const { chain, network } = params; const { web3 } = await this.getWeb3(network); const chainId = await this.getChainId({ network }); const blockRange = await this.getBlocksRange({ ...params, chainId }); @@ -146,8 +146,7 @@ export class MoralisStateProvider extends BaseEVMStateProvider { } }); - return ExternalApiStream.onStream(stream, req!, res!); - + return stream; } // @override @@ -165,10 +164,10 @@ export class MoralisStateProvider extends BaseEVMStateProvider { // @override async _buildAddressTransactionsStream(params: StreamAddressUtxosParams) { - const { req, res, args, network, address } = params; + const { args, network, address } = params; const chainId = await this.getChainId({ network }); - const txStream = await this._streamAddressTransactionsFromMoralis({ + return this._streamAddressTransactionsFromMoralis({ chainId, chain: this.chain, network, @@ -178,11 +177,6 @@ export class MoralisStateProvider extends BaseEVMStateProvider { ...args } }); - // TODO unify `ExternalApiStream.onStream` and `Storage.apiStream` which are effectively doing the same thing - const result = await ExternalApiStream.onStream(txStream, req!, res!); - if (!result?.success) { - logger.error('Error mid-stream (streamAddressTransactions): %o', result.error?.log || result.error); - } } // @override diff --git a/packages/bitcore-node/src/modules/multiProvider/api/csp.ts b/packages/bitcore-node/src/modules/multiProvider/api/csp.ts index 598aa3ba8a3..19d649acd47 100644 --- a/packages/bitcore-node/src/modules/multiProvider/api/csp.ts +++ b/packages/bitcore-node/src/modules/multiProvider/api/csp.ts @@ -186,7 +186,7 @@ export class MultiProviderEVMStateProvider extends BaseEVMStateProvider { // @override — sequential failover with preflight check. // Buffers first item before piping to response; failover only before response bytes are written. async _buildAddressTransactionsStream(params: StreamAddressUtxosParams) { - const { req, res, args, network, address } = params; + const { args, network, address } = params; const chainId = await this.getChainId({ network }); const providers = this.getProvidersForNetwork(network); const PREFLIGHT_TIMEOUT_MS = 5000; @@ -261,11 +261,7 @@ export class MultiProviderEVMStateProvider extends BaseEVMStateProvider { txStream.resume(); } - const result = await ExternalApiStream.onStream(outputStream, req!, res!); - if (!result?.success) { - logger.error('Error mid-stream (streamAddressTransactions): %o', result.error?.log || result.error); - } - return; // Stream handled + return outputStream; } catch (error) { if (error instanceof AdapterError && (error as AdapterError).code === AdapterErrorCode.INVALID_REQUEST) throw error; // 400 — no failover provider.health.recordFailure(error as Error); diff --git a/packages/bitcore-node/src/modules/ripple/api/csp.ts b/packages/bitcore-node/src/modules/ripple/api/csp.ts index 65e25bafc80..e795c08cc1d 100644 --- a/packages/bitcore-node/src/modules/ripple/api/csp.ts +++ b/packages/bitcore-node/src/modules/ripple/api/csp.ts @@ -9,7 +9,6 @@ import { CacheStorage } from '../../../models/cache'; import { ICoin } from '../../../models/coin'; import { WalletAddressStorage } from '../../../models/walletAddress'; import { InternalStateProvider } from '../../../providers/chain-state/internal/internal'; -import { Storage } from '../../../services/storage'; import { IBlock } from '../../../types/Block'; import { ChainNetwork } from '../../../types/ChainNetwork'; import { @@ -305,7 +304,7 @@ export class RippleStateProvider extends InternalStateProvider implements IChain const transformed = txs.map(tx => this.transformAccountTx(tx, params.network)); this.streamTxs(transformed, readable); readable.push(null); - Storage.stream(readable, params.req!, params.res!); + return readable; } async streamTransactions(params: StreamTransactionsParams) { @@ -316,7 +315,7 @@ export class RippleStateProvider extends InternalStateProvider implements IChain const txs = ledger.transactions || []; this.streamTxs(txs, readable); readable.push(null); - Storage.stream(readable, params.req, params.res); + return readable; } async getTransaction(params: StreamTransactionParams) { diff --git a/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts b/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts index 8fdbecbd52b..fdd24211e57 100644 --- a/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts +++ b/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts @@ -1,3 +1,4 @@ +import { Readable } from 'stream'; import { CryptoRpc } from '@bitpay-labs/crypto-rpc'; import { Utils, Web3, type Web3Types } from '@bitpay-labs/crypto-wallet-core'; import { @@ -18,7 +19,7 @@ import { SpentHeightIndicators } from '../../../../types/Coin'; import { normalizeChainNetwork, partition, range } from '../../../../utils'; import { StatsUtil } from '../../../../utils/stats'; import { TransformWithEventPipe } from '../../../../utils/streamWithEventPipe'; -import { ExternalApiStream } from '../../external/streams/apiStream'; +import { AdapterError, AdapterErrorCode } from '../../external/adapters/errors'; import { AavePoolAbi } from '../abi/aavePool'; import { AavePoolAbiV2 } from '../abi/aavePoolV2'; import { ERC20Abi } from '../abi/erc20'; @@ -531,18 +532,11 @@ export class BaseEVMStateProvider extends InternalStateProvider implements IChai } async streamAddressTransactions(params: StreamAddressUtxosParams) { - return new Promise(async (resolve, reject) => { - try { - await this._buildAddressTransactionsStream(params); - return resolve(); - } catch (err) { - return reject(err); - } - }); + return this._buildAddressTransactionsStream(params); } async _buildAddressTransactionsStream(params: StreamAddressUtxosParams) { - const { req, res, args, chain, network, address } = params; + const { args, chain, network, address } = params; const { limit, /* since,*/ tokenAddress } = args; if (!args.tokenAddress) { @@ -557,23 +551,17 @@ export class BaseEVMStateProvider extends InternalStateProvider implements IChai // NOTE: commented out since and paging for now b/c they were causing extra long query times on insight. // The case where an address has >1000 txns is an edge case ATM and can be addressed later - Storage.apiStreamingFind(EVMTransactionStorage, query, { limit /* since, paging: '_id'*/ }, req!, res!); - } else { - try { - const tokenTransfers = await this.getErc20Transfers(network, address, tokenAddress, args); - res!.json(tokenTransfers); - } catch (err: any) { - logger.error('Error streaming address transactions: %o', err.stack || err.message || err); - throw err; - } + return Storage.apiStreamingFind(EVMTransactionStorage, query, { limit /* since, paging: '_id'*/ }); } + const tokenTransfers = await this.getErc20Transfers(network, address, tokenAddress, args); + return Readable.from(tokenTransfers, { objectMode: true }); } @historical @internal async streamTransactions(params: StreamTransactionsParams) { - const { chain, network, req, res, args } = params; + const { chain, network, args } = params; const { blockHash, blockHeight } = args; if (!chain || !network) { throw new Error('Missing chain or network'); @@ -590,7 +578,7 @@ export class BaseEVMStateProvider extends InternalStateProvider implements IChai } const tip = await this.getLocalTip(params); const tipHeight = tip ? tip.height : 0; - return Storage.apiStreamingFind(EVMTransactionStorage, query, args, req, res, t => { + return Storage.apiStreamingFind(EVMTransactionStorage, query, args, t => { let confirmations = 0; if (t.blockHeight !== undefined && t.blockHeight >= 0) { confirmations = tipHeight - t.blockHeight + 1; @@ -677,47 +665,37 @@ export class BaseEVMStateProvider extends InternalStateProvider implements IChai } async streamWalletTransactions(params: StreamWalletTransactionsParams) { - return new Promise(async (resolve, reject) => { - const { network, wallet, req, res, args } = params; - const { web3 } = await this.getWeb3(network); - args.tokenAddress = args.tokenAddress ? web3.utils.toChecksumAddress(args.tokenAddress) : undefined; + const { network, wallet, args } = params; + const { web3 } = await this.getWeb3(network); + args.tokenAddress = args.tokenAddress ? web3.utils.toChecksumAddress(args.tokenAddress) : undefined; - let transactionStream = new TransformWithEventPipe({ objectMode: true, passThrough: true }); - const walletAddresses = (await this.getWalletAddresses(wallet._id!)).map(waddres => waddres.address); - if (walletAddresses.length === 0) { - res.status(400).send('No addresses found for wallet'); - return resolve(); - } - const ethTransactionTransform = new EVMListTransactionsStream(walletAddresses, args.tokenAddress); - const populateReceipt = new PopulateReceiptTransform(this); - const populateEffects = new PopulateEffectsForAddressTransform(this, walletAddresses); - - const streamParams: BuildWalletTxsStreamParams = { - transactionStream, - populateEffects, - walletAddresses - }; - transactionStream = await this._buildWalletTransactionsStream(params, streamParams); + let transactionStream = new TransformWithEventPipe({ objectMode: true, passThrough: true }); + const walletAddresses = (await this.getWalletAddresses(wallet._id!)).map(waddres => waddres.address); + if (walletAddresses.length === 0) { + throw new AdapterError('walletAddresses', AdapterErrorCode.INVALID_REQUEST, 'No addresses found for wallet'); + } + const ethTransactionTransform = new EVMListTransactionsStream(walletAddresses, args.tokenAddress); + const populateReceipt = new PopulateReceiptTransform(this); + const populateEffects = new PopulateEffectsForAddressTransform(this, walletAddresses); + + const streamParams: BuildWalletTxsStreamParams = { + transactionStream, + populateEffects, + walletAddresses + }; + transactionStream = await this._buildWalletTransactionsStream(params, streamParams); - if (!args.tokenAddress && wallet._id) { - const internalTxTransform = new InternalTxRelatedFilterTransform(web3, wallet._id); - transactionStream = transactionStream.eventPipe(internalTxTransform); - } + if (!args.tokenAddress && wallet._id) { + const internalTxTransform = new InternalTxRelatedFilterTransform(web3, wallet._id); + transactionStream = transactionStream.eventPipe(internalTxTransform); + } - transactionStream = transactionStream - .eventPipe(populateReceipt) - .eventPipe(ethTransactionTransform); + transactionStream = transactionStream + .eventPipe(populateReceipt) + .eventPipe(ethTransactionTransform); - try { - const result = await ExternalApiStream.onStream(transactionStream, req!, res!, { jsonl: true }); - if (!result?.success) { - logger.error('Error mid-stream (streamWalletTransactions): %o', result.error?.log || result.error); - } - return resolve(); - } catch (err) { - return reject(err); - } - }); + (transactionStream as any).jsonl = true; + return transactionStream; } async _buildWalletTransactionsStream(params: StreamWalletTransactionsParams, streamParams: BuildWalletTxsStreamParams) { @@ -731,7 +709,8 @@ export class BaseEVMStateProvider extends InternalStateProvider implements IChai .sort({ blockTimeNormalized: 1 }) .addCursorFlag('noCursorTimeout', true); - // Add cleanup handlers when client disconnects + // Cursor cleanup is triggered when the consumer destroys the returned stream + // (streamJsonArray destroys on req/res close). let cursorClosed = false; const cleanupCursor = () => { if (!cursorClosed) { @@ -744,12 +723,10 @@ export class BaseEVMStateProvider extends InternalStateProvider implements IChai } }; - const { req, res } = params; - req.on('close', cleanupCursor); - res.on('close', cleanupCursor); - // Pipe cursor to transform stream transactionStream = cursor.pipe(new TransformWithEventPipe({ objectMode: true, passThrough: true })); + transactionStream.on('close', cleanupCursor); + transactionStream.on('end', cleanupCursor); transactionStream = transactionStream.eventPipe(populateEffects); // For old db entries diff --git a/packages/bitcore-node/src/providers/chain-state/evm/api/gnosis.ts b/packages/bitcore-node/src/providers/chain-state/evm/api/gnosis.ts index 42c06c54cea..38dace505a1 100644 --- a/packages/bitcore-node/src/providers/chain-state/evm/api/gnosis.ts +++ b/packages/bitcore-node/src/providers/chain-state/evm/api/gnosis.ts @@ -154,7 +154,7 @@ export class GnosisApi { } async streamGnosisWalletTransactions(params: { multisigContractAddress: string } & StreamWalletTransactionsParams) { - const { chain, network, multisigContractAddress, res, args } = params; + const { chain, network, multisigContractAddress, args } = params; const transactionQuery = getCSP(chain, network).getWalletTransactionQuery(params); delete transactionQuery.wallets; delete transactionQuery['wallets.0']; @@ -201,7 +201,6 @@ export class GnosisApi { .sort({ blockTimeNormalized: 1 }) .addCursorFlag('noCursorTimeout', true); - // Add cleanup handlers when client disconnects let cursorClosed = false; const cleanupCursor = () => { if (!cursorClosed) { @@ -214,10 +213,6 @@ export class GnosisApi { } }; - const { req } = params; - req.on('close', cleanupCursor); - res.on('close', cleanupCursor); - transactionStream = cursor.pipe(populateEffects); // For old db entries if (multisigContractAddress) { @@ -225,10 +220,13 @@ export class GnosisApi { transactionStream = transactionStream.pipe(multisigTransform); } - transactionStream + const finalStream: any = transactionStream .pipe(populateReceipt) - .pipe(ethTransactionTransform) - .pipe(res); + .pipe(ethTransactionTransform); + finalStream.jsonl = true; + finalStream.on('close', cleanupCursor); + finalStream.on('end', cleanupCursor); + return finalStream; } } diff --git a/packages/bitcore-node/src/providers/chain-state/evm/api/routes.ts b/packages/bitcore-node/src/providers/chain-state/evm/api/routes.ts index 8c4e3b15459..1a8fcdb1e89 100644 --- a/packages/bitcore-node/src/providers/chain-state/evm/api/routes.ts +++ b/packages/bitcore-node/src/providers/chain-state/evm/api/routes.ts @@ -4,6 +4,7 @@ import { Router } from 'express'; import config from '../../../../config'; import logger from '../../../../logger'; import { WebhookStorage } from '../../../../models/webhook'; +import { respondWithError, streamJsonArray } from '../../../../routes/apiUtils'; import { Config } from '../../../../services/config'; import { IEVMNetworkConfig } from '../../../../types/Config'; import { castToBool } from '../../../../utils'; @@ -235,22 +236,25 @@ export class EVMRouter { }); }; - private streamGnosisWalletTransactions(router: Router) { + private streamGnosisWalletTransactions(router: Router) { router.get(`/api/${this.chain}/:network/ethmultisig/transactions/:multisigContractAddress`, async (req, res) => { const { network, multisigContractAddress } = req.params; try { - return await Gnosis.streamGnosisWalletTransactions({ + const stream = await Gnosis.streamGnosisWalletTransactions({ chain: this.chain, network, multisigContractAddress, wallet: {} as any, - req, - res, args: req.query }); + const result = await streamJsonArray(stream, req, res); + if (!result.success) { + logger.error('Error mid-stream (streamGnosisWalletTransactions): %o', result.error?.log || result.error); + } + return; } catch (err: any) { logger.error('Multisig Transactions Error::%o', err.stack || err.message || err); - return res.status(500).send(err.message || err); + return respondWithError(res, err); } }); }; diff --git a/packages/bitcore-node/src/providers/chain-state/internal/internal.ts b/packages/bitcore-node/src/providers/chain-state/internal/internal.ts index ba268033c2e..b11975f5602 100644 --- a/packages/bitcore-node/src/providers/chain-state/internal/internal.ts +++ b/packages/bitcore-node/src/providers/chain-state/internal/internal.ts @@ -81,10 +81,10 @@ export class InternalStateProvider implements IChainStateService { } async streamAddressTransactions(params: StreamAddressUtxosParams) { - const { req, res, args } = params; + const { args } = params; const { limit, since } = args; const query = this.getAddressQuery(params); - Storage.apiStreamingFind(CoinStorage, query, { limit, since, paging: '_id' }, req!, res!); + return Storage.apiStreamingFind(CoinStorage, query, { limit, since, paging: '_id' }); } async getBalanceForAddress(params: GetBalanceForAddressParams): Promise { @@ -100,10 +100,9 @@ export class InternalStateProvider implements IChainStateService { return balance; } - streamBlocks(params: StreamBlocksParams) { - const { req, res } = params; + async streamBlocks(params: StreamBlocksParams) { const { query, options } = this.getBlocksQuery(params); - Storage.apiStreamingFind(BitcoinBlockStorage, query, options, req, res); + return Storage.apiStreamingFind(BitcoinBlockStorage, query, options); } async getBlocks(params: GetBlockParams): Promise> { @@ -202,7 +201,7 @@ export class InternalStateProvider implements IChainStateService { } async streamTransactions(params: StreamTransactionsParams) { - const { chain, network, req, res, args } = params; + const { chain, network, args } = params; const { blockHash, blockHeight } = args; if (!chain || !network) { throw new Error('Missing chain or network'); @@ -219,7 +218,7 @@ export class InternalStateProvider implements IChainStateService { } const tip = await this.getLocalTip(params); const tipHeight = tip ? tip.height : 0; - return Storage.apiStreamingFind(TransactionStorage, query, args, req, res, t => { + return Storage.apiStreamingFind(TransactionStorage, query, args, t => { let confirmations = 0; if (t.blockHeight !== undefined && t.blockHeight >= 0) { confirmations = tipHeight - t.blockHeight + 1; @@ -305,9 +304,9 @@ export class InternalStateProvider implements IChainStateService { } streamWalletAddresses(params: StreamWalletAddressesParams) { - const { chain, network, walletId, req, res } = params; + const { chain, network, walletId } = params; const query = { chain, network, wallet: walletId }; - Storage.apiStreamingFind(WalletAddressStorage, query, {}, req, res); + return Storage.apiStreamingFind(WalletAddressStorage, query, {}); } async walletCheck(params: WalletCheckParams) { @@ -384,7 +383,7 @@ export class InternalStateProvider implements IChainStateService { } async streamWalletTransactions(params: StreamWalletTransactionsParams) { - const { chain, network, wallet, res, args } = params; + const { chain, network, wallet, args } = params; const query: any = { chain, network, @@ -430,8 +429,10 @@ export class InternalStateProvider implements IChainStateService { .find(query) .sort({ blockTimeNormalized: 1 }) .addCursorFlag('noCursorTimeout', true); - const listTransactionsStream = new this.WalletStreamTransform(wallet); - transactionStream.pipe(listTransactionsStream).pipe(res); + const listTransactionsStream: any = transactionStream.pipe(new this.WalletStreamTransform(wallet)); + listTransactionsStream.jsonl = true; + listTransactionsStream.on('close', () => { try { transactionStream.close(); } catch { /* noop */ } }); + return listTransactionsStream; } async getWalletBalance(params: GetWalletBalanceParams): Promise { @@ -457,7 +458,7 @@ export class InternalStateProvider implements IChainStateService { } async streamWalletUtxos(params: StreamWalletUtxosParams) { - const { wallet, limit, args = {}, req, res } = params; + const { wallet, limit, args = {} } = params; const query: any = { wallets: wallet._id, 'wallets.0': { $exists: true }, @@ -484,7 +485,7 @@ export class InternalStateProvider implements IChainStateService { return CoinStorage._apiTransform(c) as string; }; - Storage.apiStreamingFind(CoinStorage, query, { limit }, req, res, utxoTransform); + return Storage.apiStreamingFind(CoinStorage, query, { limit }, utxoTransform); } async getFee(params: GetEstimateSmartFeeParams) { diff --git a/packages/bitcore-node/src/providers/chain-state/svm/api/csp.ts b/packages/bitcore-node/src/providers/chain-state/svm/api/csp.ts index 2cff5652f09..c62def4b05a 100644 --- a/packages/bitcore-node/src/providers/chain-state/svm/api/csp.ts +++ b/packages/bitcore-node/src/providers/chain-state/svm/api/csp.ts @@ -183,89 +183,58 @@ export class BaseSVMStateProvider extends InternalStateProvider implements IChai } async streamTransactions(params: StreamTransactionsParams): Promise { - return new Promise(async (resolve, reject) => { - try { - const { chain, network, req, res, args } = params; - let { blockHeight } = args; - const { limit = 50 } = args; + const { chain, network, args } = params; + let { blockHeight } = args; + const { limit = 50 } = args; - if (!chain || !network) { - throw new Error('Missing chain or network'); - } - if (blockHeight !== undefined) { - blockHeight = Number(blockHeight); - } else { - throw new Error('Missing required block height / slot.'); - } + if (!chain || !network) { + throw new Error('Missing chain or network'); + } + if (blockHeight !== undefined) { + blockHeight = Number(blockHeight); + } else { + throw new Error('Missing required block height / slot.'); + } - const { rpc } = await this.getRpc(network); - const block: any = await rpc.getBlock({ height: blockHeight, transactionDetails: 'signatures' }); - if (!block) { - throw new Error('Block not found: ' + blockHeight); - } - const stream = new TransformWithEventPipe({ - objectMode: true, - passThrough: true - }); - let count = 0; - for (const signature of block?.signatures || []) { - if (limit && count >= limit) break; - const transformedTx = await this._getTransformedTx(rpc, network, { signature }); - stream.push(transformedTx); - count++; - } - stream.push(null); - const result = await ExternalApiStream.onStream(stream, req!, res!); - if (!result?.success) { - logger.error('Error mid-stream (streamTransactions): %o', result.error?.log || result.error); - } - return resolve(); - } catch (err: any) { - logger.error('Error streaming block transactions: %o', err.stack || err.message || err); - reject(err); - } + const { rpc } = await this.getRpc(network); + const block: any = await rpc.getBlock({ height: blockHeight, transactionDetails: 'signatures' }); + if (!block) { + throw new Error('Block not found: ' + blockHeight); + } + const stream = new TransformWithEventPipe({ + objectMode: true, + passThrough: true }); + let count = 0; + for (const signature of block?.signatures || []) { + if (limit && count >= limit) break; + const transformedTx = await this._getTransformedTx(rpc, network, { signature }); + stream.push(transformedTx); + count++; + } + stream.push(null); + return stream; } async streamAddressTransactions(params: StreamAddressUtxosParams) { - return new Promise(async (resolve, reject) => { - const { req, res } = params; - try { - const addressStream = await this._buildAddressTransactionsStream(params); - const result = await ExternalApiStream.onStream(addressStream, req!, res!, { jsonl: true }); - if (!result?.success) { - logger.error('Error mid-stream (streamAddressTransactions): %o', result.error?.log || result.error); - } - return resolve(); - } catch (err) { - return reject(err); - } - }); + const addressStream: any = await this._buildAddressTransactionsStream(params); + addressStream.jsonl = true; + return addressStream; } async streamWalletTransactions(params: StreamWalletTransactionsParams): Promise { - return new Promise(async (resolve, reject) => { - try { - const { wallet, req, res } = params; - const walletStream = new TransformWithEventPipe({ objectMode: true, passThrough: true }); - const walletAddresses = (await this.getWalletAddresses(wallet._id!)).map(waddress => waddress.address); - const addressStreams: TransformWithEventPipe[] = []; - - for (const address of walletAddresses) { - const addressStream = await this._buildAddressTransactionsStream({ ...params, address }); - addressStreams.push(addressStream); - } - ExternalApiStream.mergeStreams(addressStreams, walletStream); - const result = await ExternalApiStream.onStream(walletStream, req!, res!, { jsonl: true }); - if (!result?.success) { - logger.error('Error mid-stream (streamWalletTransactions): %o', result.error?.log || result.error); - } - return resolve(); - } catch (err: any) { - logger.error('Error streaming wallet transactions: %o', err.stack || err.message || err); - return reject(err); - } - }); + const { wallet } = params; + const walletStream: any = new TransformWithEventPipe({ objectMode: true, passThrough: true }); + const walletAddresses = (await this.getWalletAddresses(wallet._id!)).map(waddress => waddress.address); + const addressStreams: TransformWithEventPipe[] = []; + + for (const address of walletAddresses) { + const addressStream = await this._buildAddressTransactionsStream({ ...params, address }); + addressStreams.push(addressStream); + } + ExternalApiStream.mergeStreams(addressStreams, walletStream); + walletStream.jsonl = true; + return walletStream; } async _buildAddressTransactionsStream(params: StreamAddressUtxosParams) { @@ -617,48 +586,37 @@ export class BaseSVMStateProvider extends InternalStateProvider implements IChai } async streamBlocks(params: StreamBlocksParams) { - return new Promise(async (resolve, reject) => { - try { - const { chain, network, req, res } = params; - if (!chain || !network) { - throw new Error('Missing chain or network'); - } - const { rpc } = await this.getRpc(network); - const blockRange = await this.getBlocksRange({ ...params }); - const { height } = await rpc.getTip(); - const stream = new TransformWithEventPipe({ - objectMode: true, - passThrough: true - }); - try { - let block; - let nextBlock; - for (const blockNum of blockRange) { - const thisNextBlock = Number(block?.height) === blockNum + 1 ? block : await this._getTransformedBlock(rpc, network, blockNum + 1); - block = Number(nextBlock?.number) === blockNum ? nextBlock : await this._getTransformedBlock(rpc, network, blockNum); - if (!block) { - continue; - } - nextBlock = thisNextBlock; - block.nextBlockHash = nextBlock?.hash; - block.confirmations = Number(BigInt(height) - BigInt(block.height) + 1n); - stream.push(block); - } - } catch (e: any) { - logger.error('Error streaming blocks: %o', e); + const { chain, network } = params; + if (!chain || !network) { + throw new Error('Missing chain or network'); + } + const { rpc } = await this.getRpc(network); + const blockRange = await this.getBlocksRange({ ...params }); + const { height } = await rpc.getTip(); + const stream: any = new TransformWithEventPipe({ + objectMode: true, + passThrough: true + }); + try { + let block; + let nextBlock; + for (const blockNum of blockRange) { + const thisNextBlock = Number(block?.height) === blockNum + 1 ? block : await this._getTransformedBlock(rpc, network, blockNum + 1); + block = Number(nextBlock?.number) === blockNum ? nextBlock : await this._getTransformedBlock(rpc, network, blockNum); + if (!block) { + continue; } - stream.push(null); - const result = await ExternalApiStream.onStream(stream, req!, res!, { jsonl: true }); - if (!result?.success) { - logger.error('Error mid-stream (streamBlocks): %o', result.error?.log || result.error); - } - return resolve(); - } catch (err: any) { - logger.error('Error streaming blocks: %o', err.stack || err.message || err); - reject(err); + nextBlock = thisNextBlock; + block.nextBlockHash = nextBlock?.hash; + block.confirmations = Number(BigInt(height) - BigInt(block.height) + 1n); + stream.push(block); } - }); - + } catch (e: any) { + logger.error('Error streaming blocks: %o', e); + } + stream.push(null); + stream.jsonl = true; + return stream; } async _getTransformedBlock(rpc, network, height ) { diff --git a/packages/bitcore-node/src/routes/api/address.ts b/packages/bitcore-node/src/routes/api/address.ts index be5bdfe8f91..b714e8bc38b 100644 --- a/packages/bitcore-node/src/routes/api/address.ts +++ b/packages/bitcore-node/src/routes/api/address.ts @@ -2,7 +2,7 @@ import express, { Request } from 'express'; import logger from '../../logger'; import { ChainStateProvider } from '../../providers/chain-state'; import { StreamAddressUtxosParams } from '../../types/namespaces/ChainStateProvider'; -import { respondWithError } from '../apiUtils'; +import { respondWithError, streamJsonArray } from '../apiUtils'; const router = express.Router({ mergeParams: true }); @@ -14,11 +14,13 @@ async function streamCoins(req: Request, res) { chain, network, address, - req, - res, args: { ...req.query, unspent, limit, since } } as StreamAddressUtxosParams; - await ChainStateProvider.streamAddressTransactions(payload); + const stream = await ChainStateProvider.streamAddressTransactions(payload); + const result = await streamJsonArray(stream, req, res); + if (!result.success) { + logger.error('Error mid-stream (streamAddressTransactions): %o', result.error?.log || result.error); + } } catch (err: any) { logger.error('Error streaming coins: %o', err.stack || err.message || err); respondWithError(res, err); diff --git a/packages/bitcore-node/src/routes/api/block.ts b/packages/bitcore-node/src/routes/api/block.ts index dfd0d3b2576..6b1723af4d9 100644 --- a/packages/bitcore-node/src/routes/api/block.ts +++ b/packages/bitcore-node/src/routes/api/block.ts @@ -5,6 +5,7 @@ import { CoinStorage, ICoin } from '../../models/coin'; import { TransactionStorage } from '../../models/transaction'; import { ChainStateProvider } from '../../providers/chain-state'; import { isDateValid } from '../../utils'; +import { respondWithError, streamJsonArray } from '../apiUtils'; import { CacheTimes, Confirmations, SetCache } from '../middleware'; const router = express.Router({ mergeParams: true }); @@ -22,14 +23,17 @@ router.get('/', async function(req: Request, res: Response) { chain, network, sinceBlock, - args: { date, limit, since, direction, paging }, - req, - res + args: { date, limit, since, direction, paging } }; - return ChainStateProvider.streamBlocks(payload); + const stream = await ChainStateProvider.streamBlocks(payload); + const result = await streamJsonArray(stream, req, res); + if (!result.success) { + logger.error('Error mid-stream (streamBlocks): %o', result.error?.log || result.error); + } + return; } catch (err: any) { logger.error('Error getting blocks: %o', err.stack || err.message || err); - return res.status(500).send(err.message || err); + return respondWithError(res, err); } }); diff --git a/packages/bitcore-node/src/routes/api/tx.ts b/packages/bitcore-node/src/routes/api/tx.ts index c3e35322a53..cfe6f4fb76f 100644 --- a/packages/bitcore-node/src/routes/api/tx.ts +++ b/packages/bitcore-node/src/routes/api/tx.ts @@ -4,7 +4,7 @@ import { ICoin } from '../../models/coin'; import { ITransaction } from '../../models/transaction'; import { ChainStateProvider } from '../../providers/chain-state'; import { StreamTransactionsParams } from '../../types/namespaces/ChainStateProvider'; -import { respondWithError } from '../apiUtils'; +import { respondWithError, streamJsonArray } from '../apiUtils'; import { CacheTimes, SetCache } from '../middleware'; const router = Router({ mergeParams: true }); @@ -24,8 +24,6 @@ router.get('/', async function(req: Request, res: Response) { const payload: StreamTransactionsParams = { chain, network, - req, - res, args: { limit, since, direction, paging } }; @@ -35,7 +33,12 @@ router.get('/', async function(req: Request, res: Response) { if (blockHash !== undefined) { payload.args.blockHash = blockHash; } - return await ChainStateProvider.streamTransactions(payload); + const stream = await ChainStateProvider.streamTransactions(payload); + const result = await streamJsonArray(stream, req, res); + if (!result.success) { + logger.error('Error mid-stream (streamTransactions): %o', result.error?.log || result.error); + } + return; } catch (err: any) { logger.error('Error streaming transactions: %o', err.stack || err.message || err); return respondWithError(res, err); diff --git a/packages/bitcore-node/src/routes/api/wallet.ts b/packages/bitcore-node/src/routes/api/wallet.ts index 4ce02e77c20..d58d260cb45 100644 --- a/packages/bitcore-node/src/routes/api/wallet.ts +++ b/packages/bitcore-node/src/routes/api/wallet.ts @@ -5,6 +5,7 @@ import logger from '../../logger'; import { ChainStateProvider } from '../../providers/chain-state'; import { StreamWalletAddressesParams } from '../../types/namespaces/ChainStateProvider'; import { Auth, AuthenticatedRequest } from '../../utils/auth'; +import { respondWithError, streamJsonArray } from '../apiUtils'; const router = Router({ mergeParams: true }); @@ -68,14 +69,17 @@ router.get('/:pubKey/addresses', Auth.authenticateMiddleware, async (req: Authen chain, network, walletId: wallet!._id!, - limit, - req, - res + limit }; - return await ChainStateProvider.streamWalletAddresses(payload); + const stream = await ChainStateProvider.streamWalletAddresses(payload); + const result = await streamJsonArray(stream, req, res); + if (!result.success) { + logger.error('Error mid-stream (streamWalletAddresses): %o', result.error?.log || result.error); + } + return; } catch (err: any) { logger.error('Error streaming wallet addresses: %o', err.stack || err.message || err); - return res.status(500).send(err.message || err); + return respondWithError(res, err); } }); @@ -143,17 +147,20 @@ router.post('/:pubKey', Auth.authenticateMiddleware, async (req: AuthenticatedRe router.get('/:pubKey/transactions', Auth.authenticateMiddleware, async (req: AuthenticatedRequest, res: Response) => { try { const { chain, network } = req.params; - return await ChainStateProvider.streamWalletTransactions({ + const stream = await ChainStateProvider.streamWalletTransactions({ chain, network, wallet: req.wallet!, - req, - res, args: req.query }); + const result = await streamJsonArray(stream, req, res); + if (!result.success) { + logger.error('Error mid-stream (streamWalletTransactions): %o', result.error?.log || result.error); + } + return; } catch (err: any) { logger.error('Error streaming wallet txs: %o', err.stack || err.message || err); - return res.status(500).send(err.message || err); + return respondWithError(res, err); } }); @@ -194,18 +201,21 @@ router.get('/:pubKey/utxos', Auth.authenticateMiddleware, async (req: Authentica const { chain, network } = req.params; const { limit } = req.query as any; try { - return ChainStateProvider.streamWalletUtxos({ + const stream = await ChainStateProvider.streamWalletUtxos({ chain, network, wallet: req.wallet!, limit, - req, - res, args: req.query }); + const result = await streamJsonArray(stream, req, res); + if (!result.success) { + logger.error('Error mid-stream (streamWalletUtxos): %o', result.error?.log || result.error); + } + return; } catch (err: any) { logger.error('Error streaming wallet utxos: %o', err.stack || err.message || err); - return res.status(500).send(err.message || err); + return respondWithError(res, err); } }); diff --git a/packages/bitcore-node/src/services/storage.ts b/packages/bitcore-node/src/services/storage.ts index 593b40c776e..c258e6c03c4 100644 --- a/packages/bitcore-node/src/services/storage.ts +++ b/packages/bitcore-node/src/services/storage.ts @@ -1,9 +1,8 @@ import { EventEmitter } from 'events'; import { Readable } from 'stream'; import { ObjectId } from 'bson'; -import { Request, Response } from 'express'; import { ObjectID } from 'mongodb'; -import { Cursor, Db, MongoClient } from 'mongodb'; +import { Db, MongoClient } from 'mongodb'; import { LoggifyClass } from '../decorators/Loggify'; import logger from '../logger'; import '../models'; @@ -113,91 +112,6 @@ export class StorageService { return typecastedValue; } - stream(input: Readable, req: Request, res: Response) { - let closed = false; - req.on('close', function() { - closed = true; - }); - res.on('close', function() { - closed = true; - }); - input.on('error', function(err) { - if (!closed) { - closed = true; - return res.status(500).end(err.message); - } - return; - }); - let isFirst = true; - res.type('json'); - input.on('data', function(data) { - if (!closed) { - if (isFirst) { - res.write('[\n'); - isFirst = false; - } else { - res.write(',\n'); - } - res.write(JSON.stringify(data)); - } - }); - input.on('end', function() { - if (!closed) { - if (isFirst) { - // there was no data - res.write('[]'); - } else { - res.write('\n]'); - } - res.end(); - } - }); - } - - apiStream(cursor: Cursor, req: Request, res: Response) { - let closed = false; - req.on('close', function() { - closed = true; - cursor.close(); - }); - res.on('close', function() { - closed = true; - cursor.close(); - }); - cursor.on('error', function(err) { - if (!closed) { - closed = true; - return res.status(500).end(err.message); - } - return; - }); - let isFirst = true; - res.type('json'); - cursor.on('data', function(data) { - if (!closed) { - if (isFirst) { - res.write('[\n'); - isFirst = false; - } else { - res.write(',\n'); - } - res.write(data); - } else { - cursor.close(); - } - }); - cursor.on('end', function() { - if (!closed) { - if (isFirst) { - // there was no data - res.write('[]'); - } else { - res.write('\n]'); - } - res.end(); - } - }); - } getFindOptions(model: TransformableModel, originalOptions: StreamingFindOptions) { const query: any = {}; let since: any = null; @@ -232,10 +146,8 @@ export class StorageService { model: TransformableModel, originalQuery: any, originalOptions: StreamingFindOptions, - req: Request, - res: Response, transform?: (data: T) => string | Buffer - ) { + ): Readable & { close?: () => void } { const { query, options } = this.getFindOptions(model, originalOptions); const finalQuery = Object.assign({}, originalQuery, query); let cursor = model.collection @@ -247,7 +159,7 @@ export class StorageService { if (options.sort) { cursor = cursor.sort(options.sort); } - return this.apiStream(cursor, req, res); + return cursor; } } diff --git a/packages/bitcore-node/src/types/namespaces/ChainStateProvider.ts b/packages/bitcore-node/src/types/namespaces/ChainStateProvider.ts index cc6d550e62a..e2ac4de6235 100644 --- a/packages/bitcore-node/src/types/namespaces/ChainStateProvider.ts +++ b/packages/bitcore-node/src/types/namespaces/ChainStateProvider.ts @@ -61,8 +61,6 @@ export type StreamBlocksParams = ChainNetwork & { blockId?: string; sinceBlock: number | string; args?: Partial<{ startDate: Date; endDate: Date; date: Date } & StreamingFindOptions>; - req: Request; - res: Response; }; export type FeeMode = 'ECONOMICAL' | 'CONSERVATIVE'; @@ -104,14 +102,10 @@ export type GetWalletBalanceAtTimeParams = ChainNetwork & { export type StreamAddressUtxosParams = ChainNetwork & { address: string; - req?: Request; - res?: Response; args: Partial & any>; }; export type StreamTransactionsParams = ChainNetwork & { - req: Request; - res: Response; args: any; }; export type StreamTransactionParams = ChainNetwork & { @@ -119,8 +113,6 @@ export type StreamTransactionParams = ChainNetwork & { }; export type StreamWalletAddressesParams = ChainNetwork & { walletId: ObjectId; - req: Request; - res: Response; limit: number; }; @@ -141,8 +133,6 @@ export type StreamWalletMissingAddressesParams = ChainNetwork & { export type StreamWalletTransactionsParams = ChainNetwork & { wallet: MongoBound; - req: Request; - res: Response; args: StreamWalletTransactionsArgs & any; }; @@ -160,8 +150,6 @@ export type StreamWalletUtxosParams = ChainNetwork & { wallet: MongoBound; limit: number; args: Partial; - req: Request; - res: Response; }; export type isValidParams = ChainNetwork & { diff --git a/packages/bitcore-node/test/integration/ethereum/csp.test.ts b/packages/bitcore-node/test/integration/ethereum/csp.test.ts index 8b234b3e7d6..5a05af98850 100644 --- a/packages/bitcore-node/test/integration/ethereum/csp.test.ts +++ b/packages/bitcore-node/test/integration/ethereum/csp.test.ts @@ -12,6 +12,7 @@ import { ETH } from '../../../src/modules/ethereum/api/csp'; import { EVMBlockStorage } from '../../../src/providers/chain-state/evm/models/block'; import { EVMTransactionStorage } from '../../../src/providers/chain-state/evm/models/transaction'; import { IEVMTransactionInProcess } from '../../../src/providers/chain-state/evm/types'; +import { streamJsonArray } from '../../../src/routes/apiUtils'; import { StreamWalletTransactionsParams } from '../../../src/types/namespaces/ChainStateProvider'; import { ErigonEthBlocks } from '../../data/ETH/erigonDbBlocks'; import { ErigonEthTransactions } from '../../data/ETH/erigonDbTransactions'; @@ -230,7 +231,8 @@ describe('Ethereum API', function() { transform: (_data, _, cb) => cb(null) }) as unknown) as Request; - await ETH.streamAddressTransactions({ chain, network, address, res, req, args: {} }); + const stream = await ETH.streamAddressTransactions({ chain, network, address, args: {} }); + await streamJsonArray(stream as any, req, res); let counter = 0; await new Promise(r => { res @@ -271,7 +273,8 @@ describe('Ethereum API', function() { } }) as unknown) as Request; - await ETH.streamTransactions({ chain, network, res, req, args: { blockHeight: 1 } }); + const stream = await ETH.streamTransactions({ chain, network, args: { blockHeight: 1 } }); + await streamJsonArray(stream as any, req, res); let counter = 0; await new Promise(r => { res @@ -316,7 +319,8 @@ describe('Ethereum API', function() { } }) as unknown) as Request; - await ETH.streamTransactions({ chain, network, res, req, args: { blockHash: '12345' } }); + const stream = await ETH.streamTransactions({ chain, network, args: { blockHash: '12345' } }); + await streamJsonArray(stream as any, req, res); let counter = 0; await new Promise(r => { res @@ -455,12 +459,11 @@ const streamWalletTransactionsTest = async (chain: string, network: string, incl chain, network, wallet, - req, - res, args: { includeInvalidTxs } } as StreamWalletTransactionsParams) + .then((stream: any) => streamJsonArray(stream, req, res)) .catch(e => r(e)); }); @@ -485,7 +488,8 @@ const streamDexWalletTransactions = async (chain, network, wallet, address, web3 } }) as unknown) as Request; - ETH.streamWalletTransactions({ chain, network, wallet, res, req, args: {} }); + ETH.streamWalletTransactions({ chain, network, wallet, args: {} } as StreamWalletTransactionsParams) + .then((stream: any) => streamJsonArray(stream, req, res)); let total = BigInt(0); let totalRejected = BigInt(0); let totalFee = BigInt(0); diff --git a/packages/bitcore-node/test/integration/matic/csp.test.ts b/packages/bitcore-node/test/integration/matic/csp.test.ts index 20658c4da86..1fedbaecdc9 100644 --- a/packages/bitcore-node/test/integration/matic/csp.test.ts +++ b/packages/bitcore-node/test/integration/matic/csp.test.ts @@ -12,6 +12,7 @@ import { MATIC } from '../../../src/modules/matic/api/csp'; import { IEVMTransactionInProcess } from '../../../src/providers/chain-state/evm//types'; import { EVMBlockStorage } from '../../../src/providers/chain-state/evm/models/block'; import { EVMTransactionStorage } from '../../../src/providers/chain-state/evm/models/transaction'; +import { streamJsonArray } from '../../../src/routes/apiUtils'; import { StreamWalletTransactionsParams } from '../../../src/types/namespaces/ChainStateProvider'; import { intAfterHelper, intBeforeHelper } from '../../helpers/integration'; @@ -195,7 +196,8 @@ describe('Polygon/MATIC API', function() { transform: (_data, _, cb) => cb(null) }) as unknown) as Request; - await MATIC.streamAddressTransactions({ chain, network, address, res, req, args: {} }); + const stream = await MATIC.streamAddressTransactions({ chain, network, address, args: {} }); + await streamJsonArray(stream as any, req, res); let counter = 0; await new Promise(r => { res @@ -236,7 +238,8 @@ describe('Polygon/MATIC API', function() { } }) as unknown) as Request; - await MATIC.streamTransactions({ chain, network, res, req, args: { blockHeight: 1 } }); + const stream = await MATIC.streamTransactions({ chain, network, args: { blockHeight: 1 } }); + await streamJsonArray(stream as any, req, res); let counter = 0; await new Promise(r => { res @@ -281,7 +284,8 @@ describe('Polygon/MATIC API', function() { } }) as unknown) as Request; - await MATIC.streamTransactions({ chain, network, res, req, args: { blockHash: '12345' } }); + const stream = await MATIC.streamTransactions({ chain, network, args: { blockHash: '12345' } }); + await streamJsonArray(stream as any, req, res); let counter = 0; await new Promise(r => { res @@ -413,12 +417,11 @@ const streamWalletTransactionsTest = async (chain: string, network: string, incl chain, network, wallet, - req, - res, args: { includeInvalidTxs } } as StreamWalletTransactionsParams) + .then((stream: any) => streamJsonArray(stream, req, res)) .catch(e => r(e)); }); diff --git a/packages/bitcore-node/test/integration/solana/csp.test.ts b/packages/bitcore-node/test/integration/solana/csp.test.ts index 0e2ec2c214c..a85319ccc59 100644 --- a/packages/bitcore-node/test/integration/solana/csp.test.ts +++ b/packages/bitcore-node/test/integration/solana/csp.test.ts @@ -9,6 +9,7 @@ import { IWallet, WalletStorage } from '../../../src/models/wallet'; import { WalletAddressStorage } from '../../../src/models/walletAddress'; import { SOL } from '../../../src/modules/solana/api/csp'; import { SVMRouter } from '../../../src/providers/chain-state/svm/api/routes'; +import { streamJsonArray } from '../../../src/routes/apiUtils'; import { intAfterHelper, intBeforeHelper } from '../../helpers/integration'; describe('Solana API', function() { @@ -210,10 +211,9 @@ describe('Solana API', function() { chain, network, wallet, - req, - res, args: {} - }) + } as any) + .then((stream: any) => streamJsonArray(stream, req, res)) .catch(e => r(e)); }); From 64253db5e837ee23ca31b10507e2e25beb1b323f Mon Sep 17 00:00:00 2001 From: leolambo Date: Wed, 29 Apr 2026 11:20:15 -0400 Subject: [PATCH 03/19] Remove legacy onStream piping helpers ExternalApiStream.onStream, NodeQueryStream.onStream, Storage.apiStream, and Storage.stream all duplicated the same JSON-array framing and client-disconnect handling. With the route-layer streamJsonArray helper now owning that logic and every CSP migrated to return Readable, these are unused and removed along with their express imports. --- .../chain-state/external/streams/apiStream.ts | 98 +------------------ .../external/streams/nodeStream.ts | 6 -- 2 files changed, 1 insertion(+), 103 deletions(-) diff --git a/packages/bitcore-node/src/providers/chain-state/external/streams/apiStream.ts b/packages/bitcore-node/src/providers/chain-state/external/streams/apiStream.ts index 4f3255b9dc4..20dce148351 100644 --- a/packages/bitcore-node/src/providers/chain-state/external/streams/apiStream.ts +++ b/packages/bitcore-node/src/providers/chain-state/external/streams/apiStream.ts @@ -1,13 +1,8 @@ -import { Readable, Stream, Transform } from 'stream'; +import { Stream, Transform } from 'stream'; import axios from 'axios'; -import { Request, Response } from 'express'; import { ReadableWithEventPipe, TransformWithEventPipe } from '../../../../utils/streamWithEventPipe'; -export interface StreamOpts { - jsonl?: boolean; -} - export class ExternalApiStream extends ReadableWithEventPipe { url: string; headers: any; @@ -73,97 +68,6 @@ export class ExternalApiStream extends ReadableWithEventPipe { } } - // handles events emitted by the streamed response, request from client, and response to client - static onStream(stream: Readable, req: Request, res: Response, opts: StreamOpts = {}): - Promise<{ success: boolean; error?: any }> { - return new Promise<{ success: boolean; error?: any }>((resolve, reject) => { - let closed = false; - let isFirst = true; - - req.on('close', function() { - closed = true; - }); - - res.type('json'); - res.on('close', function() { - closed = true; - }); - - stream.on('error', function(err: any) { - if (!closed) { - closed = true; - if (err.isAxiosError) { - err.log = { - url: err?.config?.url, - statusCode: err?.response?.status, - statusMsg: err?.response?.statusText, - data: err?.response?.data, - }; - } - if (err.log?.data?.message?.includes('not supported')) { - res.write('[]'); - res.end(); - return resolve({ success: false, error: err }); - } - if (!isFirst) { - // Data has already been written to the stream and status 200 headers have already been sent - // We notify and log the error instead of throwing - const errMsg = '{"error": "An error occurred during data stream"}'; - if (opts.jsonl) { - res.write(`${errMsg}`); - } else { - res.write(`,\n${errMsg}\n]`); - } - res.end(); - res.destroy(); - return resolve({ success: false, error: err }); - } else { - // Rejecting here allows downstream to send status 500 - return reject(err); - } - } - return; - }); - stream.on('data', function(data) { - if (!closed) { - // We are assuming jsonl data appended a new line upstream - if (!opts.jsonl) { - if (isFirst) { - res.write('[\n'); - } else { - res.write(',\n'); - } - } - if (isFirst) { - // All cases need isFirst set correctly for proper error handling - isFirst = false; - } - if (typeof data !== 'string') { - data = JSON.stringify(data); - } - res.write(data); - } else { - stream.destroy(); - } - }); - stream.on('end', function() { - if (!closed) { - closed = true; - if (!opts.jsonl) { - if (isFirst) { - // there was no data - res.write('[]'); - } else { - res.write('\n]'); - } - } - res.end(); - resolve({ success: true }); - } - }); - }); - } - static mergeStreams(streams: Stream[], destination: Transform): Transform { let activeStreams = streams.length; diff --git a/packages/bitcore-node/src/providers/chain-state/external/streams/nodeStream.ts b/packages/bitcore-node/src/providers/chain-state/external/streams/nodeStream.ts index 8dec499012d..87461590413 100644 --- a/packages/bitcore-node/src/providers/chain-state/external/streams/nodeStream.ts +++ b/packages/bitcore-node/src/providers/chain-state/external/streams/nodeStream.ts @@ -1,5 +1,4 @@ import { Readable, Stream, Transform } from 'stream'; -import { Request, Response } from 'express'; import { ExternalApiStream } from './apiStream'; export class NodeQueryStream extends Readable { @@ -37,11 +36,6 @@ export class NodeQueryStream extends Readable { } } - static onStream(stream: Readable, req: Request, res: Response): - Promise<{ success: boolean; error?: any }> { - return ExternalApiStream.onStream(stream, req, res); - } - static mergeStreams(streams: Stream[], destination: Transform): Transform { return ExternalApiStream.mergeStreams(streams, destination); } From 5335ef033cdbb79ee78b420206ea42058af029c9 Mon Sep 17 00:00:00 2001 From: leolambo Date: Thu, 30 Apr 2026 14:14:27 -0400 Subject: [PATCH 04/19] Auto-detect jsonl format from stream property Lets CSPs that produce newline-delimited JSON (wallet transactions, SVM streams) set stream.jsonl = true once at construction. Routes call streamJsonArray uniformly without needing chain-specific format checks. The opts.jsonl override still wins when the caller passes one. --- packages/bitcore-node/src/routes/apiUtils.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/bitcore-node/src/routes/apiUtils.ts b/packages/bitcore-node/src/routes/apiUtils.ts index abf9814d64a..3348f31732b 100644 --- a/packages/bitcore-node/src/routes/apiUtils.ts +++ b/packages/bitcore-node/src/routes/apiUtils.ts @@ -29,11 +29,13 @@ export interface StreamJsonArrayResult { * - Client/response disconnects destroy the stream (and call .close() if present, e.g. mongo cursor) */ export function streamJsonArray( - stream: Readable & { close?: () => void }, + stream: Readable & { close?: () => void; jsonl?: boolean }, req: Request, res: Response, opts: StreamJsonArrayOpts = {} ): Promise { + // Auto-detect jsonl flag attached to the stream so routes stay chain-agnostic. + const jsonl = opts.jsonl ?? stream.jsonl ?? false; return new Promise((resolve, reject) => { let closed = false; let isFirst = true; @@ -71,7 +73,7 @@ export function streamJsonArray( // Headers already sent — emit inline error marker, end response, log upstream closed = true; const errMsg = '{"error": "An error occurred during data stream"}'; - if (opts.jsonl) { + if (jsonl) { res.write(`${errMsg}`); } else { res.write(`,\n${errMsg}\n]`); @@ -89,7 +91,7 @@ export function streamJsonArray( cleanup(); return; } - if (!opts.jsonl) { + if (!jsonl) { if (isFirst) { res.write('[\n'); } else { @@ -108,7 +110,7 @@ export function streamJsonArray( stream.on('end', () => { if (closed) return; closed = true; - if (!opts.jsonl) { + if (!jsonl) { if (isFirst) { res.write('[]'); } else { From fea037576a5f03a1553d05e65026b736b2aa0b0a Mon Sep 17 00:00:00 2001 From: leolambo Date: Fri, 1 May 2026 13:27:04 -0400 Subject: [PATCH 05/19] Restore eager cursor cleanup on client disconnect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit streamJsonArray's req/res close handlers now also call stream.destroy(), not just stream.close(). Transform pipelines like the EVM and Gnosis wallet-tx streams have no .close(), so disconnects previously left them running until the next data event triggered cleanup. Destroying the transform fires its 'close' event, which the CSP layer hooks for cursor teardown — restoring the synchronous cleanup the old req.on('close') wiring provided. Also documents two intentional behavioral nuances surfaced during review: ERC-20 transfers now stream as a multi-line JSON array (same JSON value, different whitespace from the prior res.json), and the empty-wallet 400 now uses the JSON {error, message} body shape from respondWithError instead of text/plain. --- .../src/providers/chain-state/evm/api/csp.ts | 5 +++++ packages/bitcore-node/src/routes/apiUtils.ts | 16 +++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts b/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts index fdd24211e57..7e43bb897ef 100644 --- a/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts +++ b/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts @@ -554,6 +554,9 @@ export class BaseEVMStateProvider extends InternalStateProvider implements IChai return Storage.apiStreamingFind(EVMTransactionStorage, query, { limit /* since, paging: '_id'*/ }); } const tokenTransfers = await this.getErc20Transfers(network, address, tokenAddress, args); + // Streams elements one-by-one so the route wraps them via streamJsonArray. + // The response remains a JSON array of the same N transfer objects; only inter-element + // whitespace differs from the prior res.json() output (compact `[..]` vs newline-separated). return Readable.from(tokenTransfers, { objectMode: true }); } @@ -672,6 +675,8 @@ export class BaseEVMStateProvider extends InternalStateProvider implements IChai let transactionStream = new TransformWithEventPipe({ objectMode: true, passThrough: true }); const walletAddresses = (await this.getWalletAddresses(wallet._id!)).map(waddres => waddres.address); if (walletAddresses.length === 0) { + // Status remains 400 via respondWithError; body shape changes from text/plain to + // the JSON {error, message} shape used by every other 4xx path. throw new AdapterError('walletAddresses', AdapterErrorCode.INVALID_REQUEST, 'No addresses found for wallet'); } const ethTransactionTransform = new EVMListTransactionsStream(walletAddresses, args.tokenAddress); diff --git a/packages/bitcore-node/src/routes/apiUtils.ts b/packages/bitcore-node/src/routes/apiUtils.ts index 3348f31732b..a8a3867ee40 100644 --- a/packages/bitcore-node/src/routes/apiUtils.ts +++ b/packages/bitcore-node/src/routes/apiUtils.ts @@ -40,18 +40,24 @@ export function streamJsonArray( let closed = false; let isFirst = true; - const cleanup = () => { - closed = true; + const tearDown = () => { + // close() handles mongo cursor streams; destroy() tears down piped Transform chains + // so cursor-cleanup listeners hooked to the Transform's 'close' event fire eagerly on disconnect. if (typeof stream.close === 'function') { try { stream.close(); } catch { /* noop */ } - } else if (typeof stream.destroy === 'function') { + } + if (typeof stream.destroy === 'function' && !stream.destroyed) { try { stream.destroy(); } catch { /* noop */ } } }; + const cleanup = () => { + closed = true; + tearDown(); + }; - req.on('close', () => { closed = true; if (typeof stream.close === 'function') stream.close(); }); + req.on('close', () => { closed = true; tearDown(); }); res.type('json'); - res.on('close', () => { closed = true; if (typeof stream.close === 'function') stream.close(); }); + res.on('close', () => { closed = true; tearDown(); }); stream.on('error', (err: any) => { if (closed) return; From 24059c642c5c4907632585a96e5a466a6a784aab Mon Sep 17 00:00:00 2001 From: leolambo Date: Mon, 4 May 2026 11:14:00 -0400 Subject: [PATCH 06/19] Fix stream-pipeline leaks on client disconnect streamJsonArray now settles its returned promise on req/res close and on stream 'close', not just on 'end' or 'error'. Aborted requests previously left the route handler awaiting forever because tearDown destroyed the stream but never resolved the promise, and a destroyed pipeline does not always emit 'end' upstream. Single-shot guards (safeResolve/safeReject) prevent double-settlement when these races overlap. EVM streamWalletTransactions now collects cursor-cleanup callbacks via streamParams.cleanups and runs them when the FINAL piped stream closes or ends. The previous wiring attached cleanup to the intermediate transform inside _buildWalletTransactionsStream, but the caller adds further eventPipe stages on top of it; destroy() does not reliably propagate upstream through those, so a mid-stream client disconnect could leak the Mongo cursor until natural exhaustion. --- .../src/providers/chain-state/evm/api/csp.ts | 37 +++++++++++-------- packages/bitcore-node/src/routes/apiUtils.ts | 32 ++++++++++++---- 2 files changed, 47 insertions(+), 22 deletions(-) diff --git a/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts b/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts index 7e43bb897ef..a01958a5ddf 100644 --- a/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts +++ b/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts @@ -60,6 +60,10 @@ export interface BuildWalletTxsStreamParams { transactionStream: TransformWithEventPipe; populateEffects: PopulateEffectsForAddressTransform; walletAddresses: string[]; + // _buildWalletTransactionsStream pushes teardown callbacks here (e.g. cursor.close). + // streamWalletTransactions runs them when the FINAL piped stream closes/ends, so the + // hook lives on the stream the route actually destroys on disconnect. + cleanups?: Array<() => void>; } @@ -683,10 +687,12 @@ export class BaseEVMStateProvider extends InternalStateProvider implements IChai const populateReceipt = new PopulateReceiptTransform(this); const populateEffects = new PopulateEffectsForAddressTransform(this, walletAddresses); + const cleanups: Array<() => void> = []; const streamParams: BuildWalletTxsStreamParams = { transactionStream, populateEffects, - walletAddresses + walletAddresses, + cleanups }; transactionStream = await this._buildWalletTransactionsStream(params, streamParams); @@ -699,6 +705,13 @@ export class BaseEVMStateProvider extends InternalStateProvider implements IChai .eventPipe(populateReceipt) .eventPipe(ethTransactionTransform); + // Run upstream teardown callbacks (e.g. cursor.close) when the FINAL stream the route + // pipes from closes or ends. destroy() on this stream does not reliably propagate + // upstream through eventPipe chains, so the cleanup must live here. + const runCleanups = () => { for (const fn of cleanups) { try { fn(); } catch { /* noop */ } } }; + transactionStream.on('close', runCleanups); + transactionStream.on('end', runCleanups); + (transactionStream as any).jsonl = true; return transactionStream; } @@ -714,24 +727,18 @@ export class BaseEVMStateProvider extends InternalStateProvider implements IChai .sort({ blockTimeNormalized: 1 }) .addCursorFlag('noCursorTimeout', true); - // Cursor cleanup is triggered when the consumer destroys the returned stream - // (streamJsonArray destroys on req/res close). + // Cursor cleanup is registered with the caller and triggered against the final piped + // stream. Hooking it here against the intermediate transform would miss disconnects + // because destroy() does not propagate upstream through eventPipe chains reliably. let cursorClosed = false; - const cleanupCursor = () => { - if (!cursorClosed) { - cursorClosed = true; - try { - cursor.close(); - } catch { - // Cursor might already be closed, ignore - } - } - }; + streamParams.cleanups?.push(() => { + if (cursorClosed) return; + cursorClosed = true; + try { cursor.close(); } catch { /* already closed */ } + }); // Pipe cursor to transform stream transactionStream = cursor.pipe(new TransformWithEventPipe({ objectMode: true, passThrough: true })); - transactionStream.on('close', cleanupCursor); - transactionStream.on('end', cleanupCursor); transactionStream = transactionStream.eventPipe(populateEffects); // For old db entries diff --git a/packages/bitcore-node/src/routes/apiUtils.ts b/packages/bitcore-node/src/routes/apiUtils.ts index a8a3867ee40..d12a252d800 100644 --- a/packages/bitcore-node/src/routes/apiUtils.ts +++ b/packages/bitcore-node/src/routes/apiUtils.ts @@ -39,6 +39,12 @@ export function streamJsonArray( return new Promise((resolve, reject) => { let closed = false; let isFirst = true; + let settled = false; + + // Single-shot guards keep the promise from being resolved/rejected twice when + // a client disconnect races a stream end/error or a stream 'close' event follows destroy(). + const safeResolve = (result: StreamJsonArrayResult) => { if (!settled) { settled = true; resolve(result); } }; + const safeReject = (err: any) => { if (!settled) { settled = true; reject(err); } }; const tearDown = () => { // close() handles mongo cursor streams; destroy() tears down piped Transform chains @@ -54,13 +60,21 @@ export function streamJsonArray( closed = true; tearDown(); }; + const onAbort = () => { + closed = true; + tearDown(); + // Settle the awaiting route handler so it can fall through to its catch/finally + // instead of hanging until the stream eventually emits 'close' (which may not happen + // on a destroyed pipeline if upstream never settles). + safeResolve({ success: false, error: new Error('client disconnected') }); + }; - req.on('close', () => { closed = true; tearDown(); }); + req.on('close', onAbort); res.type('json'); - res.on('close', () => { closed = true; tearDown(); }); + res.on('close', onAbort); stream.on('error', (err: any) => { - if (closed) return; + if (closed) { safeResolve({ success: false, error: err }); return; } if (err?.isAxiosError) { err.log = { url: err?.config?.url, @@ -73,7 +87,7 @@ export function streamJsonArray( closed = true; res.write('[]'); res.end(); - return resolve({ success: false, error: err }); + return safeResolve({ success: false, error: err }); } if (!isFirst) { // Headers already sent — emit inline error marker, end response, log upstream @@ -86,10 +100,10 @@ export function streamJsonArray( } res.end(); cleanup(); - return resolve({ success: false, error: err }); + return safeResolve({ success: false, error: err }); } // Pre-data — caller can send proper 5xx status - return reject(err); + return safeReject(err); }); stream.on('data', (data: any) => { @@ -124,7 +138,11 @@ export function streamJsonArray( } } res.end(); - resolve({ success: true }); + safeResolve({ success: true }); }); + + // Backstop: if destroy() emits 'close' without a prior 'end' or 'error', settle the promise + // so the route handler doesn't await indefinitely on a torn-down pipeline. + stream.on('close', () => safeResolve({ success: closed, error: closed ? undefined : new Error('stream closed before end') })); }); } From b20885608e3db0825fdcb8c252e579beb68a65df Mon Sep 17 00:00:00 2001 From: leolambo Date: Mon, 4 May 2026 13:42:00 -0400 Subject: [PATCH 07/19] Test streamJsonArray and EVM cursor cleanup Cover the JSON-array and JSONL framing paths, the inline-error and pre-data error branches, and the client-disconnect promise settlement that previously hung the route handler. Also assert the EVM wallet-tx pipeline closes its Mongo cursor when the final stream is destroyed, exercising the cleanup-array wiring through every pipe stage the route sees. --- .../test/integration/ethereum/csp.test.ts | 12 +++ .../test/unit/services/storage.test.ts | 80 +++++++++++++++++++ 2 files changed, 92 insertions(+) diff --git a/packages/bitcore-node/test/integration/ethereum/csp.test.ts b/packages/bitcore-node/test/integration/ethereum/csp.test.ts index 5a05af98850..2f722e9f5c3 100644 --- a/packages/bitcore-node/test/integration/ethereum/csp.test.ts +++ b/packages/bitcore-node/test/integration/ethereum/csp.test.ts @@ -383,6 +383,18 @@ describe('Ethereum API', function() { await streamDexWalletTransactions(chain, network, wallet, address, web3); }); + + it('closes the wallet-tx cursor when the final stream is destroyed', async () => { + await EVMTransactionStorage.collection.insertMany( + new Array(5).fill({}).map(() => ({ chain, network, blockHeight: 1, gasPrice: 10 * 1e9, data: Buffer.from(''), from: address } as IEVMTransactionInProcess)) + ); + const stream: any = await ETH.streamWalletTransactions({ chain, network, wallet, args: {} } as StreamWalletTransactionsParams); + const cursorCloseSpy = sandbox.spy(); + stream.on('close', cursorCloseSpy); + stream.destroy(); + await new Promise(r => setImmediate(r)); + expect(cursorCloseSpy.called).to.eq(true); + }); }); }); diff --git a/packages/bitcore-node/test/unit/services/storage.test.ts b/packages/bitcore-node/test/unit/services/storage.test.ts index 3bb6e39f8cb..fc4d2e6e0db 100644 --- a/packages/bitcore-node/test/unit/services/storage.test.ts +++ b/packages/bitcore-node/test/unit/services/storage.test.ts @@ -1,4 +1,7 @@ import { expect } from 'chai'; +import { EventEmitter } from 'events'; +import { Readable } from 'stream'; +import { streamJsonArray } from '../../../src/routes/apiUtils'; import { unitAfterHelper, unitBeforeHelper } from '../../helpers/unit'; describe('Storage Service', function() { @@ -9,3 +12,80 @@ describe('Storage Service', function() { expect(true).to.equal(true); }); }); + +describe('streamJsonArray', function() { + // Minimal req/res stand-ins: req only needs 'close', res captures writes and exposes 'close'. + function fakes() { + const req = new EventEmitter() as any; + const writes: string[] = []; + const res = Object.assign(new EventEmitter(), { + type: () => res, + write: (chunk: any) => { writes.push(typeof chunk === 'string' ? chunk : chunk.toString()); return true; }, + end: () => { (res as any).ended = true; }, + }) as any; + return { req, res, writes }; + } + + it('frames objects as a JSON array', async () => { + const { req, res, writes } = fakes(); + const result = await streamJsonArray(Readable.from([{ a: 1 }, { a: 2 }], { objectMode: true }), req, res); + expect(result.success).to.equal(true); + expect(writes.join('')).to.equal('[\n{"a":1},\n{"a":2}\n]'); + }); + + it('writes [] for empty stream', async () => { + const { req, res, writes } = fakes(); + await streamJsonArray(Readable.from([], { objectMode: true }), req, res); + expect(writes.join('')).to.equal('[]'); + }); + + it('honors stream.jsonl flag (no array framing)', async () => { + const { req, res, writes } = fakes(); + const stream: any = Readable.from(['{"a":1}\n', '{"a":2}\n'], { objectMode: true }); + stream.jsonl = true; + await streamJsonArray(stream, req, res); + expect(writes.join('')).to.equal('{"a":1}\n{"a":2}\n'); + }); + + it('appends inline error marker on mid-stream error', async () => { + const { req, res, writes } = fakes(); + const stream = new Readable({ objectMode: true, read() {} }); + setImmediate(() => { + stream.push({ a: 1 }); + setImmediate(() => stream.emit('error', new Error('boom'))); + }); + const result = await streamJsonArray(stream, req, res); + expect(result.success).to.equal(false); + expect(writes.join('')).to.contain('"error"'); + expect(writes.join('')).to.match(/,\n\{"error".*\}\n\]$/); + }); + + it('rejects pre-data errors so the route can send a 5xx', async () => { + const { req, res } = fakes(); + const stream = new Readable({ objectMode: true, read() {} }); + setImmediate(() => stream.emit('error', new Error('upstream down'))); + let caught: any; + await streamJsonArray(stream, req, res).catch(e => caught = e); + expect(caught).to.be.instanceOf(Error); + expect(caught.message).to.equal('upstream down'); + }); + + it('settles the promise on client disconnect', async () => { + const { req, res } = fakes(); + const stream = new Readable({ objectMode: true, read() {} }); + setImmediate(() => req.emit('close')); + const result = await streamJsonArray(stream, req, res); + expect(result.success).to.equal(false); + expect(result.error?.message).to.contain('disconnected'); + }); + + it('calls .close() on cursor-style streams when the client disconnects', async () => { + const { req, res } = fakes(); + const stream = new Readable({ objectMode: true, read() {} }) as any; + let closed = false; + stream.close = () => { closed = true; }; + setImmediate(() => req.emit('close')); + await streamJsonArray(stream, req, res); + expect(closed).to.equal(true); + }); +}); From 41d8727a6e21c415259da83c50467e1497643f23 Mon Sep 17 00:00:00 2001 From: leolambo Date: Mon, 4 May 2026 14:48:00 -0400 Subject: [PATCH 08/19] Use ndjson Content-Type when streaming jsonl Setting application/json on a newline-delimited stream tripped up JSON-aware HTTP clients (supertest's auto-parser among them) that tried to parse the whole body as a single JSON document. Switching to application/x-ndjson in jsonl mode keeps array responses on application/json while telling clients the right thing about the shape of the body. Updates the EVM memory-leak tests to drop the obsolete req/res params on streamWalletTransactions and pipe the returned stream through streamJsonArray, matching the rest of the migrated callers. --- packages/bitcore-node/src/routes/apiUtils.ts | 4 ++- .../integration/ethereum/memory-leaks.test.ts | 30 +++++-------------- 2 files changed, 11 insertions(+), 23 deletions(-) diff --git a/packages/bitcore-node/src/routes/apiUtils.ts b/packages/bitcore-node/src/routes/apiUtils.ts index d12a252d800..151c05481d7 100644 --- a/packages/bitcore-node/src/routes/apiUtils.ts +++ b/packages/bitcore-node/src/routes/apiUtils.ts @@ -70,7 +70,9 @@ export function streamJsonArray( }; req.on('close', onAbort); - res.type('json'); + // ndjson in jsonl mode so JSON-aware clients (supertest, fetch().json()) don't try to + // parse a stream of newline-delimited objects as a single JSON document. + res.type(jsonl ? 'application/x-ndjson' : 'json'); res.on('close', onAbort); stream.on('error', (err: any) => { diff --git a/packages/bitcore-node/test/integration/ethereum/memory-leaks.test.ts b/packages/bitcore-node/test/integration/ethereum/memory-leaks.test.ts index c95960b5621..80916095934 100644 --- a/packages/bitcore-node/test/integration/ethereum/memory-leaks.test.ts +++ b/packages/bitcore-node/test/integration/ethereum/memory-leaks.test.ts @@ -7,6 +7,7 @@ import { MongoBound } from '../../../src/models/base'; import { IWallet, WalletStorage } from '../../../src/models/wallet'; import { WalletAddressStorage } from '../../../src/models/walletAddress'; import { EVMTransactionStorage } from '../../../src/providers/chain-state/evm/models/transaction'; +import { streamJsonArray } from '../../../src/routes/apiUtils'; import { intAfterHelper, intBeforeHelper } from '../../helpers/integration'; const chain = 'ETH'; @@ -97,13 +98,8 @@ describe('EVM Memory Leak Prevention', function() { const { req, res, reqEmitter } = createMockReqRes(); const streamPromise = ETH.streamWalletTransactions({ - chain, - network, - wallet, - req, - res, - args: {} - }); + chain, network, wallet, args: {} + }).then(stream => streamJsonArray(stream, req, res)); // Wait for stream to start await new Promise(resolve => setTimeout(resolve, 100)); @@ -164,13 +160,8 @@ describe('EVM Memory Leak Prevention', function() { const { req, res, reqEmitter } = createMockReqRes(); const streamPromise = ETH.streamWalletTransactions({ - chain, - network, - wallet, - req, - res, - args: {} - }); + chain, network, wallet, args: {} + }).then(stream => streamJsonArray(stream, req, res)); await new Promise(resolve => setTimeout(resolve, 50)); reqEmitter.emit('close'); @@ -232,14 +223,9 @@ describe('EVM Memory Leak Prevention', function() { resEmitter.on('finish', resolve); resEmitter.on('error', reject); - ETH.streamWalletTransactions({ - chain, - network, - wallet, - req, - res, - args: {} - }).catch(reject); + ETH.streamWalletTransactions({ chain, network, wallet, args: {} }) + .then(stream => streamJsonArray(stream, req, res)) + .catch(reject); }); // Verify that we received some transactions (stream worked) From 795ce7437eabba0e4aa74b8da5201d17dd7fbf8c Mon Sep 17 00:00:00 2001 From: leolambo Date: Tue, 9 Jun 2026 15:53:01 -0400 Subject: [PATCH 09/19] Assert Mongo cursor closes when tx stream tears down The wallet-tx cursor-cleanup test spied on the returned stream's 'close' event and called destroy(). Since destroy() emits 'close' unconditionally, the test passed even if cursor teardown were removed entirely, giving false confidence. Pin the backing collection and spy on the real cursor's close() so the test fails if the teardown wiring regresses. --- .../test/integration/ethereum/csp.test.ts | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/bitcore-node/test/integration/ethereum/csp.test.ts b/packages/bitcore-node/test/integration/ethereum/csp.test.ts index 2f722e9f5c3..2b2551a0e8b 100644 --- a/packages/bitcore-node/test/integration/ethereum/csp.test.ts +++ b/packages/bitcore-node/test/integration/ethereum/csp.test.ts @@ -388,12 +388,26 @@ describe('Ethereum API', function() { await EVMTransactionStorage.collection.insertMany( new Array(5).fill({}).map(() => ({ chain, network, blockHeight: 1, gasPrice: 10 * 1e9, data: Buffer.from(''), from: address } as IEVMTransactionInProcess)) ); + // Capture the real Mongo cursor backing the stream and spy on its close(), so this + // asserts the cursor itself is torn down — not merely that the returned stream emits + // 'close' (which destroy() does unconditionally, regardless of cursor cleanup). + // `.collection` is a getter that returns a fresh wrapper each access, so pin a single + // collection instance and wrap its find() to capture the cursor. + const collection = EVMTransactionStorage.collection; + const realFind = collection.find.bind(collection); + let cursorCloseSpy: sinon.SinonSpy | undefined; + sandbox.stub(collection, 'find').callsFake((...findArgs: any[]) => { + const cursor = realFind(...findArgs); + cursorCloseSpy = cursorCloseSpy || sandbox.spy(cursor, 'close'); + return cursor; + }); + sandbox.stub(EVMTransactionStorage, 'collection').get(() => collection); + const stream: any = await ETH.streamWalletTransactions({ chain, network, wallet, args: {} } as StreamWalletTransactionsParams); - const cursorCloseSpy = sandbox.spy(); - stream.on('close', cursorCloseSpy); stream.destroy(); await new Promise(r => setImmediate(r)); - expect(cursorCloseSpy.called).to.eq(true); + expect(cursorCloseSpy, 'wallet-tx cursor was created').to.exist; + expect(cursorCloseSpy!.called, 'cursor.close() was called on destroy').to.eq(true); }); }); }); From c0a1c4f247d322d6c243c955fcb81c594b41526b Mon Sep 17 00:00:00 2001 From: leolambo Date: Tue, 9 Jun 2026 16:06:34 -0400 Subject: [PATCH 10/19] Stream Solana blocks as a JSON array streamBlocks set the jsonl flag while pushing plain block objects, so the shared /block route emitted concatenated JSON with no delimiters: invalid as both JSON and NDJSON, and inconsistent with every other chain, which return arrays. Drop the flag so the route frames an array. jsonl stays reserved for unbounded streams like wallet-tx history, whose producers emit newline-delimited strings. Add a streamBlocks test asserting array framing and content type. --- .../src/providers/chain-state/svm/api/csp.ts | 4 ++- .../test/integration/solana/csp.test.ts | 35 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/packages/bitcore-node/src/providers/chain-state/svm/api/csp.ts b/packages/bitcore-node/src/providers/chain-state/svm/api/csp.ts index c62def4b05a..1846e374e34 100644 --- a/packages/bitcore-node/src/providers/chain-state/svm/api/csp.ts +++ b/packages/bitcore-node/src/providers/chain-state/svm/api/csp.ts @@ -615,7 +615,9 @@ export class BaseSVMStateProvider extends InternalStateProvider implements IChai logger.error('Error streaming blocks: %o', e); } stream.push(null); - stream.jsonl = true; + // No jsonl flag: streamBlocks pushes plain block objects, so the route frames them as a + // JSON array — consistent with every other chain's block endpoint. (jsonl is reserved for + // unbounded streams like wallet-tx history where producers emit newline-delimited strings.) return stream; } diff --git a/packages/bitcore-node/test/integration/solana/csp.test.ts b/packages/bitcore-node/test/integration/solana/csp.test.ts index a85319ccc59..c6dbcf0715d 100644 --- a/packages/bitcore-node/test/integration/solana/csp.test.ts +++ b/packages/bitcore-node/test/integration/solana/csp.test.ts @@ -222,6 +222,41 @@ describe('Solana API', function() { }); }); + describe('#streamBlocks', () => { + it('frames blocks as a JSON array (no jsonl), consistent with other chains', async () => { + const blockRange = [100, 101]; + sandbox.stub(SOL, 'getRpc').resolves({ rpc: { getTip: sandbox.stub().resolves({ height: 200 }) } } as any); + sandbox.stub(SOL as any, 'getBlocksRange').resolves(blockRange); + sandbox.stub(SOL, '_getTransformedBlock').callsFake(((_rpc: any, _network: string, height: any) => + Promise.resolve({ height: Number(height), hash: `hash${Number(height)}` })) as any); + + const chunks: string[] = []; + let contentType: string | undefined; + const req = (new Writable({ write: (_d, _e, cb) => cb() }) as unknown) as Request; + const res = (new Writable({ write: (d, _e, cb) => { chunks.push(d.toString()); cb(); } }) as unknown) as Response; + res.type = ((type: string) => { contentType = type; return res; }) as any; + + const stream: any = await SOL.streamBlocks({ chain, network, args: {} } as any); + // The route auto-detects jsonl off the stream; blocks must not set it. + expect(stream.jsonl).to.be.undefined; + + await new Promise((resolve, reject) => { + res.on('error', reject).on('finish', () => resolve()); + streamJsonArray(stream, req, res).catch(reject); + }); + + const body = chunks.join(''); + // Array framing => application/json, and a single JSON.parse must succeed (concatenated + // objects from a stray jsonl flag would throw here). + expect(contentType).to.equal('json'); + const parsed = JSON.parse(body); + expect(parsed).to.be.an('array').with.lengthOf(2); + expect(parsed[0].height).to.equal(100); + expect(parsed[0].nextBlockHash).to.equal('hash101'); + expect(parsed[1].height).to.equal(101); + }); + }); + it('should correctly transform transaction data', () => { const tx = { txid: 'tx1', From 15fe6d27ddf58b9a60b523b921edb966c8a60bca Mon Sep 17 00:00:00 2001 From: leolambo Date: Wed, 10 Jun 2026 14:04:58 -0400 Subject: [PATCH 11/19] Adapt Solana ATA tests to stream-returning API The SPL-ATA history tests passed req/res to streamWalletTransactions, but the streaming refactor removed them from the params type, so the tests failed to compile (TS2353). Drop req/res from the params and frame the returned stream via streamJsonArray, matching the sibling wallet-transaction test. --- .../bitcore-node/test/integration/solana/csp.test.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/bitcore-node/test/integration/solana/csp.test.ts b/packages/bitcore-node/test/integration/solana/csp.test.ts index ceb64d84405..f2c2b2e3bcf 100644 --- a/packages/bitcore-node/test/integration/solana/csp.test.ts +++ b/packages/bitcore-node/test/integration/solana/csp.test.ts @@ -260,10 +260,9 @@ describe('Solana API', function() { chain, network, wallet, - req, - res, args: { tokenAddress } - }) + } as any) + .then((stream: any) => streamJsonArray(stream, req, res)) .catch(e => r(e)); }); @@ -324,10 +323,9 @@ describe('Solana API', function() { chain, network, wallet, - req, - res, args: { tokenAddress, limit: 1 } - }) + } as any) + .then((stream: any) => streamJsonArray(stream, req, res)) .catch(e => r(e)); }); From 899a9c20d8f4af4e26d45994b86d6d0701cb7077 Mon Sep 17 00:00:00 2001 From: leolambo Date: Mon, 15 Jun 2026 11:14:31 -0400 Subject: [PATCH 12/19] Adapt MATIC wallet-tx row helper to stream-returning API The streamWalletTransactionRows test helper still passed req/res to streamWalletTransactions and waited on the response's finish event. After the streaming refactor that method returns a stream and never writes to res, so the helper hung until the 30s mocha timeout. An 'as' cast had hidden the excess-property type error that caught the analogous Solana cases. Drop req/res from the params and frame the returned stream via streamJsonArray, matching the sibling streamWalletTransactionsTest. --- packages/bitcore-node/test/integration/matic/csp.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/bitcore-node/test/integration/matic/csp.test.ts b/packages/bitcore-node/test/integration/matic/csp.test.ts index 2be3ca46fc5..791710e1b65 100644 --- a/packages/bitcore-node/test/integration/matic/csp.test.ts +++ b/packages/bitcore-node/test/integration/matic/csp.test.ts @@ -506,10 +506,9 @@ const streamWalletTransactionRows = async (chain: string, network: string, walle chain, network, wallet, - req, - res, args: {} } as StreamWalletTransactionsParams) + .then((stream: any) => streamJsonArray(stream, req, res)) .catch(e => r(e)); }); From c99c92db6fe0e71395c33394aa85fcb55f0355cd Mon Sep 17 00:00:00 2001 From: lyambo Date: Wed, 12 Aug 2026 16:58:52 -0400 Subject: [PATCH 13/19] Return dispose handles from wallet tx builders The cursor teardown was smuggled through a cleanups array on the builder's input params. The builder now returns its own dispose alongside the stream, and the caller hooks it to the final piped stream's close, end, and error events via a shared disposeOnce helper that keeps overlapping events idempotent and routes teardown failures to a logger instead of the event handler. cursor.close() resolves a promise, so it is now awaited inside dispose; a sync try/catch could never see its failures. Gnosis gets the same treatment. An error on the final EVM stream now also destroys it so failures fired before the route attaches handlers surface instead of hanging the request. Also rewrite the Gnosis token history test helper, which still passed req/res and waited on a finish event that no longer fires now that the CSP returns a stream. --- .../src/modules/moralis/api/csp.ts | 10 ++- .../src/modules/multiProvider/api/csp.ts | 11 +++- .../src/providers/chain-state/evm/api/csp.ts | 65 +++++++++++-------- .../providers/chain-state/evm/api/gnosis.ts | 20 +++--- packages/bitcore-node/src/utils/index.ts | 29 +++++++++ .../test/integration/ethereum/csp.test.ts | 8 +-- .../test/unit/models/transaction.test.ts | 36 +++++----- .../test/unit/utils/index.test.ts | 33 ++++++++++ 8 files changed, 142 insertions(+), 70 deletions(-) diff --git a/packages/bitcore-node/src/modules/moralis/api/csp.ts b/packages/bitcore-node/src/modules/moralis/api/csp.ts index b39e71021f1..eb08b64530a 100644 --- a/packages/bitcore-node/src/modules/moralis/api/csp.ts +++ b/packages/bitcore-node/src/modules/moralis/api/csp.ts @@ -7,7 +7,7 @@ import { MongoBound } from '../../../models/base'; import { CacheStorage } from '../../../models/cache'; import { CoinEvent } from '../../../models/events'; import { WalletAddressStorage } from '../../../models/walletAddress'; -import { BaseEVMStateProvider, BuildWalletTxsStreamParams } from '../../../providers/chain-state/evm/api/csp'; +import { BaseEVMStateProvider, BuildWalletTxsStreamParams, BuildWalletTxsStreamResult } from '../../../providers/chain-state/evm/api/csp'; import { EVMBlockStorage } from '../../../providers/chain-state/evm/models/block'; import { EVMTransactionStorage } from '../../../providers/chain-state/evm/models/transaction'; import { EVMTransactionJSON, IEVMBlock, IEVMTransactionTransformed } from '../../../providers/chain-state/evm/types'; @@ -180,7 +180,10 @@ export class MoralisStateProvider extends BaseEVMStateProvider { } // @override - async _buildWalletTransactionsStream(params: StreamWalletTransactionsParams, streamParams: BuildWalletTxsStreamParams) { + async _buildWalletTransactionsStream( + params: StreamWalletTransactionsParams, + streamParams: BuildWalletTxsStreamParams + ): Promise { const { network, args } = params; let { transactionStream } = streamParams; const { walletAddresses } = streamParams; @@ -206,7 +209,8 @@ export class MoralisStateProvider extends BaseEVMStateProvider { this._addAddressToSubscription({ chainId, address }) .catch(e => logger.warn(`Failed to add address to ${this.chain}:${network} Moralis subscription: %o`, e)); } - return transactionStream; + // nothing to tear down here; the adapter streams run to completion on their own + return { stream: transactionStream }; } // @override diff --git a/packages/bitcore-node/src/modules/multiProvider/api/csp.ts b/packages/bitcore-node/src/modules/multiProvider/api/csp.ts index de15b9373af..1275918e0ec 100644 --- a/packages/bitcore-node/src/modules/multiProvider/api/csp.ts +++ b/packages/bitcore-node/src/modules/multiProvider/api/csp.ts @@ -24,7 +24,8 @@ import { EthDater } from '../../../utils/ethDater'; import { ReadableWithEventPipe } from '../../../utils/streamWithEventPipe'; import type { MongoBound } from '../../../models/base'; import type { - BuildWalletTxsStreamParams + BuildWalletTxsStreamParams, + BuildWalletTxsStreamResult } from '../../../providers/chain-state/evm/api/csp'; import type { EVMTransactionJSON, IEVMBlock } from '../../../providers/chain-state/evm/types'; import type { IIndexedAPIAdapter } from '../../../providers/chain-state/external/adapters/IIndexedAPIAdapter'; @@ -337,7 +338,10 @@ export class MultiProviderEVMStateProvider extends BaseEVMStateProvider { } // @override — sequential failover for wallet transaction streaming - async _buildWalletTransactionsStream(params: StreamWalletTransactionsParams, streamParams: BuildWalletTxsStreamParams) { + async _buildWalletTransactionsStream( + params: StreamWalletTransactionsParams, + streamParams: BuildWalletTxsStreamParams + ): Promise { const { network, args } = params; let { transactionStream } = streamParams; const { walletAddresses } = streamParams; @@ -387,7 +391,8 @@ export class MultiProviderEVMStateProvider extends BaseEVMStateProvider { } } - return transactionStream; + // nothing to tear down here; the adapter streams run to completion on their own + return { stream: transactionStream }; } // @override diff --git a/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts b/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts index a01958a5ddf..d86c2fff92e 100644 --- a/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts +++ b/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts @@ -16,7 +16,7 @@ import { Storage } from '../../../../services/storage'; import { IBlock } from '../../../../types/Block'; import { ChainId } from '../../../../types/ChainNetwork'; import { SpentHeightIndicators } from '../../../../types/Coin'; -import { normalizeChainNetwork, partition, range } from '../../../../utils'; +import { disposeOnce, normalizeChainNetwork, partition, range } from '../../../../utils'; import { StatsUtil } from '../../../../utils/stats'; import { TransformWithEventPipe } from '../../../../utils/streamWithEventPipe'; import { AdapterError, AdapterErrorCode } from '../../external/adapters/errors'; @@ -60,10 +60,14 @@ export interface BuildWalletTxsStreamParams { transactionStream: TransformWithEventPipe; populateEffects: PopulateEffectsForAddressTransform; walletAddresses: string[]; - // _buildWalletTransactionsStream pushes teardown callbacks here (e.g. cursor.close). - // streamWalletTransactions runs them when the FINAL piped stream closes/ends, so the - // hook lives on the stream the route actually destroys on disconnect. - cleanups?: Array<() => void>; +} + +export interface BuildWalletTxsStreamResult { + stream: TransformWithEventPipe; + // Teardown for whatever the builder opened (e.g. the mongo cursor). streamWalletTransactions + // runs it against the FINAL piped stream, because destroy() does not propagate upstream + // through eventPipe chains and the route only ever destroys the final stream. + dispose?: () => void | Promise; } @@ -687,14 +691,13 @@ export class BaseEVMStateProvider extends InternalStateProvider implements IChai const populateReceipt = new PopulateReceiptTransform(this); const populateEffects = new PopulateEffectsForAddressTransform(this, walletAddresses); - const cleanups: Array<() => void> = []; const streamParams: BuildWalletTxsStreamParams = { transactionStream, populateEffects, - walletAddresses, - cleanups + walletAddresses }; - transactionStream = await this._buildWalletTransactionsStream(params, streamParams); + const { stream: builtStream, dispose } = await this._buildWalletTransactionsStream(params, streamParams); + transactionStream = builtStream; if (!args.tokenAddress && wallet._id) { const internalTxTransform = new InternalTxRelatedFilterTransform(web3, wallet._id); @@ -705,38 +708,41 @@ export class BaseEVMStateProvider extends InternalStateProvider implements IChai .eventPipe(populateReceipt) .eventPipe(ethTransactionTransform); - // Run upstream teardown callbacks (e.g. cursor.close) when the FINAL stream the route - // pipes from closes or ends. destroy() on this stream does not reliably propagate - // upstream through eventPipe chains, so the cleanup must live here. - const runCleanups = () => { for (const fn of cleanups) { try { fn(); } catch { /* noop */ } } }; - transactionStream.on('close', runCleanups); - transactionStream.on('end', runCleanups); + // Run the builder's teardown (e.g. cursor.close) when the FINAL stream the route pipes + // from settles. destroy() on this stream does not reliably propagate upstream through + // eventPipe chains, so the hook must live here rather than beside the cursor. + const runDispose = disposeOnce(dispose, err => + logger.warn(`Failed to tear down ${this.chain} wallet transaction stream: %o`, err)); + const finalStream = transactionStream; + finalStream.on('close', runDispose); + finalStream.on('end', runDispose); + finalStream.on('error', err => { + runDispose(); + // Don't just consume the error: if it fires before the route attaches its own + // handlers the request would hang. Destroying the stream surfaces it, since + // streamJsonArray rejects up front on a destroyed stream. + if (!finalStream.destroyed) { + finalStream.destroy(err); + } + }); (transactionStream as any).jsonl = true; return transactionStream; } - async _buildWalletTransactionsStream(params: StreamWalletTransactionsParams, streamParams: BuildWalletTxsStreamParams) { + async _buildWalletTransactionsStream( + params: StreamWalletTransactionsParams, + streamParams: BuildWalletTxsStreamParams + ): Promise { const query = this.getWalletTransactionQuery(params); let { transactionStream } = streamParams; const { populateEffects } = streamParams; - // Store cursor reference for cleanup const cursor = EVMTransactionStorage.collection .find(query) .sort({ blockTimeNormalized: 1 }) .addCursorFlag('noCursorTimeout', true); - // Cursor cleanup is registered with the caller and triggered against the final piped - // stream. Hooking it here against the intermediate transform would miss disconnects - // because destroy() does not propagate upstream through eventPipe chains reliably. - let cursorClosed = false; - streamParams.cleanups?.push(() => { - if (cursorClosed) return; - cursorClosed = true; - try { cursor.close(); } catch { /* already closed */ } - }); - // Pipe cursor to transform stream transactionStream = cursor.pipe(new TransformWithEventPipe({ objectMode: true, passThrough: true })); @@ -746,7 +752,10 @@ export class BaseEVMStateProvider extends InternalStateProvider implements IChai const erc20Transform = new Erc20RelatedFilterTransform(params.args.tokenAddress); transactionStream = transactionStream.eventPipe(erc20Transform); } - return transactionStream; + + // cursor.close() resolves a promise; awaiting it is what lets a failure reach the + // caller's dispose error handler instead of becoming an unhandled rejection. + return { stream: transactionStream, dispose: async () => { await cursor.close(); } }; } async getErc20Transfers( diff --git a/packages/bitcore-node/src/providers/chain-state/evm/api/gnosis.ts b/packages/bitcore-node/src/providers/chain-state/evm/api/gnosis.ts index 251dedf0b51..71c63f863b6 100644 --- a/packages/bitcore-node/src/providers/chain-state/evm/api/gnosis.ts +++ b/packages/bitcore-node/src/providers/chain-state/evm/api/gnosis.ts @@ -1,9 +1,11 @@ import { Readable } from 'stream'; import { Utils, Web3 } from '@bitpay-labs/crypto-wallet-core'; import { ChainStateProvider } from '../../'; +import logger from '../../../../logger'; import { Config } from '../../../../services/config'; import { IEVMNetworkConfig } from '../../../../types/Config'; import { StreamWalletTransactionsParams } from '../../../../types/namespaces/ChainStateProvider'; +import { disposeOnce } from '../../../../utils'; import { MultisigAbi } from '../abi/multisig'; import { MultisigRelatedFilterTransform } from '../api/multisigTransform'; import { PopulateEffectsTransform } from '../api/populateEffectsTransform'; @@ -212,17 +214,12 @@ export class GnosisApi { .sort({ blockTimeNormalized: 1 }) .addCursorFlag('noCursorTimeout', true); - let cursorClosed = false; - const cleanupCursor = () => { - if (!cursorClosed) { - cursorClosed = true; - try { - cursor.close(); - } catch { - // Cursor might already be closed, ignore - } - } - }; + // cursor.close() resolves a promise, so a plain try/catch around it would never see a + // failure. disposeOnce awaits it and keeps the overlapping stream events idempotent. + const cleanupCursor = disposeOnce( + async () => { await cursor.close(); }, + err => logger.warn(`Failed to close ${chain}:${network} gnosis transaction cursor: %o`, err) + ); transactionStream = cursor.pipe(populateEffects); // For old db entries @@ -237,6 +234,7 @@ export class GnosisApi { finalStream.jsonl = true; finalStream.on('close', cleanupCursor); finalStream.on('end', cleanupCursor); + finalStream.on('error', cleanupCursor); return finalStream; } } diff --git a/packages/bitcore-node/src/utils/index.ts b/packages/bitcore-node/src/utils/index.ts index 74ac61a0f0b..48a93c6bdba 100644 --- a/packages/bitcore-node/src/utils/index.ts +++ b/packages/bitcore-node/src/utils/index.ts @@ -160,4 +160,33 @@ export function merge(dest: TDest, src: TSrc): TDest & TSrc { export function normalizeChainNetwork(chain: string, network: string): string { return chain.toUpperCase() + ':' + network.toLowerCase(); +} + +/** + * Wrap a teardown callback so it runs at most once and never throws at the call site. + * Stream events overlap ('end' then 'close', 'error' then either), so disposal hooked to + * all three has to be idempotent. Sync throws and rejected promises both go to onError. + * + * @param dispose teardown to run, or undefined for a no-op + * @param onError receives anything the teardown throws or rejects with + */ +export function disposeOnce( + dispose: (() => void | Promise) | undefined, + onError: (err: any) => void +): () => void { + let disposed = false; + return () => { + if (disposed || !dispose) { + return; + } + disposed = true; + try { + const result = dispose(); + if (typeof (result as Promise)?.catch === 'function') { + (result as Promise).catch(onError); + } + } catch (err) { + onError(err); + } + }; } \ No newline at end of file diff --git a/packages/bitcore-node/test/integration/ethereum/csp.test.ts b/packages/bitcore-node/test/integration/ethereum/csp.test.ts index 2b2551a0e8b..96a1a86463b 100644 --- a/packages/bitcore-node/test/integration/ethereum/csp.test.ts +++ b/packages/bitcore-node/test/integration/ethereum/csp.test.ts @@ -388,11 +388,9 @@ describe('Ethereum API', function() { await EVMTransactionStorage.collection.insertMany( new Array(5).fill({}).map(() => ({ chain, network, blockHeight: 1, gasPrice: 10 * 1e9, data: Buffer.from(''), from: address } as IEVMTransactionInProcess)) ); - // Capture the real Mongo cursor backing the stream and spy on its close(), so this - // asserts the cursor itself is torn down — not merely that the returned stream emits - // 'close' (which destroy() does unconditionally, regardless of cursor cleanup). - // `.collection` is a getter that returns a fresh wrapper each access, so pin a single - // collection instance and wrap its find() to capture the cursor. + // Spy on the real cursor's close() so this asserts the cursor is torn down, not just + // that the stream emits 'close' (destroy() always does that). + // .collection returns a fresh wrapper each access, so pin one and wrap its find(). const collection = EVMTransactionStorage.collection; const realFind = collection.find.bind(collection); let cursorCloseSpy: sinon.SinonSpy | undefined; diff --git a/packages/bitcore-node/test/unit/models/transaction.test.ts b/packages/bitcore-node/test/unit/models/transaction.test.ts index 189ca6c395d..9b76083f5ca 100644 --- a/packages/bitcore-node/test/unit/models/transaction.test.ts +++ b/packages/bitcore-node/test/unit/models/transaction.test.ts @@ -1,7 +1,7 @@ import { ObjectId } from 'bson'; import { expect } from 'chai'; import sinon from 'sinon'; -import { Readable, Writable } from 'stream'; +import { Readable } from 'stream'; import { Web3 } from '@bitpay-labs/crypto-wallet-core'; import { CoinStorage } from '../../../src/models/coin'; import { MintOp, SpendOp, TaggedBitcoinTx, TransactionStorage, TxOp } from '../../../src/models/transaction'; @@ -321,31 +321,27 @@ describe('Transaction Model', function() { populateReceipt: sandbox.stub().callsFake(tx => tx) } as any); - const rows = new Array(); - const req = new Readable({ read() {} }) as any; - const res = new Writable({ - write(chunk, _, done) { - for (const line of chunk.toString().split('\n').filter(Boolean)) { - rows.push(JSON.parse(line)); - } - done(); - } - }) as any; - const finished = new Promise((resolve, reject) => { - res.on('finish', resolve); - res.on('error', reject); - }); - - await Gnosis.streamGnosisWalletTransactions({ + // streamGnosisWalletTransactions returns a jsonl stream now; the route does the + // HTTP framing, so collect the newline-delimited rows straight off the stream. + const stream = await Gnosis.streamGnosisWalletTransactions({ chain: 'ETH', network: 'mainnet', multisigContractAddress: requestOverrides.multisigContractAddress || gnosisMultisigContractAddress, wallet: { _id: new ObjectId() }, - req, - res, args: { tokenAddress: requestOverrides.tokenAddress || gnosisBusdToken } } as any); - await finished; + + const rows = new Array(); + await new Promise((resolve, reject) => { + stream + .on('data', chunk => { + for (const line of chunk.toString().split('\n').filter(Boolean)) { + rows.push(JSON.parse(line)); + } + }) + .on('error', reject) + .on('end', resolve); + }); return { findQuery, rows }; }; diff --git a/packages/bitcore-node/test/unit/utils/index.test.ts b/packages/bitcore-node/test/unit/utils/index.test.ts index 68a17e9f833..2f262f91627 100644 --- a/packages/bitcore-node/test/unit/utils/index.test.ts +++ b/packages/bitcore-node/test/unit/utils/index.test.ts @@ -399,4 +399,37 @@ describe('Utils', function() { expect(redactUrl('')).to.eq(''); }); }); + + describe('disposeOnce', function() { + it('should run the teardown only once across repeated calls', function() { + let count = 0; + const dispose = utils.disposeOnce(() => { count++; }, () => { /* noop */ }); + dispose(); + dispose(); + dispose(); + expect(count).to.eq(1); + }); + + it('should be a no-op when there is nothing to dispose', function() { + let errored = false; + const dispose = utils.disposeOnce(undefined, () => { errored = true; }); + dispose(); + expect(errored).to.eq(false); + }); + + it('should route synchronous throws to onError', function() { + let caught: any; + const dispose = utils.disposeOnce(() => { throw new Error('sync boom'); }, err => caught = err); + dispose(); + expect(caught?.message).to.eq('sync boom'); + }); + + it('should route rejected promises to onError', async function() { + let caught: any; + const dispose = utils.disposeOnce(async () => { throw new Error('async boom'); }, err => caught = err); + dispose(); + await new Promise(r => setImmediate(r)); + expect(caught?.message).to.eq('async boom'); + }); + }); }); \ No newline at end of file From 120cae40f2fe2edb91e9c6382781aa7476b7be48 Mon Sep 17 00:00:00 2001 From: lyambo Date: Wed, 12 Aug 2026 16:59:07 -0400 Subject: [PATCH 14/19] Harden streamJsonArray failure handling Failure results now carry a reason so routes can tell a client disconnect from a real stream error. A shared logStreamFailure helper logs disconnects at info and everything else at error; before, every failure logged as a mid-stream error, so routine hangups looked like server faults. Fix the not supported branch writing '[]' after rows had already gone out, which produced an unclosed array the client could not parse. It now only short-circuits before any data and falls through to the mid-stream error framing otherwise, and tears down the source stream in both cases. Serialize each row before writing the separator so a stringify failure cannot leave a dangling comma, and route it through the error path instead of throwing out of the data handler. BigInts stringify as strings via jsonStringify. Reject up front on a stream that was destroyed before piping began so the route returns a 5xx instead of awaiting a promise that never settles. --- .../providers/chain-state/evm/api/routes.ts | 4 +- .../bitcore-node/src/routes/api/address.ts | 4 +- packages/bitcore-node/src/routes/api/block.ts | 4 +- packages/bitcore-node/src/routes/api/tx.ts | 4 +- .../bitcore-node/src/routes/api/wallet.ts | 8 +- packages/bitcore-node/src/routes/apiUtils.ts | 115 ++++++++++++++---- .../test/unit/services/storage.test.ts | 78 ++++++++++++ 7 files changed, 181 insertions(+), 36 deletions(-) diff --git a/packages/bitcore-node/src/providers/chain-state/evm/api/routes.ts b/packages/bitcore-node/src/providers/chain-state/evm/api/routes.ts index 4788d767e2d..53f154d0551 100644 --- a/packages/bitcore-node/src/providers/chain-state/evm/api/routes.ts +++ b/packages/bitcore-node/src/providers/chain-state/evm/api/routes.ts @@ -4,7 +4,7 @@ import { Router } from 'express'; import config from '../../../../config'; import logger from '../../../../logger'; import { WebhookStorage } from '../../../../models/webhook'; -import { respondWithError, streamJsonArray } from '../../../../routes/apiUtils'; +import { logStreamFailure, respondWithError, streamJsonArray } from '../../../../routes/apiUtils'; import { Config } from '../../../../services/config'; import { IEVMNetworkConfig } from '../../../../types/Config'; import { castToBool } from '../../../../utils'; @@ -258,7 +258,7 @@ export class EVMRouter { }); const result = await streamJsonArray(stream, req, res); if (!result.success) { - logger.error('Error mid-stream (streamGnosisWalletTransactions): %o', result.error?.log || result.error); + logStreamFailure(result, 'streamGnosisWalletTransactions'); } return; } catch (err: any) { diff --git a/packages/bitcore-node/src/routes/api/address.ts b/packages/bitcore-node/src/routes/api/address.ts index b714e8bc38b..52aa4df57d6 100644 --- a/packages/bitcore-node/src/routes/api/address.ts +++ b/packages/bitcore-node/src/routes/api/address.ts @@ -2,7 +2,7 @@ import express, { Request } from 'express'; import logger from '../../logger'; import { ChainStateProvider } from '../../providers/chain-state'; import { StreamAddressUtxosParams } from '../../types/namespaces/ChainStateProvider'; -import { respondWithError, streamJsonArray } from '../apiUtils'; +import { logStreamFailure, respondWithError, streamJsonArray } from '../apiUtils'; const router = express.Router({ mergeParams: true }); @@ -19,7 +19,7 @@ async function streamCoins(req: Request, res) { const stream = await ChainStateProvider.streamAddressTransactions(payload); const result = await streamJsonArray(stream, req, res); if (!result.success) { - logger.error('Error mid-stream (streamAddressTransactions): %o', result.error?.log || result.error); + logStreamFailure(result, 'streamAddressTransactions'); } } catch (err: any) { logger.error('Error streaming coins: %o', err.stack || err.message || err); diff --git a/packages/bitcore-node/src/routes/api/block.ts b/packages/bitcore-node/src/routes/api/block.ts index 6b1723af4d9..1dcdb3d80fe 100644 --- a/packages/bitcore-node/src/routes/api/block.ts +++ b/packages/bitcore-node/src/routes/api/block.ts @@ -5,7 +5,7 @@ import { CoinStorage, ICoin } from '../../models/coin'; import { TransactionStorage } from '../../models/transaction'; import { ChainStateProvider } from '../../providers/chain-state'; import { isDateValid } from '../../utils'; -import { respondWithError, streamJsonArray } from '../apiUtils'; +import { logStreamFailure, respondWithError, streamJsonArray } from '../apiUtils'; import { CacheTimes, Confirmations, SetCache } from '../middleware'; const router = express.Router({ mergeParams: true }); @@ -28,7 +28,7 @@ router.get('/', async function(req: Request, res: Response) { const stream = await ChainStateProvider.streamBlocks(payload); const result = await streamJsonArray(stream, req, res); if (!result.success) { - logger.error('Error mid-stream (streamBlocks): %o', result.error?.log || result.error); + logStreamFailure(result, 'streamBlocks'); } return; } catch (err: any) { diff --git a/packages/bitcore-node/src/routes/api/tx.ts b/packages/bitcore-node/src/routes/api/tx.ts index cfe6f4fb76f..5c25aafa1e4 100644 --- a/packages/bitcore-node/src/routes/api/tx.ts +++ b/packages/bitcore-node/src/routes/api/tx.ts @@ -4,7 +4,7 @@ import { ICoin } from '../../models/coin'; import { ITransaction } from '../../models/transaction'; import { ChainStateProvider } from '../../providers/chain-state'; import { StreamTransactionsParams } from '../../types/namespaces/ChainStateProvider'; -import { respondWithError, streamJsonArray } from '../apiUtils'; +import { logStreamFailure, respondWithError, streamJsonArray } from '../apiUtils'; import { CacheTimes, SetCache } from '../middleware'; const router = Router({ mergeParams: true }); @@ -36,7 +36,7 @@ router.get('/', async function(req: Request, res: Response) { const stream = await ChainStateProvider.streamTransactions(payload); const result = await streamJsonArray(stream, req, res); if (!result.success) { - logger.error('Error mid-stream (streamTransactions): %o', result.error?.log || result.error); + logStreamFailure(result, 'streamTransactions'); } return; } catch (err: any) { diff --git a/packages/bitcore-node/src/routes/api/wallet.ts b/packages/bitcore-node/src/routes/api/wallet.ts index d58d260cb45..22ea5212ec0 100644 --- a/packages/bitcore-node/src/routes/api/wallet.ts +++ b/packages/bitcore-node/src/routes/api/wallet.ts @@ -5,7 +5,7 @@ import logger from '../../logger'; import { ChainStateProvider } from '../../providers/chain-state'; import { StreamWalletAddressesParams } from '../../types/namespaces/ChainStateProvider'; import { Auth, AuthenticatedRequest } from '../../utils/auth'; -import { respondWithError, streamJsonArray } from '../apiUtils'; +import { logStreamFailure, respondWithError, streamJsonArray } from '../apiUtils'; const router = Router({ mergeParams: true }); @@ -74,7 +74,7 @@ router.get('/:pubKey/addresses', Auth.authenticateMiddleware, async (req: Authen const stream = await ChainStateProvider.streamWalletAddresses(payload); const result = await streamJsonArray(stream, req, res); if (!result.success) { - logger.error('Error mid-stream (streamWalletAddresses): %o', result.error?.log || result.error); + logStreamFailure(result, 'streamWalletAddresses'); } return; } catch (err: any) { @@ -155,7 +155,7 @@ router.get('/:pubKey/transactions', Auth.authenticateMiddleware, async (req: Aut }); const result = await streamJsonArray(stream, req, res); if (!result.success) { - logger.error('Error mid-stream (streamWalletTransactions): %o', result.error?.log || result.error); + logStreamFailure(result, 'streamWalletTransactions'); } return; } catch (err: any) { @@ -210,7 +210,7 @@ router.get('/:pubKey/utxos', Auth.authenticateMiddleware, async (req: Authentica }); const result = await streamJsonArray(stream, req, res); if (!result.success) { - logger.error('Error mid-stream (streamWalletUtxos): %o', result.error?.log || result.error); + logStreamFailure(result, 'streamWalletUtxos'); } return; } catch (err: any) { diff --git a/packages/bitcore-node/src/routes/apiUtils.ts b/packages/bitcore-node/src/routes/apiUtils.ts index 151c05481d7..7ce2947b5e8 100644 --- a/packages/bitcore-node/src/routes/apiUtils.ts +++ b/packages/bitcore-node/src/routes/apiUtils.ts @@ -1,6 +1,8 @@ import { Readable } from 'stream'; import { Request, Response } from 'express'; +import logger from '../logger'; import { AdapterError, AdapterErrorCode, AllProvidersUnavailableError } from '../providers/chain-state/external/adapters/errors'; +import { jsonStringify } from '../utils'; export function respondWithError(res: Response, err: any) { if (err instanceof AllProvidersUnavailableError) { @@ -16,9 +18,51 @@ export interface StreamJsonArrayOpts { jsonl?: boolean; } -export interface StreamJsonArrayResult { - success: boolean; - error?: any; +/** + * Why a stream stopped short of a clean end. + * + * - `client-disconnect` req or res closed before the stream finished + * - `not-supported` upstream reported the query is unsupported before any data; `[]` was sent + * - `stream-error` stream errored mid-body; an inline error marker was appended + * - `closed-before-end` stream emitted 'close' without ever emitting 'end' or 'error' + */ +export type StreamJsonArrayFailureReason = + | 'client-disconnect' + | 'not-supported' + | 'stream-error' + | 'closed-before-end'; + +export type StreamJsonArrayResult = + | { success: true } + | { success: false; reason: StreamJsonArrayFailureReason; error?: any }; + +// One log message per reason, kept out of the result shape so they can't drift per call site. +const FAILURE_MESSAGES: Record = { + 'client-disconnect': 'Client disconnected mid-stream', + 'not-supported': 'Upstream reported the query is not supported', + 'stream-error': 'Error mid-stream', + 'closed-before-end': 'Stream closed before end', +}; + +/** + * Log a non-success streamJsonArray() result. Disconnects log at info: a single hangup is + * normal traffic, but a stalled upstream shows up as every client timing out, so the + * pattern needs to stay visible in production logs without tripping error alerting. + * + * @param result the result returned by streamJsonArray + * @param context the stream entry point, e.g. 'streamAddressTransactions' + */ +export function logStreamFailure(result: StreamJsonArrayResult, context: string) { + if (result.success) { + return; + } + const detail = result.error?.log || result.error; + const message = `${FAILURE_MESSAGES[result.reason]} (${context}): %o`; + if (result.reason === 'client-disconnect') { + logger.info(message, detail); + } else { + logger.error(message, detail); + } } /** @@ -36,6 +80,11 @@ export function streamJsonArray( ): Promise { // Auto-detect jsonl flag attached to the stream so routes stay chain-agnostic. const jsonl = opts.jsonl ?? stream.jsonl ?? false; + // A stream that died before this call already emitted 'error'/'close', so we'd never + // hear it. Reject now so the route sends a 5xx instead of hanging. + if (stream.destroyed) { + return Promise.reject(new Error('stream destroyed before piping began')); + } return new Promise((resolve, reject) => { let closed = false; let isFirst = true; @@ -45,6 +94,8 @@ export function streamJsonArray( // a client disconnect races a stream end/error or a stream 'close' event follows destroy(). const safeResolve = (result: StreamJsonArrayResult) => { if (!settled) { settled = true; resolve(result); } }; const safeReject = (err: any) => { if (!settled) { settled = true; reject(err); } }; + const fail = (reason: StreamJsonArrayFailureReason, error?: any) => + safeResolve({ success: false, reason, error }); const tearDown = () => { // close() handles mongo cursor streams; destroy() tears down piped Transform chains @@ -66,7 +117,7 @@ export function streamJsonArray( // Settle the awaiting route handler so it can fall through to its catch/finally // instead of hanging until the stream eventually emits 'close' (which may not happen // on a destroyed pipeline if upstream never settles). - safeResolve({ success: false, error: new Error('client disconnected') }); + fail('client-disconnect', new Error('client disconnected')); }; req.on('close', onAbort); @@ -75,8 +126,11 @@ export function streamJsonArray( res.type(jsonl ? 'application/x-ndjson' : 'json'); res.on('close', onAbort); - stream.on('error', (err: any) => { - if (closed) { safeResolve({ success: false, error: err }); return; } + // Named so the 'data' handler can route serialization failures through the same path + // instead of throwing out of an event handler (which becomes an uncaughtException). + const onStreamError = (err: any) => { + // closed means the promise already settled; nothing left to do + if (closed) { return; } if (err?.isAxiosError) { err.log = { url: err?.config?.url, @@ -85,11 +139,14 @@ export function streamJsonArray( data: err?.response?.data, }; } - if (err?.log?.data?.message?.includes('not supported')) { + // '[]' is only a valid body if no rows went out yet. After data, fall through to + // the mid-stream path so the array gets closed properly. + if (isFirst && err?.log?.data?.message?.includes('not supported')) { closed = true; res.write('[]'); res.end(); - return safeResolve({ success: false, error: err }); + tearDown(); + return fail('not-supported', err); } if (!isFirst) { // Headers already sent — emit inline error marker, end response, log upstream @@ -102,31 +159,37 @@ export function streamJsonArray( } res.end(); cleanup(); - return safeResolve({ success: false, error: err }); + return fail('stream-error', err); } - // Pre-data — caller can send proper 5xx status + // Pre-data: caller can send a proper 5xx. Mark closed first so a still-live stream + // (e.g. after a serialization failure) can't emit 'end' and write '[]' into a body + // the route is about to replace with an error. + cleanup(); return safeReject(err); - }); + }; + stream.on('error', onStreamError); stream.on('data', (data: any) => { if (closed) { cleanup(); return; } - if (!jsonl) { - if (isFirst) { - res.write('[\n'); - } else { - res.write(',\n'); + // Serialize before writing anything. A stringify failure must not leave a dangling + // '[\n' or ',\n' in the body, and throwing from here would unwind through the stream + // internals to uncaughtException rather than the route's catch block. + let payload = data; + if (typeof data !== 'string' && !Buffer.isBuffer(data)) { + try { + payload = jsonStringify(data); + } catch (err) { + return onStreamError(err); } } - if (isFirst) { - isFirst = false; - } - if (typeof data !== 'string' && !Buffer.isBuffer(data)) { - data = JSON.stringify(data); + if (!jsonl) { + res.write(isFirst ? '[\n' : ',\n'); } - res.write(data); + isFirst = false; + res.write(payload); }); stream.on('end', () => { @@ -144,7 +207,11 @@ export function streamJsonArray( }); // Backstop: if destroy() emits 'close' without a prior 'end' or 'error', settle the promise - // so the route handler doesn't await indefinitely on a torn-down pipeline. - stream.on('close', () => safeResolve({ success: closed, error: closed ? undefined : new Error('stream closed before end') })); + // so the route handler doesn't await indefinitely on a torn-down pipeline. When `closed` + // is set the promise was already settled in the same frame, so there is nothing to do. + stream.on('close', () => { + if (closed) return; + fail('closed-before-end', new Error('stream closed before end')); + }); }); } diff --git a/packages/bitcore-node/test/unit/services/storage.test.ts b/packages/bitcore-node/test/unit/services/storage.test.ts index fc4d2e6e0db..73e17f2b711 100644 --- a/packages/bitcore-node/test/unit/services/storage.test.ts +++ b/packages/bitcore-node/test/unit/services/storage.test.ts @@ -56,6 +56,8 @@ describe('streamJsonArray', function() { }); const result = await streamJsonArray(stream, req, res); expect(result.success).to.equal(false); + if (result.success) throw new Error('expected a failure result'); + expect(result.reason).to.equal('stream-error'); expect(writes.join('')).to.contain('"error"'); expect(writes.join('')).to.match(/,\n\{"error".*\}\n\]$/); }); @@ -76,6 +78,8 @@ describe('streamJsonArray', function() { setImmediate(() => req.emit('close')); const result = await streamJsonArray(stream, req, res); expect(result.success).to.equal(false); + if (result.success) throw new Error('expected a failure result'); + expect(result.reason).to.equal('client-disconnect'); expect(result.error?.message).to.contain('disconnected'); }); @@ -88,4 +92,78 @@ describe('streamJsonArray', function() { await streamJsonArray(stream, req, res); expect(closed).to.equal(true); }); + + it('serializes BigInt values instead of throwing', async () => { + const { req, res, writes } = fakes(); + const result = await streamJsonArray(Readable.from([{ a: 1n }], { objectMode: true }), req, res); + expect(result.success).to.equal(true); + expect(writes.join('')).to.equal('[\n{"a":"1"}\n]'); + }); + + it('rejects an unserializable first element without writing a partial array', async () => { + const { req, res, writes } = fakes(); + const circular: any = {}; + circular.self = circular; + let caught: any; + await streamJsonArray(Readable.from([circular], { objectMode: true }), req, res).catch(e => caught = e); + expect(caught).to.be.instanceOf(TypeError); + // The opening '[\n' must not be written for an element we could never serialize + expect(writes.join('')).to.equal(''); + }); + + it('closes the array on an unserializable later element', async () => { + const { req, res, writes } = fakes(); + const circular: any = {}; + circular.self = circular; + const result = await streamJsonArray(Readable.from([{ a: 1 }, circular], { objectMode: true }), req, res); + expect(result.success).to.equal(false); + if (result.success) throw new Error('expected a failure result'); + expect(result.reason).to.equal('stream-error'); + // no dangling ',\n'; the separator is only written after serialization succeeds + expect(writes.join('')).to.equal('[\n{"a":1},\n{"error": "An error occurred during data stream"}\n]'); + }); + + const notSupportedError = () => { + const err: any = new Error('not supported'); + err.log = { data: { message: 'query not supported for this chain' } }; + return err; + }; + + it('writes [] and tears down the source on a pre-data "not supported" error', async () => { + const { req, res, writes } = fakes(); + const stream = new Readable({ objectMode: true, read() {} }); + setImmediate(() => stream.emit('error', notSupportedError())); + const result = await streamJsonArray(stream, req, res); + expect(result.success).to.equal(false); + if (result.success) throw new Error('expected a failure result'); + expect(result.reason).to.equal('not-supported'); + expect(writes.join('')).to.equal('[]'); + expect(stream.destroyed).to.equal(true); + }); + + it('closes the array like any mid-stream error when "not supported" arrives after data', async () => { + const { req, res, writes } = fakes(); + const stream = new Readable({ objectMode: true, read() {} }); + setImmediate(() => { + stream.push({ a: 1 }); + setImmediate(() => stream.emit('error', notSupportedError())); + }); + const result = await streamJsonArray(stream, req, res); + expect(result.success).to.equal(false); + if (result.success) throw new Error('expected a failure result'); + expect(result.reason).to.equal('stream-error'); + // body must stay parseable JSON, not '[\n{..}[]' + expect(writes.join('')).to.equal('[\n{"a":1},\n{"error": "An error occurred during data stream"}\n]'); + }); + + it('rejects a stream that was destroyed before piping began', async () => { + const { req, res, writes } = fakes(); + const stream = new Readable({ objectMode: true, read() {} }); + stream.destroy(); + let caught: any; + await streamJsonArray(stream, req, res).catch(e => caught = e); + expect(caught).to.be.instanceOf(Error); + expect(caught.message).to.contain('destroyed'); + expect(writes.join('')).to.equal(''); + }); }); From 007b63a8c29abc167e704e0f43fdb14100946fd4 Mon Sep 17 00:00:00 2001 From: lyambo Date: Thu, 20 Aug 2026 16:05:00 -0400 Subject: [PATCH 15/19] Fix stream abort detection and teardown lifecycle Detect client aborts from the response, not the request. On current Node, close on the request only means the body was consumed, so body-parser draining a bodyless GET could destroy an active stream and truncate the response. Only a response that closes without finishing counts as an abort now; an unlistened response error is treated the same instead of throwing. Make teardown single-shot and call close() or destroy(), never both; the mongo cursor's destroy() delegates to close() without setting the destroyed flag, so both meant a double close. A close() that returns a promise is observed so a rejection on newer drivers cannot go unhandled. A stream that closes without end or error now finalizes the response instead of leaving the connection open: before any data it rejects so the route sends a 5xx, after data it appends the error marker and closes the array. Cover cursor teardown with spies on the cursor itself: gnosis closes its cursor exactly once across repeated terminal events, and cursor-style streams close once with destroy() never called. --- packages/bitcore-node/src/routes/apiUtils.ts | 61 ++++++---- .../test/integration/ethereum/csp.test.ts | 2 +- .../test/unit/models/transaction.test.ts | 27 ++++- .../test/unit/services/storage.test.ts | 110 ++++++++++++++++-- 4 files changed, 168 insertions(+), 32 deletions(-) diff --git a/packages/bitcore-node/src/routes/apiUtils.ts b/packages/bitcore-node/src/routes/apiUtils.ts index 7ce2947b5e8..1c92adca4bd 100644 --- a/packages/bitcore-node/src/routes/apiUtils.ts +++ b/packages/bitcore-node/src/routes/apiUtils.ts @@ -21,10 +21,10 @@ export interface StreamJsonArrayOpts { /** * Why a stream stopped short of a clean end. * - * - `client-disconnect` req or res closed before the stream finished + * - `client-disconnect` the response closed before it finished (client hung up) * - `not-supported` upstream reported the query is unsupported before any data; `[]` was sent * - `stream-error` stream errored mid-body; an inline error marker was appended - * - `closed-before-end` stream emitted 'close' without ever emitting 'end' or 'error' + * - `closed-before-end` stream emitted 'close' without 'end'/'error'; the body was finalized */ export type StreamJsonArrayFailureReason = | 'client-disconnect' @@ -70,18 +70,19 @@ export function logStreamFailure(result: StreamJsonArrayResult, context: string) * * - Pre-data errors reject so the route can send a proper status code * - Mid-stream errors append an inline error marker and end the response - * - Client/response disconnects destroy the stream (and call .close() if present, e.g. mongo cursor) + * - A response that closes without finishing is a client abort; it tears down the source + * (close() for cursor streams, destroy() otherwise) and settles as client-disconnect */ export function streamJsonArray( - stream: Readable & { close?: () => void; jsonl?: boolean }, - req: Request, + stream: Readable & { close?: () => void | Promise; jsonl?: boolean }, + // unused: req 'close' only signals the body was consumed; abort detection lives on res + _req: Request, res: Response, opts: StreamJsonArrayOpts = {} ): Promise { // Auto-detect jsonl flag attached to the stream so routes stay chain-agnostic. const jsonl = opts.jsonl ?? stream.jsonl ?? false; - // A stream that died before this call already emitted 'error'/'close', so we'd never - // hear it. Reject now so the route sends a 5xx instead of hanging. + // a dead stream already emitted its events; reject so the route can 5xx if (stream.destroyed) { return Promise.reject(new Error('stream destroyed before piping began')); } @@ -97,13 +98,20 @@ export function streamJsonArray( const fail = (reason: StreamJsonArrayFailureReason, error?: any) => safeResolve({ success: false, reason, error }); + let toreDown = false; const tearDown = () => { - // close() handles mongo cursor streams; destroy() tears down piped Transform chains - // so cursor-cleanup listeners hooked to the Transform's 'close' event fire eagerly on disconnect. + // once only, and close() or destroy(), never both: cursor destroy() delegates to + // close() without setting the destroyed flag, so both would double-close + if (toreDown) return; + toreDown = true; if (typeof stream.close === 'function') { - try { stream.close(); } catch { /* noop */ } - } - if (typeof stream.destroy === 'function' && !stream.destroyed) { + // close() may return a promise; keep it observed + try { + Promise.resolve(stream.close()).catch(err => logger.warn('Failed to close stream: %o', err)); + } catch (err) { + logger.warn('Failed to close stream: %o', err); + } + } else if (typeof stream.destroy === 'function' && !stream.destroyed) { try { stream.destroy(); } catch { /* noop */ } } }; @@ -114,17 +122,21 @@ export function streamJsonArray( const onAbort = () => { closed = true; tearDown(); - // Settle the awaiting route handler so it can fall through to its catch/finally - // instead of hanging until the stream eventually emits 'close' (which may not happen - // on a destroyed pipeline if upstream never settles). + // settle so the route doesn't hang on a stream that may never emit 'close' fail('client-disconnect', new Error('client disconnected')); }; - req.on('close', onAbort); // ndjson in jsonl mode so JSON-aware clients (supertest, fetch().json()) don't try to // parse a stream of newline-delimited objects as a single JSON document. res.type(jsonl ? 'application/x-ndjson' : 'json'); - res.on('close', onAbort); + // req 'close' only means the body was consumed, and res 'close' also fires after a + // normal finish; a response closed without finishing is the only real abort signal + res.on('close', () => { + if (res.writableFinished) return; + onAbort(); + }); + // unlistened res 'error' (e.g. EPIPE) would throw; treat as an abort + res.on('error', onAbort); // Named so the 'data' handler can route serialization failures through the same path // instead of throwing out of an event handler (which becomes an uncaughtException). @@ -139,8 +151,7 @@ export function streamJsonArray( data: err?.response?.data, }; } - // '[]' is only a valid body if no rows went out yet. After data, fall through to - // the mid-stream path so the array gets closed properly. + // '[]' is only valid before any rows; after data, fall through so the array gets closed if (isFirst && err?.log?.data?.message?.includes('not supported')) { closed = true; res.write('[]'); @@ -206,11 +217,17 @@ export function streamJsonArray( safeResolve({ success: true }); }); - // Backstop: if destroy() emits 'close' without a prior 'end' or 'error', settle the promise - // so the route handler doesn't await indefinitely on a torn-down pipeline. When `closed` - // is set the promise was already settled in the same frame, so there is nothing to do. + // backstop: 'close' without a prior 'end'/'error' still finalizes the response and settles stream.on('close', () => { if (closed) return; + closed = true; + if (isFirst) { + // nothing written yet; let the route 5xx + return safeReject(new Error('stream closed before end')); + } + const errMsg = '{"error": "An error occurred during data stream"}'; + res.write(jsonl ? errMsg : `,\n${errMsg}\n]`); + res.end(); fail('closed-before-end', new Error('stream closed before end')); }); }); diff --git a/packages/bitcore-node/test/integration/ethereum/csp.test.ts b/packages/bitcore-node/test/integration/ethereum/csp.test.ts index 96a1a86463b..e8a3812b318 100644 --- a/packages/bitcore-node/test/integration/ethereum/csp.test.ts +++ b/packages/bitcore-node/test/integration/ethereum/csp.test.ts @@ -405,7 +405,7 @@ describe('Ethereum API', function() { stream.destroy(); await new Promise(r => setImmediate(r)); expect(cursorCloseSpy, 'wallet-tx cursor was created').to.exist; - expect(cursorCloseSpy!.called, 'cursor.close() was called on destroy').to.eq(true); + expect(cursorCloseSpy!.callCount, 'cursor.close() ran exactly once on destroy').to.eq(1); }); }); }); diff --git a/packages/bitcore-node/test/unit/models/transaction.test.ts b/packages/bitcore-node/test/unit/models/transaction.test.ts index 9b76083f5ca..af494f7bd94 100644 --- a/packages/bitcore-node/test/unit/models/transaction.test.ts +++ b/packages/bitcore-node/test/unit/models/transaction.test.ts @@ -295,7 +295,9 @@ describe('Transaction Model', function() { await done; return rows; }; - const collectGnosisTokenHistoryRows = async ( + // stream over a stubbed cursor, separate from the collector so lifecycle tests + // can destroy it unconsumed + const buildGnosisTokenStream = async ( tx: Record, requestOverrides: { multisigContractAddress?: string; tokenAddress?: string } = {} ) => { @@ -322,7 +324,7 @@ describe('Transaction Model', function() { } as any); // streamGnosisWalletTransactions returns a jsonl stream now; the route does the - // HTTP framing, so collect the newline-delimited rows straight off the stream. + // HTTP framing, so consumers read the newline-delimited rows straight off the stream. const stream = await Gnosis.streamGnosisWalletTransactions({ chain: 'ETH', network: 'mainnet', @@ -330,7 +332,14 @@ describe('Transaction Model', function() { wallet: { _id: new ObjectId() }, args: { tokenAddress: requestOverrides.tokenAddress || gnosisBusdToken } } as any); + return { stream, cursor, findQuery }; + }; + const collectGnosisTokenHistoryRows = async ( + tx: Record, + requestOverrides: { multisigContractAddress?: string; tokenAddress?: string } = {} + ) => { + const { stream, findQuery } = await buildGnosisTokenStream(tx, requestOverrides); const rows = new Array(); await new Promise((resolve, reject) => { stream @@ -466,6 +475,20 @@ describe('Transaction Model', function() { expect(rows.map(row => row.satoshis)).to.deep.equal(['100', '200']); }); + it('should close the mongo cursor once when the gnosis stream is destroyed', async () => { + const { stream, cursor } = await buildGnosisTokenStream(gnosisTokenTx({ + effects: [gnosisTokenEffect({ amount: '100' })] + })); + stream.destroy(); + await new Promise(r => setImmediate(r)); + expect(cursor.close.callCount).to.equal(1); + // repeated terminal events must not re-close the cursor + stream.destroy(); + stream.emit('close'); + await new Promise(r => setImmediate(r)); + expect(cursor.close.callCount).to.equal(1); + }); + it('should not emit Gnosis token history rows for failed ERC20 sends', async () => { const { rows } = await collectGnosisTokenHistoryRows(gnosisTokenTx({ receipt: { status: false } diff --git a/packages/bitcore-node/test/unit/services/storage.test.ts b/packages/bitcore-node/test/unit/services/storage.test.ts index 73e17f2b711..11e8053e84f 100644 --- a/packages/bitcore-node/test/unit/services/storage.test.ts +++ b/packages/bitcore-node/test/unit/services/storage.test.ts @@ -1,6 +1,8 @@ import { expect } from 'chai'; import { EventEmitter } from 'events'; +import sinon from 'sinon'; import { Readable } from 'stream'; +import logger from '../../../src/logger'; import { streamJsonArray } from '../../../src/routes/apiUtils'; import { unitAfterHelper, unitBeforeHelper } from '../../helpers/unit'; @@ -19,9 +21,10 @@ describe('streamJsonArray', function() { const req = new EventEmitter() as any; const writes: string[] = []; const res = Object.assign(new EventEmitter(), { + writableFinished: false, type: () => res, write: (chunk: any) => { writes.push(typeof chunk === 'string' ? chunk : chunk.toString()); return true; }, - end: () => { (res as any).ended = true; }, + end: () => { res.ended = true; res.writableFinished = true; }, }) as any; return { req, res, writes }; } @@ -75,7 +78,7 @@ describe('streamJsonArray', function() { it('settles the promise on client disconnect', async () => { const { req, res } = fakes(); const stream = new Readable({ objectMode: true, read() {} }); - setImmediate(() => req.emit('close')); + setImmediate(() => res.emit('close')); const result = await streamJsonArray(stream, req, res); expect(result.success).to.equal(false); if (result.success) throw new Error('expected a failure result'); @@ -83,14 +86,107 @@ describe('streamJsonArray', function() { expect(result.error?.message).to.contain('disconnected'); }); - it('calls .close() on cursor-style streams when the client disconnects', async () => { + it('calls close() once and skips destroy() on cursor-style streams when the client disconnects', async () => { const { req, res } = fakes(); const stream = new Readable({ objectMode: true, read() {} }) as any; - let closed = false; - stream.close = () => { closed = true; }; - setImmediate(() => req.emit('close')); + let closeCalls = 0; + let destroyed = false; + stream.close = () => { closeCalls++; }; + stream.destroy = () => { destroyed = true; }; + setImmediate(() => { + res.emit('close'); + res.emit('close'); // overlapping terminal events must not re-run teardown + }); await streamJsonArray(stream, req, res); - expect(closed).to.equal(true); + await new Promise(r => setImmediate(r)); + expect(closeCalls).to.equal(1); + expect(destroyed).to.equal(false); + }); + + it('treats a response error as a client abort and tears down the source', async () => { + const { req, res } = fakes(); + const stream = new Readable({ objectMode: true, read() {} }) as any; + let closeCalls = 0; + stream.close = () => { closeCalls++; }; + setImmediate(() => { + res.emit('error', new Error('EPIPE')); + res.emit('close'); // real responses emit close after error + }); + const result = await streamJsonArray(stream, req, res); + expect(result.success).to.equal(false); + if (result.success) throw new Error('expected a failure result'); + expect(result.reason).to.equal('client-disconnect'); + await new Promise(r => setImmediate(r)); + expect(closeCalls).to.equal(1); + }); + + it('ignores request close so body draining cannot abort the stream', async () => { + // req 'close' fires once the body is consumed; it says nothing about the connection + const { req, res, writes } = fakes(); + const stream = new Readable({ objectMode: true, read() {} }); + setImmediate(() => { + req.emit('close'); + setImmediate(() => { + stream.push({ a: 1 }); + stream.push(null); + }); + }); + const result = await streamJsonArray(stream, req, res); + expect(result.success).to.equal(true); + expect(writes.join('')).to.equal('[\n{"a":1}\n]'); + }); + + it('does not tear down when the response closes after a normal finish', async () => { + const { req, res } = fakes(); + const stream = Readable.from([{ a: 1 }], { objectMode: true }) as any; + let closeCalls = 0; + stream.close = () => { closeCalls++; }; + const result = await streamJsonArray(stream, req, res); + expect(result.success).to.equal(true); + res.emit('close'); // normal post-finish close + await new Promise(r => setImmediate(r)); + expect(closeCalls).to.equal(0); + }); + + it('observes a rejected close() promise instead of leaving it unhandled', async () => { + const warnSpy = sinon.spy(logger, 'warn'); + try { + const { req, res } = fakes(); + const stream = new Readable({ objectMode: true, read() {} }) as any; + stream.close = () => Promise.reject(new Error('session ended')); + setImmediate(() => res.emit('close')); + await streamJsonArray(stream, req, res); + await new Promise(r => setImmediate(r)); + expect(warnSpy.called).to.equal(true); + } finally { + warnSpy.restore(); + } + }); + + it('finalizes the body when the stream closes without end after data', async () => { + const { req, res, writes } = fakes(); + const stream = new Readable({ objectMode: true, read() {} }); + setImmediate(() => { + stream.push({ a: 1 }); + setImmediate(() => stream.destroy()); + }); + const result = await streamJsonArray(stream, req, res); + expect(result.success).to.equal(false); + if (result.success) throw new Error('expected a failure result'); + expect(result.reason).to.equal('closed-before-end'); + expect(res.ended).to.equal(true); + expect(writes.join('')).to.equal('[\n{"a":1},\n{"error": "An error occurred during data stream"}\n]'); + }); + + it('rejects when the stream closes without end before any data', async () => { + const { req, res, writes } = fakes(); + const stream = new Readable({ objectMode: true, read() {} }); + setImmediate(() => stream.destroy()); + let caught: any; + await streamJsonArray(stream, req, res).catch(e => caught = e); + expect(caught).to.be.instanceOf(Error); + expect(caught.message).to.contain('closed before end'); + expect(writes.join('')).to.equal(''); }); it('serializes BigInt values instead of throwing', async () => { From a05d48d4787061887e591528b04831a3e86570e1 Mon Sep 17 00:00:00 2001 From: lyambo Date: Tue, 25 Aug 2026 10:37:58 -0400 Subject: [PATCH 16/19] Fix empty preflight result and honor backpressure MultiProvider returned the preflighted stream itself when an address had no transactions, but that stream had already ended and been destroyed, so the route responded 500 instead of an empty array. Return a fresh empty stream and record provider health success, which the after-the-fact end listener also missed for this case. Honor backpressure in streamJsonArray: a full response buffer now pauses the source until drain, so a slow client throttles the cursor instead of buffering the whole result set in memory. The piped paths had this before the streaming refactor. Await the wallet transaction cursor close in the internal provider so a rejected close promise is observed, matching the other cursor teardown paths. --- .../src/modules/multiProvider/api/csp.ts | 24 +++---- .../chain-state/internal/internal.ts | 12 +++- packages/bitcore-node/src/routes/apiUtils.ts | 7 +- .../test/unit/modules/base/csp.test.ts | 65 ++++++++++++++++++- .../test/unit/services/storage.test.ts | 51 +++++++++++++++ 5 files changed, 143 insertions(+), 16 deletions(-) diff --git a/packages/bitcore-node/src/modules/multiProvider/api/csp.ts b/packages/bitcore-node/src/modules/multiProvider/api/csp.ts index 1275918e0ec..ff367d62647 100644 --- a/packages/bitcore-node/src/modules/multiProvider/api/csp.ts +++ b/packages/bitcore-node/src/modules/multiProvider/api/csp.ts @@ -250,6 +250,13 @@ export class MultiProviderEVMStateProvider extends BaseEVMStateProvider { continue; } + // empty result set: the stream already ended (and is destroyed), so hand back a + // fresh empty stream instead of a dead one + if (preflight.firstItem == null) { + provider.health.recordSuccess(); + return Readable.from([], { objectMode: true }); + } + logger.debug(`MultiProvider: ${provider.adapter.name} streaming ${address} on ${this.chain}:${network}`); txStream.on('error', (err) => { logger.warn(`MultiProvider: ${provider.adapter.name} mid-stream error for ${address}: ${err.message}`); @@ -260,18 +267,11 @@ export class MultiProviderEVMStateProvider extends BaseEVMStateProvider { txStream.on('end', () => provider.health.recordSuccess()); // PassThrough prepends the buffered first item to the stream - let outputStream: Readable = txStream; - if (preflight.firstItem) { - const wrapper = new PassThrough({ objectMode: true }); - wrapper.write(preflight.firstItem); - txStream.resume(); - txStream.pipe(wrapper); - outputStream = wrapper; - } else { - txStream.resume(); - } - - return outputStream; + const wrapper = new PassThrough({ objectMode: true }); + wrapper.write(preflight.firstItem); + txStream.resume(); + txStream.pipe(wrapper); + return wrapper; } catch (error) { if (error instanceof AdapterError && (error as AdapterError).code === AdapterErrorCode.INVALID_REQUEST) throw error; // 400 — no failover provider.health.recordFailure(error as Error); diff --git a/packages/bitcore-node/src/providers/chain-state/internal/internal.ts b/packages/bitcore-node/src/providers/chain-state/internal/internal.ts index b11975f5602..ba73826dad5 100644 --- a/packages/bitcore-node/src/providers/chain-state/internal/internal.ts +++ b/packages/bitcore-node/src/providers/chain-state/internal/internal.ts @@ -3,6 +3,7 @@ import { Transform } from 'stream'; import { Validation } from '@bitpay-labs/crypto-wallet-core'; import { LRUCache } from 'lru-cache'; import { LoggifyClass } from '../../../decorators/Loggify'; +import logger from '../../../logger'; import { BitcoinBlockStorage, type IBtcBlock } from '../../../models/block'; import { CacheStorage } from '../../../models/cache'; import { CoinStorage, type ICoin } from '../../../models/coin'; @@ -14,7 +15,7 @@ import { RPC } from '../../../rpc'; import { Config } from '../../../services/config'; import { Storage } from '../../../services/storage'; import { type CoinJSON, SpentHeightIndicators } from '../../../types/Coin'; -import { normalizeChainNetwork } from '../../../utils'; +import { disposeOnce, normalizeChainNetwork } from '../../../utils'; import { StringifyJsonStream } from '../../../utils/jsonStream'; import { ListTransactionsStream } from './transforms'; import type { MongoBound } from '../../../models/base'; @@ -431,7 +432,14 @@ export class InternalStateProvider implements IChainStateService { .addCursorFlag('noCursorTimeout', true); const listTransactionsStream: any = transactionStream.pipe(new this.WalletStreamTransform(wallet)); listTransactionsStream.jsonl = true; - listTransactionsStream.on('close', () => { try { transactionStream.close(); } catch { /* noop */ } }); + // close() resolves a promise; disposeOnce keeps it observed and single-shot + const closeCursor = disposeOnce( + async () => { await transactionStream.close(); }, + err => logger.warn(`Failed to close ${this.chain} wallet transaction cursor: %o`, err) + ); + listTransactionsStream.on('close', closeCursor); + listTransactionsStream.on('end', closeCursor); + listTransactionsStream.on('error', closeCursor); return listTransactionsStream; } diff --git a/packages/bitcore-node/src/routes/apiUtils.ts b/packages/bitcore-node/src/routes/apiUtils.ts index 1c92adca4bd..7858776fcea 100644 --- a/packages/bitcore-node/src/routes/apiUtils.ts +++ b/packages/bitcore-node/src/routes/apiUtils.ts @@ -200,7 +200,12 @@ export function streamJsonArray( res.write(isFirst ? '[\n' : ',\n'); } isFirst = false; - res.write(payload); + // honor backpressure so a slow client throttles the source instead of buffering the + // whole result set in the response + if (!res.write(payload) && !closed) { + stream.pause(); + res.once('drain', () => { if (!closed) stream.resume(); }); + } }); stream.on('end', () => { diff --git a/packages/bitcore-node/test/unit/modules/base/csp.test.ts b/packages/bitcore-node/test/unit/modules/base/csp.test.ts index 37aa6b6d5aa..009ef45344c 100644 --- a/packages/bitcore-node/test/unit/modules/base/csp.test.ts +++ b/packages/bitcore-node/test/unit/modules/base/csp.test.ts @@ -1,5 +1,7 @@ import { expect } from 'chai'; +import { EventEmitter } from 'events'; import * as sinon from 'sinon'; +import { Readable } from 'stream'; import { CryptoRpc } from '@bitpay-labs/crypto-rpc'; import { EthDater } from '../../../../src/utils/ethDater'; import { MultiProviderEVMStateProvider } from '../../../../src/modules/multiProvider/api/csp'; @@ -7,6 +9,7 @@ import { MoralisStateProvider } from '../../../../src/modules/moralis/api/csp'; import { CacheStorage } from '../../../../src/models/cache'; import { BaseEVMStateProvider } from '../../../../src/providers/chain-state/evm/api/csp'; import { EVMBlockStorage } from '../../../../src/providers/chain-state/evm/models/block'; +import { streamJsonArray } from '../../../../src/routes/apiUtils'; import { Config } from '../../../../src/services/config'; @@ -485,6 +488,66 @@ describe('MultiProviderEVMStateProvider: streamBlocks and _getBlocks', function( }); }); +describe('MultiProviderEVMStateProvider: address transaction streaming', function() { + let cfgStub: sinon.SinonStub; + + before(function() { + cfgStub = sinon.stub(Config, 'get').returns({ chains: { ETH: {} } } as any); + (BaseEVMStateProvider as any).rpcInitialized = { ETH: true }; + }); + + after(function() { + cfgStub.restore(); + }); + + it('returns an empty JSON array when provider preflight finds no transactions', async function() { + const provider = new MultiProviderEVMStateProvider('ETH'); + const upstream = Readable.from([], { objectMode: true }); + const health = { + isAvailable: sinon.stub().returns(true), + recordSuccess: sinon.stub(), + recordFailure: sinon.stub() + }; + const adapter = { + name: 'fake', + streamAddressTransactions: sinon.stub().returns(upstream) + }; + + (provider as any).providersByNetwork = new Map([ + ['mainnet', [{ adapter, health, priority: 1 }]] + ]); + (provider as any).getChainId = sinon.stub().resolves(1n); + + const stream = await (provider as any)._buildAddressTransactionsStream({ + chain: 'ETH', + network: 'mainnet', + address: '0xaddress', + args: {} + }); + + const req = new EventEmitter() as any; + const writes: string[] = []; + const res = Object.assign(new EventEmitter(), { + writableFinished: false, + type: () => res, + write: (chunk: any) => { + writes.push(String(chunk)); + return true; + }, + end: () => { + res.writableFinished = true; + } + }) as any; + + const result = await streamJsonArray(stream, req, res); + + expect(result).to.deep.equal({ success: true }); + expect(writes.join('')).to.equal('[]'); + expect(health.recordFailure.called).to.equal(false); + expect(health.recordSuccess.calledOnce).to.equal(true); + }); +}); + describe('MultiProviderEVMStateProvider: _buildWalletTransactionsStream tokenAddress routing', function() { let cfgStub: sinon.SinonStub; before(function() { @@ -574,4 +637,4 @@ describe('MultiProviderEVMStateProvider: _verifyBlockBeforeDate EthDater fallbac expect((provider as any).daters.size).to.equal(1); }); -}); \ No newline at end of file +}); diff --git a/packages/bitcore-node/test/unit/services/storage.test.ts b/packages/bitcore-node/test/unit/services/storage.test.ts index 11e8053e84f..24967cef686 100644 --- a/packages/bitcore-node/test/unit/services/storage.test.ts +++ b/packages/bitcore-node/test/unit/services/storage.test.ts @@ -189,6 +189,57 @@ describe('streamJsonArray', function() { expect(writes.join('')).to.equal(''); }); + it('pauses the source when the response buffer is full and resumes on drain', async () => { + const { req, res, writes } = fakes(); + // reject writes until drain + res.write = (chunk: any) => { writes.push(String(chunk)); return false; }; + const stream = new Readable({ objectMode: true, read() {} }); + const promise = streamJsonArray(stream, req, res); + // count after attach; on('data') itself calls resume() + let paused = 0; + let resumed = 0; + const origPause = stream.pause.bind(stream); + const origResume = stream.resume.bind(stream); + stream.pause = () => { paused++; return origPause(); }; + stream.resume = () => { resumed++; return origResume(); }; + setImmediate(() => { + stream.push({ a: 1 }); + setImmediate(() => { + expect(paused).to.equal(1); + expect(resumed).to.equal(0); + res.write = (chunk: any) => { writes.push(String(chunk)); return true; }; + res.emit('drain'); + expect(resumed).to.equal(1); + stream.push({ a: 2 }); + stream.push(null); + }); + }); + const result = await promise; + expect(result.success).to.equal(true); + expect(writes.join('')).to.equal('[\n{"a":1},\n{"a":2}\n]'); + }); + + it('does not resume the source on drain after the response settled', async () => { + const { req, res } = fakes(); + res.write = () => false; + const stream = new Readable({ objectMode: true, read() {} }); + const promise = streamJsonArray(stream, req, res); + // count after attach; on('data') itself calls resume() + let resumed = 0; + const origResume = stream.resume.bind(stream); + stream.resume = () => { resumed++; return origResume(); }; + setImmediate(() => { + stream.push({ a: 1 }); + setImmediate(() => { + res.emit('close'); // client disconnects while paused + res.emit('drain'); + expect(resumed).to.equal(0); + }); + }); + const result = await promise; + expect(result.success).to.equal(false); + }); + it('serializes BigInt values instead of throwing', async () => { const { req, res, writes } = fakes(); const result = await streamJsonArray(Readable.from([{ a: 1n }], { objectMode: true }), req, res); From 91b34e423d24fbd76f2d89eddc6f72267ff10158 Mon Sep 17 00:00:00 2001 From: lyambo Date: Tue, 25 Aug 2026 16:12:02 -0400 Subject: [PATCH 17/19] Surface cursor errors on wallet transaction streams pipe() does not forward source errors, so a mongo cursor failure mid-stream left the piped transform silent: the route awaited a stream that would never end or error, the request hung, and the noCursorTimeout cursor leaked. The cursor error emit also had no listener, which risks crashing the process. Handle cursor errors explicitly in the internal and gnosis wallet transaction streams: close the cursor and destroy the returned stream with the error so callers observe the failure. Install the handlers before piping starts. Cover both paths with tests: cursor errors propagate to the returned stream, the stream is destroyed, and the cursor closes exactly once. --- .../providers/chain-state/evm/api/gnosis.ts | 8 ++ .../chain-state/internal/internal.ts | 12 ++- .../test/unit/models/transaction.test.ts | 14 +++ .../providers/chain-state/internal.test.ts | 92 +++++++++++++++++++ 4 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 packages/bitcore-node/test/unit/providers/chain-state/internal.test.ts diff --git a/packages/bitcore-node/src/providers/chain-state/evm/api/gnosis.ts b/packages/bitcore-node/src/providers/chain-state/evm/api/gnosis.ts index 71c63f863b6..0b2dd39688b 100644 --- a/packages/bitcore-node/src/providers/chain-state/evm/api/gnosis.ts +++ b/packages/bitcore-node/src/providers/chain-state/evm/api/gnosis.ts @@ -235,6 +235,14 @@ export class GnosisApi { finalStream.on('close', cleanupCursor); finalStream.on('end', cleanupCursor); finalStream.on('error', cleanupCursor); + // pipe() does not forward source errors; surface cursor failures on the returned + // stream so the caller settles instead of hanging + cursor.on('error', (err: any) => { + cleanupCursor(); + if (!finalStream.destroyed) { + finalStream.destroy(err); + } + }); return finalStream; } } diff --git a/packages/bitcore-node/src/providers/chain-state/internal/internal.ts b/packages/bitcore-node/src/providers/chain-state/internal/internal.ts index ba73826dad5..724844ba2e8 100644 --- a/packages/bitcore-node/src/providers/chain-state/internal/internal.ts +++ b/packages/bitcore-node/src/providers/chain-state/internal/internal.ts @@ -430,7 +430,7 @@ export class InternalStateProvider implements IChainStateService { .find(query) .sort({ blockTimeNormalized: 1 }) .addCursorFlag('noCursorTimeout', true); - const listTransactionsStream: any = transactionStream.pipe(new this.WalletStreamTransform(wallet)); + const listTransactionsStream: any = new this.WalletStreamTransform(wallet); listTransactionsStream.jsonl = true; // close() resolves a promise; disposeOnce keeps it observed and single-shot const closeCursor = disposeOnce( @@ -440,6 +440,16 @@ export class InternalStateProvider implements IChainStateService { listTransactionsStream.on('close', closeCursor); listTransactionsStream.on('end', closeCursor); listTransactionsStream.on('error', closeCursor); + // pipe() does not forward source errors; surface cursor failures on the returned + // stream so the caller settles instead of hanging + transactionStream.on('error', err => { + closeCursor(); + if (!listTransactionsStream.destroyed) { + listTransactionsStream.destroy(err); + } + }); + // start flowing only after the full lifecycle contract is installed + transactionStream.pipe(listTransactionsStream); return listTransactionsStream; } diff --git a/packages/bitcore-node/test/unit/models/transaction.test.ts b/packages/bitcore-node/test/unit/models/transaction.test.ts index af494f7bd94..f09cfe72c70 100644 --- a/packages/bitcore-node/test/unit/models/transaction.test.ts +++ b/packages/bitcore-node/test/unit/models/transaction.test.ts @@ -489,6 +489,20 @@ describe('Transaction Model', function() { expect(cursor.close.callCount).to.equal(1); }); + it('should propagate cursor errors to the gnosis stream and close the cursor', async () => { + // pipe() does not forward source errors; the explicit cursor listener must + const { stream, cursor } = await buildGnosisTokenStream(gnosisTokenTx({ + effects: [gnosisTokenEffect({ amount: '100' })] + })); + const observed = new Promise(resolve => stream.on('error', resolve)); + const boom = new Error('cursor boom'); + cursor.emit('error', boom); + expect(await observed).to.equal(boom); + expect(stream.destroyed).to.equal(true); + await new Promise(r => setImmediate(r)); + expect(cursor.close.callCount).to.equal(1); + }); + it('should not emit Gnosis token history rows for failed ERC20 sends', async () => { const { rows } = await collectGnosisTokenHistoryRows(gnosisTokenTx({ receipt: { status: false } diff --git a/packages/bitcore-node/test/unit/providers/chain-state/internal.test.ts b/packages/bitcore-node/test/unit/providers/chain-state/internal.test.ts new file mode 100644 index 00000000000..ebcff2ca60b --- /dev/null +++ b/packages/bitcore-node/test/unit/providers/chain-state/internal.test.ts @@ -0,0 +1,92 @@ +import { expect } from 'chai'; +import * as sinon from 'sinon'; +import { Readable } from 'stream'; +import { InternalStateProvider } from '../../../../src/providers/chain-state/internal/internal'; +import { TransactionStorage } from '../../../../src/models/transaction'; +import logger from '../../../../src/logger'; +import { unitAfterHelper, unitBeforeHelper } from '../../../helpers/unit'; + +describe('InternalStateProvider: streamWalletTransactions cursor teardown', function() { + before(unitBeforeHelper); + after(unitAfterHelper); + + let collectionStub: sinon.SinonStub; + let warnStub: sinon.SinonStub; + let cursor: any; + let cursorClose: sinon.SinonStub; + let provider: InternalStateProvider; + + const wallet = { _id: 'wallet-id', chain: 'BTC', network: 'mainnet' } as any; + const params = { chain: 'BTC', network: 'mainnet', wallet, args: {} } as any; + + beforeEach(function() { + provider = new InternalStateProvider('BTC'); + // push-driven stand-in for the Mongo cursor; close() is the only cursor API under test + cursor = new Readable({ objectMode: true, read() {} }); + cursorClose = sinon.stub().resolves(); + cursor.close = cursorClose; + collectionStub = sinon + .stub(TransactionStorage, 'collection') + .get( + () => + ({ + find: () => ({ + sort: () => ({ + addCursorFlag: () => cursor + }) + }) + }) as any + ); + warnStub = sinon.stub(logger, 'warn'); + }); + + afterEach(function() { + collectionStub.restore(); + warnStub.restore(); + }); + + it('closes the cursor once when the stream ends, even though both end and close fire', async function() { + const stream: any = await provider.streamWalletTransactions(params); + const ended = new Promise(resolve => stream.on('end', resolve)); + // consume the stream like the route does; a paused readable never emits 'end' + stream.on('data', () => {}); + cursor.push(null); + await ended; + expect(cursorClose.callCount).to.equal(1); + }); + + it('logs a warning when cursor close() rejects instead of leaving the rejection unhandled', async function() { + cursorClose.rejects(new Error('cursor close failed')); + const stream: any = await provider.streamWalletTransactions(params); + const ended = new Promise(resolve => stream.on('end', resolve)); + stream.on('data', () => {}); + cursor.push(null); + await ended; + // dispose is async; let the rejected close() settle before asserting + await new Promise(resolve => setImmediate(resolve)); + expect(warnStub.callCount).to.equal(1); + expect(warnStub.firstCall.args[0]).to.contain('Failed to close BTC wallet transaction cursor'); + }); + + it('closes the cursor when the output stream itself errors before ending', async function() { + const stream: any = await provider.streamWalletTransactions(params); + stream.on('error', () => { + /* swallowed; only the teardown is under test */ + }); + stream.emit('error', new Error('boom')); + expect(cursorClose.callCount).to.equal(1); + }); + + it('closes the cursor and propagates the error when the underlying cursor errors mid-stream', async function() { + // Readable.pipe() does not propagate source errors to the destination. + // Without the cursor error listener, this error is unhandled by the source + // and the returned transform never receives the failure. + const stream: any = await provider.streamWalletTransactions(params); + const observedError = new Promise(resolve => stream.on('error', resolve)); + const boom = new Error('cursor boom'); + cursor.emit('error', boom); + expect(await observedError).to.equal(boom); + expect(cursorClose.callCount).to.equal(1); + expect(stream.destroyed).to.equal(true); + }); +}); From 78d92962b8b229e31268c413d0b834330ccea430 Mon Sep 17 00:00:00 2001 From: lyambo Date: Tue, 25 Aug 2026 16:58:38 -0400 Subject: [PATCH 18/19] Propagate stream errors across remaining pipe boundaries The mongo driver only emits cursor errors when a listener exists; without one it closes the cursor and ends the stream cleanly, so a mid-iteration failure on the EVM wallet path produced a 200 with a truncated body. Hook cursor errors into the transform chain so they reach the returned stream. Convert the gnosis transform chain from pipe() to eventPipe(). A mid-chain transform error re-emitted with no listeners and crashed the worker; eventPipe forwards it to the handled final stream. Destroy the multiProvider wrapper when an adapter errors after preflight. Logging alone left the response hanging on a partial array until the client timed out. End the mergeStreams destination when there are no source streams; a wallet with no addresses hung the request indefinitely. Cover each path with tests and give the cursor lifecycle tests their own sandboxes so their stubs cannot collide. --- .../src/modules/multiProvider/api/csp.ts | 8 ++- .../src/providers/chain-state/evm/api/csp.ts | 10 ++- .../providers/chain-state/evm/api/gnosis.ts | 13 ++-- .../chain-state/evm/api/multisigTransform.ts | 4 +- .../chain-state/external/streams/apiStream.ts | 5 ++ .../test/integration/ethereum/csp.test.ts | 62 ++++++++++++++----- .../test/unit/modules/base/csp.test.ts | 32 ++++++++++ .../test/unit/services/storage.test.ts | 13 +++- 8 files changed, 118 insertions(+), 29 deletions(-) diff --git a/packages/bitcore-node/src/modules/multiProvider/api/csp.ts b/packages/bitcore-node/src/modules/multiProvider/api/csp.ts index ff367d62647..2841ce502ac 100644 --- a/packages/bitcore-node/src/modules/multiProvider/api/csp.ts +++ b/packages/bitcore-node/src/modules/multiProvider/api/csp.ts @@ -258,16 +258,20 @@ export class MultiProviderEVMStateProvider extends BaseEVMStateProvider { } logger.debug(`MultiProvider: ${provider.adapter.name} streaming ${address} on ${this.chain}:${network}`); + // PassThrough prepends the buffered first item to the stream + const wrapper = new PassThrough({ objectMode: true }); txStream.on('error', (err) => { logger.warn(`MultiProvider: ${provider.adapter.name} mid-stream error for ${address}: ${err.message}`); if (err instanceof AdapterError && err.affectsHealth) { provider.health.recordFailure(err); } + // surface the failure downstream; logging alone leaves the response hanging + if (!wrapper.destroyed) { + wrapper.destroy(err); + } }); txStream.on('end', () => provider.health.recordSuccess()); - // PassThrough prepends the buffered first item to the stream - const wrapper = new PassThrough({ objectMode: true }); wrapper.write(preflight.firstItem); txStream.resume(); txStream.pipe(wrapper); diff --git a/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts b/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts index d86c2fff92e..060bf071ef3 100644 --- a/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts +++ b/packages/bitcore-node/src/providers/chain-state/evm/api/csp.ts @@ -744,7 +744,15 @@ export class BaseEVMStateProvider extends InternalStateProvider implements IChai .addCursorFlag('noCursorTimeout', true); // Pipe cursor to transform stream - transactionStream = cursor.pipe(new TransformWithEventPipe({ objectMode: true, passThrough: true })); + const cursorTransform = new TransformWithEventPipe({ objectMode: true, passThrough: true }); + // pipe() does not forward source errors, and the driver only emits them when a listener + // exists; without this a cursor failure ends the stream cleanly as a truncated success + cursor.on('error', (err: any) => { + if (!cursorTransform.destroyed) { + cursorTransform.destroy(err); + } + }); + transactionStream = cursor.pipe(cursorTransform); transactionStream = transactionStream.eventPipe(populateEffects); // For old db entries diff --git a/packages/bitcore-node/src/providers/chain-state/evm/api/gnosis.ts b/packages/bitcore-node/src/providers/chain-state/evm/api/gnosis.ts index 0b2dd39688b..2b490bf5963 100644 --- a/packages/bitcore-node/src/providers/chain-state/evm/api/gnosis.ts +++ b/packages/bitcore-node/src/providers/chain-state/evm/api/gnosis.ts @@ -1,4 +1,3 @@ -import { Readable } from 'stream'; import { Utils, Web3 } from '@bitpay-labs/crypto-wallet-core'; import { ChainStateProvider } from '../../'; import logger from '../../../../logger'; @@ -6,6 +5,7 @@ import { Config } from '../../../../services/config'; import { IEVMNetworkConfig } from '../../../../types/Config'; import { StreamWalletTransactionsParams } from '../../../../types/namespaces/ChainStateProvider'; import { disposeOnce } from '../../../../utils'; +import { TransformWithEventPipe } from '../../../../utils/streamWithEventPipe'; import { MultisigAbi } from '../abi/multisig'; import { MultisigRelatedFilterTransform } from '../api/multisigTransform'; import { PopulateEffectsTransform } from '../api/populateEffectsTransform'; @@ -202,7 +202,6 @@ export class GnosisApi { }; } - let transactionStream = new Readable({ objectMode: true }); const ethTransactionTransform = new EVMListTransactionsStream([normalizedMultisigContractAddress], tokenAddress); const EVM = getCSP(chain, network); const populateReceipt = new PopulateReceiptTransform(EVM); @@ -221,16 +220,18 @@ export class GnosisApi { err => logger.warn(`Failed to close ${chain}:${network} gnosis transaction cursor: %o`, err) ); - transactionStream = cursor.pipe(populateEffects); // For old db entries + let transactionStream: TransformWithEventPipe = cursor.pipe(populateEffects); // For old db entries if (multisigContractAddress) { const multisigTransform = new MultisigRelatedFilterTransform(normalizedMultisigContractAddress, tokenAddress); - transactionStream = transactionStream.pipe(multisigTransform); + transactionStream = transactionStream.eventPipe(multisigTransform); } + // eventPipe so a mid-chain transform error reaches finalStream instead of + // re-emitting with no listeners and crashing the worker const finalStream: any = transactionStream - .pipe(populateReceipt) - .pipe(ethTransactionTransform); + .eventPipe(populateReceipt) + .eventPipe(ethTransactionTransform); finalStream.jsonl = true; finalStream.on('close', cleanupCursor); finalStream.on('end', cleanupCursor); diff --git a/packages/bitcore-node/src/providers/chain-state/evm/api/multisigTransform.ts b/packages/bitcore-node/src/providers/chain-state/evm/api/multisigTransform.ts index 6bcfba81de5..fd5dead84c9 100644 --- a/packages/bitcore-node/src/providers/chain-state/evm/api/multisigTransform.ts +++ b/packages/bitcore-node/src/providers/chain-state/evm/api/multisigTransform.ts @@ -1,8 +1,8 @@ -import { Transform } from 'stream'; import { MongoBound } from '../../../../models/base'; +import { TransformWithEventPipe } from '../../../../utils/streamWithEventPipe'; import { IEVMTransactionInProcess } from '../types'; -export class MultisigRelatedFilterTransform extends Transform { +export class MultisigRelatedFilterTransform extends TransformWithEventPipe { constructor(private multisigContractAddress: string, private tokenAddress?: string) { super({ objectMode: true }); } diff --git a/packages/bitcore-node/src/providers/chain-state/external/streams/apiStream.ts b/packages/bitcore-node/src/providers/chain-state/external/streams/apiStream.ts index 20dce148351..cb6913bb3fe 100644 --- a/packages/bitcore-node/src/providers/chain-state/external/streams/apiStream.ts +++ b/packages/bitcore-node/src/providers/chain-state/external/streams/apiStream.ts @@ -70,6 +70,11 @@ export class ExternalApiStream extends ReadableWithEventPipe { static mergeStreams(streams: Stream[], destination: Transform): Transform { let activeStreams = streams.length; + // no sources means no 'end' events; end the destination now or it never settles + if (activeStreams === 0) { + destination.end(); + return destination; + } for (const stream of streams) { // Pipe each stream to the destination diff --git a/packages/bitcore-node/test/integration/ethereum/csp.test.ts b/packages/bitcore-node/test/integration/ethereum/csp.test.ts index e8a3812b318..fb7ab55408d 100644 --- a/packages/bitcore-node/test/integration/ethereum/csp.test.ts +++ b/packages/bitcore-node/test/integration/ethereum/csp.test.ts @@ -384,28 +384,56 @@ describe('Ethereum API', function() { await streamDexWalletTransactions(chain, network, wallet, address, web3); }); + // pins one collection wrapper (the getter returns a fresh one per access) and captures + // the real cursor so tests can assert on cursor.close() itself + const captureCursor = (box: sinon.SinonSandbox) => { + const collection = EVMTransactionStorage.collection; + const realFind = collection.find.bind(collection); + const captured: { cursor?: any; closeSpy?: sinon.SinonSpy } = {}; + box.stub(collection, 'find').callsFake((...findArgs: any[]) => { + captured.cursor = realFind(...findArgs); + captured.closeSpy = captured.closeSpy || box.spy(captured.cursor, 'close'); + return captured.cursor; + }); + box.stub(EVMTransactionStorage, 'collection').get(() => collection); + return captured; + }; + it('closes the wallet-tx cursor when the final stream is destroyed', async () => { await EVMTransactionStorage.collection.insertMany( new Array(5).fill({}).map(() => ({ chain, network, blockHeight: 1, gasPrice: 10 * 1e9, data: Buffer.from(''), from: address } as IEVMTransactionInProcess)) ); - // Spy on the real cursor's close() so this asserts the cursor is torn down, not just - // that the stream emits 'close' (destroy() always does that). - // .collection returns a fresh wrapper each access, so pin one and wrap its find(). - const collection = EVMTransactionStorage.collection; - const realFind = collection.find.bind(collection); - let cursorCloseSpy: sinon.SinonSpy | undefined; - sandbox.stub(collection, 'find').callsFake((...findArgs: any[]) => { - const cursor = realFind(...findArgs); - cursorCloseSpy = cursorCloseSpy || sandbox.spy(cursor, 'close'); - return cursor; - }); - sandbox.stub(EVMTransactionStorage, 'collection').get(() => collection); + const box = sinon.createSandbox(); + try { + const captured = captureCursor(box); + const stream: any = await ETH.streamWalletTransactions({ chain, network, wallet, args: {} } as StreamWalletTransactionsParams); + stream.destroy(); + await new Promise(r => setImmediate(r)); + expect(captured.closeSpy, 'wallet-tx cursor was created').to.exist; + expect(captured.closeSpy!.callCount, 'cursor.close() ran exactly once on destroy').to.eq(1); + } finally { + box.restore(); + } + }); - const stream: any = await ETH.streamWalletTransactions({ chain, network, wallet, args: {} } as StreamWalletTransactionsParams); - stream.destroy(); - await new Promise(r => setImmediate(r)); - expect(cursorCloseSpy, 'wallet-tx cursor was created').to.exist; - expect(cursorCloseSpy!.callCount, 'cursor.close() ran exactly once on destroy').to.eq(1); + it('propagates a wallet-tx cursor error to the final stream', async () => { + await EVMTransactionStorage.collection.insertMany( + new Array(5).fill({}).map(() => ({ chain, network, blockHeight: 1, gasPrice: 10 * 1e9, data: Buffer.from(''), from: address } as IEVMTransactionInProcess)) + ); + const box = sinon.createSandbox(); + try { + const captured = captureCursor(box); + const stream: any = await ETH.streamWalletTransactions({ chain, network, wallet, args: {} } as StreamWalletTransactionsParams); + const observed = new Promise(resolve => stream.on('error', resolve)); + const boom = new Error('cursor boom'); + captured.cursor.emit('error', boom); + expect(await observed).to.eq(boom); + await new Promise(r => setImmediate(r)); + expect(stream.destroyed).to.eq(true); + expect(captured.closeSpy!.callCount, 'cursor.close() ran exactly once on error').to.eq(1); + } finally { + box.restore(); + } }); }); }); diff --git a/packages/bitcore-node/test/unit/modules/base/csp.test.ts b/packages/bitcore-node/test/unit/modules/base/csp.test.ts index 009ef45344c..f420d0fb2db 100644 --- a/packages/bitcore-node/test/unit/modules/base/csp.test.ts +++ b/packages/bitcore-node/test/unit/modules/base/csp.test.ts @@ -500,6 +500,38 @@ describe('MultiProviderEVMStateProvider: address transaction streaming', functio cfgStub.restore(); }); + it('destroys the returned stream when the adapter errors after preflight', async function() { + const provider = new MultiProviderEVMStateProvider('ETH'); + const upstream: any = new Readable({ objectMode: true, read() {} }); + const health = { + isAvailable: sinon.stub().returns(true), + recordSuccess: sinon.stub(), + recordFailure: sinon.stub() + }; + const adapter = { + name: 'fake', + streamAddressTransactions: sinon.stub().returns(upstream) + }; + (provider as any).providersByNetwork = new Map([ + ['mainnet', [{ adapter, health, priority: 1 }]] + ]); + (provider as any).getChainId = sinon.stub().resolves(1n); + + setImmediate(() => upstream.push({ txid: '0x1' })); // preflight commits to this provider + const stream = await (provider as any)._buildAddressTransactionsStream({ + chain: 'ETH', + network: 'mainnet', + address: '0xaddress', + args: {} + }); + + const observed = new Promise(resolve => stream.on('error', resolve)); + const boom = new Error('page 2 failed'); + upstream.emit('error', boom); + expect(await observed).to.equal(boom); + expect(stream.destroyed).to.equal(true); + }); + it('returns an empty JSON array when provider preflight finds no transactions', async function() { const provider = new MultiProviderEVMStateProvider('ETH'); const upstream = Readable.from([], { objectMode: true }); diff --git a/packages/bitcore-node/test/unit/services/storage.test.ts b/packages/bitcore-node/test/unit/services/storage.test.ts index 24967cef686..b1caebd1667 100644 --- a/packages/bitcore-node/test/unit/services/storage.test.ts +++ b/packages/bitcore-node/test/unit/services/storage.test.ts @@ -1,8 +1,9 @@ import { expect } from 'chai'; import { EventEmitter } from 'events'; import sinon from 'sinon'; -import { Readable } from 'stream'; +import { PassThrough, Readable } from 'stream'; import logger from '../../../src/logger'; +import { ExternalApiStream } from '../../../src/providers/chain-state/external/streams/apiStream'; import { streamJsonArray } from '../../../src/routes/apiUtils'; import { unitAfterHelper, unitBeforeHelper } from '../../helpers/unit'; @@ -15,6 +16,16 @@ describe('Storage Service', function() { }); }); +describe('mergeStreams', function() { + it('ends the destination immediately when there are no source streams', async () => { + const dest = new PassThrough({ objectMode: true }); + const ended = new Promise(resolve => dest.on('end', resolve)); + dest.on('data', () => {}); + ExternalApiStream.mergeStreams([], dest); + await ended; + }); +}); + describe('streamJsonArray', function() { // Minimal req/res stand-ins: req only needs 'close', res captures writes and exposes 'close'. function fakes() { From 65cc0c404ad91d976b38241402327d487ad21911 Mon Sep 17 00:00:00 2001 From: lyambo Date: Wed, 26 Aug 2026 12:17:47 -0400 Subject: [PATCH 19/19] Test mid-chain transform errors on the gnosis stream The eventPipe conversion in the gnosis chain had no test that would catch a regression back to pipe(), which re-emits a transform error with no listeners and crashes the worker. Capture the multisig transform built inside streamGnosisWalletTransactions by wrapping its constructor, emit an error on it mid-chain, and assert the error reaches the returned stream and the cursor still closes. --- .../test/unit/models/transaction.test.ts | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/packages/bitcore-node/test/unit/models/transaction.test.ts b/packages/bitcore-node/test/unit/models/transaction.test.ts index f09cfe72c70..aa6e7691fa2 100644 --- a/packages/bitcore-node/test/unit/models/transaction.test.ts +++ b/packages/bitcore-node/test/unit/models/transaction.test.ts @@ -9,6 +9,7 @@ import { ChainStateProvider } from '../../../src/providers/chain-state'; import { Gnosis } from '../../../src/providers/chain-state/evm/api/gnosis'; import { Erc20RelatedFilterTransform } from '../../../src/providers/chain-state/evm/api/erc20Transform'; import { MultisigRelatedFilterTransform } from '../../../src/providers/chain-state/evm/api/multisigTransform'; +import * as MultisigTransformModule from '../../../src/providers/chain-state/evm/api/multisigTransform'; import { Config } from '../../../src/services/config'; import { EVMListTransactionsStream } from '../../../src/providers/chain-state/evm/api/transform'; import { EVMTransactionStorage } from '../../../src/providers/chain-state/evm/models/transaction'; @@ -503,6 +504,34 @@ describe('Transaction Model', function() { expect(cursor.close.callCount).to.equal(1); }); + it('should propagate a mid-chain transform error to the gnosis stream and close the cursor', async () => { + // pipe() re-emits a mid-chain transform error with no listener, which crashes the + // worker; eventPipe wires each stage to forward into the next so it reaches + // finalStream instead. Capture the real multisigTransform instance mid-chain (it's + // constructed inside streamGnosisWalletTransactions) so we can trigger its error + // directly, the way an unexpected per-item failure would. + const RealMultisigTransform = MultisigTransformModule.MultisigRelatedFilterTransform; + let capturedMultisigTransform: MultisigRelatedFilterTransform | undefined; + sandbox.stub(MultisigTransformModule, 'MultisigRelatedFilterTransform').callsFake((...args: any[]) => { + capturedMultisigTransform = new (RealMultisigTransform as any)(...args); + return capturedMultisigTransform; + }); + + const { stream, cursor } = await buildGnosisTokenStream(gnosisTokenTx({ + effects: [gnosisTokenEffect({ amount: '100' })] + })); + const observed = new Promise(resolve => stream.on('error', resolve)); + // let the pipeline construct and wire up before triggering the failure + await new Promise(r => setImmediate(r)); + expect(capturedMultisigTransform, 'multisig transform was constructed').to.exist; + + const boom = new Error('multisig transform boom'); + capturedMultisigTransform!.emit('error', boom); + expect(await observed).to.equal(boom); + await new Promise(r => setImmediate(r)); + expect(cursor.close.callCount).to.equal(1); + }); + it('should not emit Gnosis token history rows for failed ERC20 sends', async () => { const { rows } = await collectGnosisTokenHistoryRows(gnosisTokenTx({ receipt: { status: false }