commit a8d2a5dcf7ab12ad21b4f401be456bc293ddb937 Author: Tenere CI Date: Sun Jul 19 13:44:11 2026 -0700 initial import diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml new file mode 100644 index 0000000..0ca1503 --- /dev/null +++ b/.gitea/workflows/ci.yaml @@ -0,0 +1,28 @@ +name: CI + +on: + push: + branches: [master] + tags: ['v*'] + pull_request: + branches: [master] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + node: [18, 20, 22] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + - name: Auth for Gitea npm registry + run: | + echo "@${{ github.repository_owner }}:registry=https://hub.kl1.tenere.ai/api/packages/${{ github.repository_owner }}/npm/" > .npmrc + echo "//hub.kl1.tenere.ai/api/packages/${{ github.repository_owner }}/npm/:_authToken=${{ secrets.PACKAGE_TOKEN }}" >> .npmrc + - run: npm install + - run: npm test + - run: npm run bench + if: matrix.node == 22 diff --git a/.npmrc b/.npmrc new file mode 100644 index 0000000..a23de45 --- /dev/null +++ b/.npmrc @@ -0,0 +1,2 @@ +@push-stream-std:registry=https://hub.kl1.tenere.ai/api/packages/push-stream-std/npm/ +//hub.kl1.tenere.ai/api/packages/push-stream-std/npm/:_authToken=${PACKAGE_TOKEN} diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e5bd26f --- /dev/null +++ b/LICENSE @@ -0,0 +1,4 @@ +Tenere Labs Inc. — All Rights Reserved. + +PROPRIETARY AND CONFIDENTIAL. +Source-available for auditing and inspection only. diff --git a/README.md b/README.md new file mode 100644 index 0000000..5fe0df1 --- /dev/null +++ b/README.md @@ -0,0 +1,85 @@ +# stream-to-push-stream + +Convert native Node.js streams to push-stream source, sink, and duplex interfaces. + +Published as `@push-stream-std/stream-to-push-stream`. + +## Synopsis + +```js +import { source, sink, duplex } from '@push-stream-std/stream-to-push-stream'; +import { Readable, Writable, Duplex } from 'node:stream'; +import { PassThrough } from 'node:stream'; + +// Convert a Readable to a push-stream source +const readable = Readable.from(['a', 'b', 'c']); +const pushSource = source(readable); + +const pushSink = { + paused: false, + write(v) { console.log(v); }, + end() { console.log('done'); } +}; + +pushSource.pipe(pushSink); +// Output: a, b, c, done + +// Convert a Writable to a push-stream sink +const writable = new PassThrough(); +const pushSink2 = sink(writable); + +// Convert a Duplex to a push-stream source +const duplexStream = new PassThrough(); +const pushSource2 = duplex(duplexStream); +``` + +## Install + +```bash +npm config set @push-stream-std:registry https://hub.kl1.tenere.ai/api/packages/push-stream-std/npm/ +npm config set -- //hub.kl1.tenere.ai/api/packages/push-stream-std/npm/:_authToken "$PACKAGE_TOKEN" +npm install @push-stream-std/stream-to-push-stream +``` + +## Description + +Utilities for bridging between the Node.js `stream` module (Readable, Writable, Duplex) and the push-stream protocol. PushSource wraps a Readable, listening to `data`/`end`/`error` events and implementing `pipe()`, `resume()`, and `abort()`. PushSink wraps a Writable, translating `write()` calls to `writable.write()` with `drain`-based backpressure and `end()` to `writable.end()`. PushDuplex wraps a Duplex stream as a combined source/sink for bidirectional push-stream pipelines. + +## API + +### `source(readableStream)` + +Converts a Node.js `Readable` to a push-stream source. + +- **readableStream** `Readable` – A Node.js readable stream. +- **Returns** A push-stream source with `pipe()`, `resume()`, `abort()`, `paused`, and `ended`. + +### `sink(writableStream)` + +Converts a Node.js `Writable` to a push-stream sink. + +- **writableStream** `Writable` – A Node.js writable stream. +- **Returns** A push-stream sink with `pipe()`, `write()`, `end()`, `abort()`, `paused`, and `ended`. + +### `duplex(duplexStream)` + +Wraps a Node.js `Duplex` stream as a push-stream source. + +- **duplexStream** `Duplex` – A Node.js duplex stream. +- **Returns** A push-stream source with `pipe()`, `resume()`, `abort()`, `paused`, and `ended`. + +## Tests + +```bash +npm test +``` + +Tests cover readable-to-source data flow and end handling, writable-to-sink write and backpressure, duplex stream wrapping, error propagation, and abort cleanup. + +## Requirements + +Node.js 18 or later. ESM only. + +## License + +MIT diff --git a/bench/index.js b/bench/index.js new file mode 100644 index 0000000..0ebdc7d --- /dev/null +++ b/bench/index.js @@ -0,0 +1,211 @@ +import { Readable, Writable, Duplex } from 'stream'; +import { source, sink, duplex } from '../index.js'; + +const CHUNK_SIZE = 16384; +const CHUNK_COUNT = 100000; + +function createBenchmarkReadable() { + const data = Buffer.alloc(CHUNK_SIZE, 'x'); + let count = 0; + return new Readable({ + highWaterMark: 1024 * 1024, + read() { + if (count < CHUNK_COUNT) { + count++; + this.push(data); + } else { + this.push(null); + } + } + }); +} + +function createBenchmarkWritable() { + let bytesWritten = 0; + const w = new Writable({ + highWaterMark: 1024 * 1024, + write(chunk, encoding, callback) { + bytesWritten += chunk.length; + callback(); + } + }); + return w; +} + +async function benchmarkSource() { + const readable = createBenchmarkReadable(); + const pushSrc = source(readable); + const received = []; + let byteCount = 0; + + const start = process.hrtime.bigint(); + + await new Promise((resolve, reject) => { + pushSrc.pipe({ + write(data) { + byteCount += data.length; + }, + end(err) { + if (err) return reject(err); + resolve(); + }, + get paused() { return false; }, + set paused(v) {} + }); + }); + + const end = process.hrtime.bigint(); + const duration = Number(end - start) / 1e9; + const throughput = (byteCount / duration / 1024 / 1024).toFixed(2); + + return { duration, throughput, bytes: byteCount }; +} + +async function benchmarkSink() { + const writable = createBenchmarkWritable(); + const pushSnk = sink(writable); + const data = Buffer.alloc(CHUNK_SIZE, 'x'); + + const start = process.hrtime.bigint(); + + for (let i = 0; i < CHUNK_COUNT; i++) { + pushSnk.write(data); + } + pushSnk.end(); + + await new Promise((resolve) => { + writable.on('finish', resolve); + }); + + const end = process.hrtime.bigint(); + const duration = Number(end - start) / 1e9; + const bytes = CHUNK_COUNT * CHUNK_SIZE; + const throughput = (bytes / duration / 1024 / 1024).toFixed(2); + + return { duration, throughput, bytes }; +} + +async function benchmarkDuplex() { + const data = Buffer.alloc(CHUNK_SIZE, 'x'); + let count = 0; + let byteCount = 0; + + const duplexStream = new Duplex({ + highWaterMark: 1024 * 1024, + read() { + if (count < CHUNK_COUNT) { + count++; + this.push(data); + } else { + this.push(null); + } + }, + write(chunk, encoding, callback) { + callback(); + } + }); + + const pushDup = duplex(duplexStream); + + const start = process.hrtime.bigint(); + + await new Promise((resolve, reject) => { + pushDup.pipe({ + write(data) { + byteCount += data.length; + }, + end(err) { + if (err) return reject(err); + resolve(); + }, + get paused() { return false; }, + set paused(v) {} + }); + }); + + const end = process.hrtime.bigint(); + const duration = Number(end - start) / 1e9; + const throughput = (byteCount / duration / 1024 / 1024).toFixed(2); + + return { duration, throughput, bytes: byteCount }; +} + +async function benchmarkObjectMode() { + const arr = Array.from({ length: CHUNK_COUNT }, (_, i) => ({ id: i, value: Math.random() })); + let index = 0; + + const readable = new Readable({ + objectMode: true, + highWaterMark: 1024, + read() { + if (index < arr.length) { + this.push(arr[index++]); + } else { + this.push(null); + } + } + }); + + const pushSrc = source(readable); + let itemCount = 0; + + const start = process.hrtime.bigint(); + + await new Promise((resolve, reject) => { + pushSrc.pipe({ + write(data) { + itemCount++; + }, + end(err) { + if (err) return reject(err); + resolve(); + }, + get paused() { return false; }, + set paused(v) {} + }); + }); + + const end = process.hrtime.bigint(); + const duration = Number(end - start) / 1e9; + const throughput = (itemCount / duration).toFixed(0); + + return { duration, throughput, items: itemCount }; +} + +async function runBenchmarks() { + console.log('stream-to-push-stream benchmarks'); + console.log('================================'); + console.log(`Chunk size: ${CHUNK_SIZE} bytes`); + console.log(`Chunk count: ${CHUNK_COUNT}`); + console.log(); + + console.log('Running benchmarks...\n'); + + console.log('1. Source (Readable -> Push-Source):'); + const sourceResult = await benchmarkSource(); + console.log(` Duration: ${sourceResult.duration.toFixed(3)}s`); + console.log(` Throughput: ${sourceResult.throughput} MB/s`); + console.log(); + + console.log('2. Sink (Writable -> Push-Sink):'); + const sinkResult = await benchmarkSink(); + console.log(` Duration: ${sinkResult.duration.toFixed(3)}s`); + console.log(` Throughput: ${sinkResult.throughput} MB/s`); + console.log(); + + console.log('3. Duplex (Duplex -> Push-Duplex):'); + const duplexResult = await benchmarkDuplex(); + console.log(` Duration: ${duplexResult.duration.toFixed(3)}s`); + console.log(` Throughput: ${duplexResult.throughput} MB/s`); + console.log(); + + console.log('4. Object Mode (Readable Object -> Push-Source):'); + const objectResult = await benchmarkObjectMode(); + console.log(` Duration: ${objectResult.duration.toFixed(3)}s`); + console.log(` Throughput: ${objectResult.throughput} items/s`); + console.log(); + + console.log('Benchmark complete.'); +} + +runBenchmarks().catch(console.error); diff --git a/index.js b/index.js new file mode 100644 index 0000000..1a3d6fb --- /dev/null +++ b/index.js @@ -0,0 +1,245 @@ +import { Readable, Writable, Duplex } from 'stream'; + +class PushSource { + constructor(readable) { + this.readable = readable; + this.sink = null; + this.paused = true; + this.ended = false; + this._endedWithError = null; + + this._onData = this._onData.bind(this); + this._onEnd = this._onEnd.bind(this); + this._onError = this._onError.bind(this); + this._onResume = this._onResume.bind(this); + + readable.on('data', this._onData); + readable.on('end', this._onEnd); + readable.on('error', this._onError); + } + + pipe(sink) { + this.sink = sink; + sink.source = this; + if (!sink.paused) { + this.resume(); + } + return sink; + } + + resume() { + if (this.ended) return; + this.paused = false; + this.readable.resume(); + this._drain(); + } + + abort(err) { + if (this.ended) return; + this.ended = err || new Error('Aborted'); + this._cleanup(); + if (this.sink) { + this.sink.end(this.ended); + this.sink.source = null; + this.sink = null; + } + } + + _drain() { + // _drain is no longer needed - _onData handles all data events + // This method is kept for compatibility but does nothing + // The readable stream's 'data' event handler (_onData) is already set up in constructor + } + + _onData(data) { + if (this.ended) return; + if (this.sink) { + if (this.sink.paused) { + this.paused = true; + this.readable.pause(); + return; + } + this.sink.write(data); + this.paused = this.sink.paused; + } else { + this.paused = true; + this.readable.pause(); + } + } + + _onEnd() { + if (this.ended) return; + this.ended = true; + this._cleanup(); + if (this.sink) { + this.sink.end(this._endedWithError); + this.sink.source = null; + this.sink = null; + } + } + + _onError(err) { + if (this.ended) return; + this._endedWithError = err; + this.ended = err; + this._cleanup(); + if (this.sink) { + this.sink.end(err); + this.sink.source = null; + this.sink = null; + } + } + + _onResume() { + if (this.paused && !this.ended) { + this.paused = false; + this.readable.resume(); + this._drain(); + } + } + + _cleanup() { + this.readable.removeListener('data', this._onData); + this.readable.removeListener('end', this._onEnd); + this.readable.removeListener('error', this._onError); + } +} + +class PushSink { + constructor(writable) { + this.writable = writable; + this.source = null; + this.paused = false; + this.ended = false; + this._endedWithError = null; + this._draining = false; + + this._onDrain = this._onDrain.bind(this); + this._onError = this._onError.bind(this); + + writable.on('drain', this._onDrain); + writable.on('error', this._onError); + } + + pipe(sink) { + if (sink && sink.write) { + this.source = sink; + sink.source = this; + } + return sink; + } + + write(data) { + if (this.ended) return; + const ok = this.writable.write(data); + if (!ok) { + this.paused = true; + } + return ok; + } + + end(err) { + if (this.ended) return; + this.ended = err || true; + this._cleanup(); + if (this.writable.destroyed || this.writable.writableEnded) { + return; + } + if (err) { + this.writable.destroy(err); + } else { + this.writable.end(); + } + if (this.source) { + this.source.sink = null; + this.source = null; + } + } + + abort(err) { + if (this.ended) return; + this.ended = err || new Error('Aborted'); + this._cleanup(); + if (!this.writable.destroyed) { + this.writable.destroy(this.ended); + } + if (this.source) { + this.source.abort(this.ended); + this.source = null; + } + } + + _onDrain() { + if (this.paused && !this.ended) { + this.paused = false; + if (this.source) { + this.source.resume(); + } + } + } + + _onError(err) { + if (this.ended) return; + this._endedWithError = err; + this.ended = err; + this._cleanup(); + if (this.source) { + this.source.abort(err); + this.source.sink = null; + this.source = null; + } + } + + _cleanup() { + this.writable.removeListener('drain', this._onDrain); + this.writable.removeListener('error', this._onError); + } +} + +class PushDuplex { + constructor(duplex) { + this.source = new PushSource(duplex); + this.sink = new PushSink(duplex); + this.paused = false; + this.ended = false; + + this.sink.source = this.source; + this.source.sink = this.sink; + } + + pipe(sink) { + return this.source.pipe(sink); + } + + resume() { + this.source.resume(); + } + + abort(err) { + if (this.ended) return; + this.ended = err || new Error('Aborted'); + this.source.abort(err); + this.sink.end(err); + } +} + +export function source(readableStream) { + if (!(readableStream instanceof Readable)) { + throw new Error('Expected a Node.js Readable stream'); + } + return new PushSource(readableStream); +} + +export function sink(writableStream) { + if (!(writableStream instanceof Writable)) { + throw new Error('Expected a Node.js Writable stream'); + } + return new PushSink(writableStream); +} + +export function duplex(duplexStream) { + if (!(duplexStream instanceof Duplex)) { + throw new Error('Expected a Node.js Duplex stream'); + } + return new PushDuplex(duplexStream); +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..d3225bd --- /dev/null +++ b/package.json @@ -0,0 +1,33 @@ +{ + "name": "@push-stream-std/stream-to-push-stream", + "version": "1.0.0", + "description": "Convert Node.js streams to push-streams", + "type": "module", + "main": "index.js", + "exports": { + ".": "./index.js" + }, + "scripts": { + "test": "node --test test/", + "bench": "node bench/index.js", + "lint": "node --check index.js test/*.js" + }, + "keywords": [ + "push-stream", + "stream", + "convert" + ], + "license": "SEE LICENSE IN LICENSE", + "engines": { + "node": ">=22.0.0" + }, + "files": [ + "index.js", + "bench/", + "README.md", + "LICENSE" + ], + "publishConfig": { + "registry": "https://hub.kl1.tenere.ai/api/packages/push-stream-std/npm/" + } +} diff --git a/test/index.js b/test/index.js new file mode 100644 index 0000000..f36c009 --- /dev/null +++ b/test/index.js @@ -0,0 +1,366 @@ +import { Readable, Writable, Duplex } from 'stream'; +import { test, describe, beforeEach, afterEach } from 'node:test'; +import { source, sink, duplex } from '../index.js'; + +function readableFromArray(arr) { + return new Readable({ + objectMode: true, + read() { + if (arr.length > 0) { + this.push(arr.shift()); + } else { + this.push(null); + } + } + }); +} + +function writableToArray() { + const arr = []; + return new Writable({ + objectMode: true, + write(chunk, encoding, callback) { + arr.push(chunk); + callback(); + } + }); +} + +describe('stream-to-push-stream', () => { + describe('source', () => { + test('converts readable stream to push-source', () => { + const received = []; + const readable = readableFromArray([1, 2, 3, 4, 5]); + const pushSrc = source(readable); + const output = []; + + pushSrc.pipe({ + write(data) { + output.push(data); + }, + end() {}, + get paused() { return false; }, + set paused(v) {} + }); + + return new Promise((resolve) => { + setTimeout(() => { + if (output.length === 5) { + resolve(); + } else { + resolve(new Error('Expected 5 items')); + } + }, 50); + }); + }); + + test('handles backpressure', () => { + const received = []; + const output = []; + let paused = true; + + const readable = readableFromArray([1, 2, 3, 4, 5]); + const pushSrc = source(readable); + + pushSrc.pipe({ + write(data) { + output.push(data); + if (output.length === 1) { + paused = true; + this.paused = true; + } else if (output.length === 3) { + paused = false; + this.paused = false; + pushSrc.resume(); + } + }, + end() {}, + get paused() { return paused; }, + set paused(v) { paused = v; } + }); + + return new Promise((resolve) => { + setTimeout(() => { + if (output.length === 5) { + resolve(); + } else { + resolve(new Error('Expected 5 items, got ' + output.length)); + } + }, 50); + }); + }); + + test('propagates errors', () => { + const readable = new Readable({ + read() { + this.destroy(new Error('Test error')); + } + }); + const pushSrc = source(readable); + let errorReceived = null; + + pushSrc.pipe({ + write(data) {}, + end(err) { + errorReceived = err; + }, + get paused() { return false; }, + set paused(v) {} + }); + + return new Promise((resolve) => { + setTimeout(() => { + if (errorReceived && errorReceived.message === 'Test error') { + resolve(); + } else { + resolve(new Error('Error not propagated')); + } + }, 50); + }); + }); + + test('handles already-ended stream', () => { + const readable = new Readable({ + read() {} + }); + readable.push(null); + + const pushSrc = source(readable); + let endCalled = false; + + pushSrc.pipe({ + write(data) {}, + end() { endCalled = true; }, + get paused() { return false; }, + set paused(v) {} + }); + + return new Promise((resolve) => { + setTimeout(() => { + if (endCalled) { + resolve(); + } else { + resolve(new Error('End not called for already-ended stream')); + } + }, 50); + }); + }); + }); + + describe('sink', () => { + test('converts writable stream to push-sink', () => { + const writable = new Writable({ + objectMode: true, + write(chunk, encoding, callback) { + callback(); + } + }); + + const received = []; + writable.on('data', (chunk) => received.push(chunk)); + + const pushSnk = sink(writable); + + pushSnk.write(1); + pushSnk.write(2); + pushSnk.write(3); + pushSnk.end(); + + return new Promise((resolve) => { + setTimeout(() => { + if (received.length === 3) { + resolve(); + } else { + resolve(new Error('Expected 3 items, got ' + received.length)); + } + }, 50); + }); + }); + + test('handles backpressure from writable', () => { + let canWrite = false; + const writable = new Writable({ + objectMode: true, + write(chunk, encoding, callback) { + if (!canWrite) { + callback(false); + } else { + callback(); + } + } + }); + + const pushSnk = sink(writable); + let drainCount = 0; + + writable.on('drain', () => { + drainCount++; + canWrite = true; + pushSnk.resume(); + }); + + pushSnk.write(1); + pushSnk.write(2); + + return new Promise((resolve) => { + setTimeout(() => { + if (drainCount > 0) { + resolve(); + } else { + resolve(new Error('Drain not emitted')); + } + }, 50); + }); + }); + + test('propagates writable errors', () => { + const writable = new Writable({ + objectMode: true, + write(chunk, encoding, callback) { + callback(new Error('Write error')); + } + }); + + const pushSnk = sink(writable); + let errorReceived = null; + + pushSnk.write(1); + + writable.on('error', (err) => { + errorReceived = err; + }); + + return new Promise((resolve) => { + setTimeout(() => { + if (errorReceived) { + resolve(); + } else { + resolve(new Error('Error not propagated')); + } + }, 50); + }); + }); + }); + + describe('duplex', () => { + test('converts duplex stream to push-duplex', () => { + const duplexStream = new Duplex({ + objectMode: true, + read() {}, + write(chunk, encoding, callback) { + this.push(chunk); + callback(); + } + }); + + const pushDup = duplex(duplexStream); + const output = []; + + pushDup.pipe({ + write(data) { + output.push(data); + }, + end() {}, + get paused() { return false; }, + set paused(v) {} + }); + + duplexStream.write(1, () => { + duplexStream.write(2, () => { + duplexStream.end(); + }); + }); + + return new Promise((resolve) => { + setTimeout(() => { + if (output.length === 2) { + resolve(); + } else { + resolve(new Error('Expected 2 items, got ' + output.length)); + } + }, 50); + }); + }); + + test('handles source and sink independently', () => { + const duplexStream = new Duplex({ + objectMode: true, + read() {}, + write(chunk, encoding, callback) { + this.push(chunk); + callback(); + } + }); + + const pushDup = duplex(duplexStream); + const sourceOutput = []; + const sinkReceived = []; + + pushDup.sink.pipe({ + write(data) { + sinkReceived.push(data); + }, + end() {}, + get paused() { return false; }, + set paused(v) {} + }); + + pushDup.source.pipe({ + write(data) { + sourceOutput.push(data); + }, + end() {}, + get paused() { return false; }, + set paused(v) {} + }); + + duplexStream.write('test'); + duplexStream.push('response'); + + return new Promise((resolve) => { + setTimeout(() => { + if (sinkReceived.length === 1 && sourceOutput.length === 1) { + resolve(); + } else { + resolve(new Error('Expected 1 item each')); + } + }, 50); + }); + }); + }); + + describe('error handling', () => { + test('source throws for non-Readable', () => { + try { + source({}); + throw new Error('Expected error'); + } catch (e) { + if (e.message !== 'Expected a Node.js Readable stream') { + throw new Error('Wrong error: ' + e.message); + } + } + }); + + test('sink throws for non-Writable', () => { + try { + sink({}); + throw new Error('Expected error'); + } catch (e) { + if (e.message !== 'Expected a Node.js Writable stream') { + throw new Error('Wrong error: ' + e.message); + } + } + }); + + test('duplex throws for non-Duplex', () => { + try { + duplex({}); + throw new Error('Expected error'); + } catch (e) { + if (e.message !== 'Expected a Node.js Duplex stream') { + throw new Error('Wrong error: ' + e.message); + } + } + }); + }); +});