initial import
CI / test (18) (push) Successful in 9s
CI / test (20) (push) Successful in 7s
CI / test (22) (push) Successful in 7s

This commit is contained in:
Tenere CI
2026-07-19 13:44:11 -07:00
commit a8d2a5dcf7
8 changed files with 974 additions and 0 deletions
+28
View File
@@ -0,0 +1,28 @@
name: CI
on:
push:
branches: [master]
tags: ['v*']
pull_request:
branches: [master]
jobs:
test:
runs-on: ubuntu-latest
strategy:
matrix:
node: [18, 20, 22]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node }}
- name: Auth for Gitea npm registry
run: |
echo "@${{ github.repository_owner }}:registry=https://hub.kl1.tenere.ai/api/packages/${{ github.repository_owner }}/npm/" > .npmrc
echo "//hub.kl1.tenere.ai/api/packages/${{ github.repository_owner }}/npm/:_authToken=${{ secrets.PACKAGE_TOKEN }}" >> .npmrc
- run: npm install
- run: npm test
- run: npm run bench
if: matrix.node == 22
+2
View File
@@ -0,0 +1,2 @@
@push-stream-std:registry=https://hub.kl1.tenere.ai/api/packages/push-stream-std/npm/
//hub.kl1.tenere.ai/api/packages/push-stream-std/npm/:_authToken=${PACKAGE_TOKEN}
+4
View File
@@ -0,0 +1,4 @@
Tenere Labs Inc. — All Rights Reserved.
PROPRIETARY AND CONFIDENTIAL.
Source-available for auditing and inspection only.
+85
View File
@@ -0,0 +1,85 @@
# stream-to-push-stream
Convert native Node.js streams to push-stream source, sink, and duplex interfaces.
Published as `@push-stream-std/stream-to-push-stream`.
## Synopsis
```js
import { source, sink, duplex } from '@push-stream-std/stream-to-push-stream';
import { Readable, Writable, Duplex } from 'node:stream';
import { PassThrough } from 'node:stream';
// Convert a Readable to a push-stream source
const readable = Readable.from(['a', 'b', 'c']);
const pushSource = source(readable);
const pushSink = {
paused: false,
write(v) { console.log(v); },
end() { console.log('done'); }
};
pushSource.pipe(pushSink);
// Output: a, b, c, done
// Convert a Writable to a push-stream sink
const writable = new PassThrough();
const pushSink2 = sink(writable);
// Convert a Duplex to a push-stream source
const duplexStream = new PassThrough();
const pushSource2 = duplex(duplexStream);
```
## Install
```bash
npm config set @push-stream-std:registry https://hub.kl1.tenere.ai/api/packages/push-stream-std/npm/
npm config set -- //hub.kl1.tenere.ai/api/packages/push-stream-std/npm/:_authToken "$PACKAGE_TOKEN"
npm install @push-stream-std/stream-to-push-stream
```
## Description
Utilities for bridging between the Node.js `stream` module (Readable, Writable, Duplex) and the push-stream protocol. PushSource wraps a Readable, listening to `data`/`end`/`error` events and implementing `pipe()`, `resume()`, and `abort()`. PushSink wraps a Writable, translating `write()` calls to `writable.write()` with `drain`-based backpressure and `end()` to `writable.end()`. PushDuplex wraps a Duplex stream as a combined source/sink for bidirectional push-stream pipelines.
## API
### `source(readableStream)`
Converts a Node.js `Readable` to a push-stream source.
- **readableStream** `Readable` A Node.js readable stream.
- **Returns** A push-stream source with `pipe()`, `resume()`, `abort()`, `paused`, and `ended`.
### `sink(writableStream)`
Converts a Node.js `Writable` to a push-stream sink.
- **writableStream** `Writable` A Node.js writable stream.
- **Returns** A push-stream sink with `pipe()`, `write()`, `end()`, `abort()`, `paused`, and `ended`.
### `duplex(duplexStream)`
Wraps a Node.js `Duplex` stream as a push-stream source.
- **duplexStream** `Duplex` A Node.js duplex stream.
- **Returns** A push-stream source with `pipe()`, `resume()`, `abort()`, `paused`, and `ended`.
## Tests
```bash
npm test
```
Tests cover readable-to-source data flow and end handling, writable-to-sink write and backpressure, duplex stream wrapping, error propagation, and abort cleanup.
## Requirements
Node.js 18 or later. ESM only.
## License
MIT
+211
View File
@@ -0,0 +1,211 @@
import { Readable, Writable, Duplex } from 'stream';
import { source, sink, duplex } from '../index.js';
const CHUNK_SIZE = 16384;
const CHUNK_COUNT = 100000;
function createBenchmarkReadable() {
const data = Buffer.alloc(CHUNK_SIZE, 'x');
let count = 0;
return new Readable({
highWaterMark: 1024 * 1024,
read() {
if (count < CHUNK_COUNT) {
count++;
this.push(data);
} else {
this.push(null);
}
}
});
}
function createBenchmarkWritable() {
let bytesWritten = 0;
const w = new Writable({
highWaterMark: 1024 * 1024,
write(chunk, encoding, callback) {
bytesWritten += chunk.length;
callback();
}
});
return w;
}
async function benchmarkSource() {
const readable = createBenchmarkReadable();
const pushSrc = source(readable);
const received = [];
let byteCount = 0;
const start = process.hrtime.bigint();
await new Promise((resolve, reject) => {
pushSrc.pipe({
write(data) {
byteCount += data.length;
},
end(err) {
if (err) return reject(err);
resolve();
},
get paused() { return false; },
set paused(v) {}
});
});
const end = process.hrtime.bigint();
const duration = Number(end - start) / 1e9;
const throughput = (byteCount / duration / 1024 / 1024).toFixed(2);
return { duration, throughput, bytes: byteCount };
}
async function benchmarkSink() {
const writable = createBenchmarkWritable();
const pushSnk = sink(writable);
const data = Buffer.alloc(CHUNK_SIZE, 'x');
const start = process.hrtime.bigint();
for (let i = 0; i < CHUNK_COUNT; i++) {
pushSnk.write(data);
}
pushSnk.end();
await new Promise((resolve) => {
writable.on('finish', resolve);
});
const end = process.hrtime.bigint();
const duration = Number(end - start) / 1e9;
const bytes = CHUNK_COUNT * CHUNK_SIZE;
const throughput = (bytes / duration / 1024 / 1024).toFixed(2);
return { duration, throughput, bytes };
}
async function benchmarkDuplex() {
const data = Buffer.alloc(CHUNK_SIZE, 'x');
let count = 0;
let byteCount = 0;
const duplexStream = new Duplex({
highWaterMark: 1024 * 1024,
read() {
if (count < CHUNK_COUNT) {
count++;
this.push(data);
} else {
this.push(null);
}
},
write(chunk, encoding, callback) {
callback();
}
});
const pushDup = duplex(duplexStream);
const start = process.hrtime.bigint();
await new Promise((resolve, reject) => {
pushDup.pipe({
write(data) {
byteCount += data.length;
},
end(err) {
if (err) return reject(err);
resolve();
},
get paused() { return false; },
set paused(v) {}
});
});
const end = process.hrtime.bigint();
const duration = Number(end - start) / 1e9;
const throughput = (byteCount / duration / 1024 / 1024).toFixed(2);
return { duration, throughput, bytes: byteCount };
}
async function benchmarkObjectMode() {
const arr = Array.from({ length: CHUNK_COUNT }, (_, i) => ({ id: i, value: Math.random() }));
let index = 0;
const readable = new Readable({
objectMode: true,
highWaterMark: 1024,
read() {
if (index < arr.length) {
this.push(arr[index++]);
} else {
this.push(null);
}
}
});
const pushSrc = source(readable);
let itemCount = 0;
const start = process.hrtime.bigint();
await new Promise((resolve, reject) => {
pushSrc.pipe({
write(data) {
itemCount++;
},
end(err) {
if (err) return reject(err);
resolve();
},
get paused() { return false; },
set paused(v) {}
});
});
const end = process.hrtime.bigint();
const duration = Number(end - start) / 1e9;
const throughput = (itemCount / duration).toFixed(0);
return { duration, throughput, items: itemCount };
}
async function runBenchmarks() {
console.log('stream-to-push-stream benchmarks');
console.log('================================');
console.log(`Chunk size: ${CHUNK_SIZE} bytes`);
console.log(`Chunk count: ${CHUNK_COUNT}`);
console.log();
console.log('Running benchmarks...\n');
console.log('1. Source (Readable -> Push-Source):');
const sourceResult = await benchmarkSource();
console.log(` Duration: ${sourceResult.duration.toFixed(3)}s`);
console.log(` Throughput: ${sourceResult.throughput} MB/s`);
console.log();
console.log('2. Sink (Writable -> Push-Sink):');
const sinkResult = await benchmarkSink();
console.log(` Duration: ${sinkResult.duration.toFixed(3)}s`);
console.log(` Throughput: ${sinkResult.throughput} MB/s`);
console.log();
console.log('3. Duplex (Duplex -> Push-Duplex):');
const duplexResult = await benchmarkDuplex();
console.log(` Duration: ${duplexResult.duration.toFixed(3)}s`);
console.log(` Throughput: ${duplexResult.throughput} MB/s`);
console.log();
console.log('4. Object Mode (Readable Object -> Push-Source):');
const objectResult = await benchmarkObjectMode();
console.log(` Duration: ${objectResult.duration.toFixed(3)}s`);
console.log(` Throughput: ${objectResult.throughput} items/s`);
console.log();
console.log('Benchmark complete.');
}
runBenchmarks().catch(console.error);
+245
View File
@@ -0,0 +1,245 @@
import { Readable, Writable, Duplex } from 'stream';
class PushSource {
constructor(readable) {
this.readable = readable;
this.sink = null;
this.paused = true;
this.ended = false;
this._endedWithError = null;
this._onData = this._onData.bind(this);
this._onEnd = this._onEnd.bind(this);
this._onError = this._onError.bind(this);
this._onResume = this._onResume.bind(this);
readable.on('data', this._onData);
readable.on('end', this._onEnd);
readable.on('error', this._onError);
}
pipe(sink) {
this.sink = sink;
sink.source = this;
if (!sink.paused) {
this.resume();
}
return sink;
}
resume() {
if (this.ended) return;
this.paused = false;
this.readable.resume();
this._drain();
}
abort(err) {
if (this.ended) return;
this.ended = err || new Error('Aborted');
this._cleanup();
if (this.sink) {
this.sink.end(this.ended);
this.sink.source = null;
this.sink = null;
}
}
_drain() {
// _drain is no longer needed - _onData handles all data events
// This method is kept for compatibility but does nothing
// The readable stream's 'data' event handler (_onData) is already set up in constructor
}
_onData(data) {
if (this.ended) return;
if (this.sink) {
if (this.sink.paused) {
this.paused = true;
this.readable.pause();
return;
}
this.sink.write(data);
this.paused = this.sink.paused;
} else {
this.paused = true;
this.readable.pause();
}
}
_onEnd() {
if (this.ended) return;
this.ended = true;
this._cleanup();
if (this.sink) {
this.sink.end(this._endedWithError);
this.sink.source = null;
this.sink = null;
}
}
_onError(err) {
if (this.ended) return;
this._endedWithError = err;
this.ended = err;
this._cleanup();
if (this.sink) {
this.sink.end(err);
this.sink.source = null;
this.sink = null;
}
}
_onResume() {
if (this.paused && !this.ended) {
this.paused = false;
this.readable.resume();
this._drain();
}
}
_cleanup() {
this.readable.removeListener('data', this._onData);
this.readable.removeListener('end', this._onEnd);
this.readable.removeListener('error', this._onError);
}
}
class PushSink {
constructor(writable) {
this.writable = writable;
this.source = null;
this.paused = false;
this.ended = false;
this._endedWithError = null;
this._draining = false;
this._onDrain = this._onDrain.bind(this);
this._onError = this._onError.bind(this);
writable.on('drain', this._onDrain);
writable.on('error', this._onError);
}
pipe(sink) {
if (sink && sink.write) {
this.source = sink;
sink.source = this;
}
return sink;
}
write(data) {
if (this.ended) return;
const ok = this.writable.write(data);
if (!ok) {
this.paused = true;
}
return ok;
}
end(err) {
if (this.ended) return;
this.ended = err || true;
this._cleanup();
if (this.writable.destroyed || this.writable.writableEnded) {
return;
}
if (err) {
this.writable.destroy(err);
} else {
this.writable.end();
}
if (this.source) {
this.source.sink = null;
this.source = null;
}
}
abort(err) {
if (this.ended) return;
this.ended = err || new Error('Aborted');
this._cleanup();
if (!this.writable.destroyed) {
this.writable.destroy(this.ended);
}
if (this.source) {
this.source.abort(this.ended);
this.source = null;
}
}
_onDrain() {
if (this.paused && !this.ended) {
this.paused = false;
if (this.source) {
this.source.resume();
}
}
}
_onError(err) {
if (this.ended) return;
this._endedWithError = err;
this.ended = err;
this._cleanup();
if (this.source) {
this.source.abort(err);
this.source.sink = null;
this.source = null;
}
}
_cleanup() {
this.writable.removeListener('drain', this._onDrain);
this.writable.removeListener('error', this._onError);
}
}
class PushDuplex {
constructor(duplex) {
this.source = new PushSource(duplex);
this.sink = new PushSink(duplex);
this.paused = false;
this.ended = false;
this.sink.source = this.source;
this.source.sink = this.sink;
}
pipe(sink) {
return this.source.pipe(sink);
}
resume() {
this.source.resume();
}
abort(err) {
if (this.ended) return;
this.ended = err || new Error('Aborted');
this.source.abort(err);
this.sink.end(err);
}
}
export function source(readableStream) {
if (!(readableStream instanceof Readable)) {
throw new Error('Expected a Node.js Readable stream');
}
return new PushSource(readableStream);
}
export function sink(writableStream) {
if (!(writableStream instanceof Writable)) {
throw new Error('Expected a Node.js Writable stream');
}
return new PushSink(writableStream);
}
export function duplex(duplexStream) {
if (!(duplexStream instanceof Duplex)) {
throw new Error('Expected a Node.js Duplex stream');
}
return new PushDuplex(duplexStream);
}
+33
View File
@@ -0,0 +1,33 @@
{
"name": "@push-stream-std/stream-to-push-stream",
"version": "1.0.0",
"description": "Convert Node.js streams to push-streams",
"type": "module",
"main": "index.js",
"exports": {
".": "./index.js"
},
"scripts": {
"test": "node --test test/",
"bench": "node bench/index.js",
"lint": "node --check index.js test/*.js"
},
"keywords": [
"push-stream",
"stream",
"convert"
],
"license": "SEE LICENSE IN LICENSE",
"engines": {
"node": ">=22.0.0"
},
"files": [
"index.js",
"bench/",
"README.md",
"LICENSE"
],
"publishConfig": {
"registry": "https://hub.kl1.tenere.ai/api/packages/push-stream-std/npm/"
}
}
+366
View File
@@ -0,0 +1,366 @@
import { Readable, Writable, Duplex } from 'stream';
import { test, describe, beforeEach, afterEach } from 'node:test';
import { source, sink, duplex } from '../index.js';
function readableFromArray(arr) {
return new Readable({
objectMode: true,
read() {
if (arr.length > 0) {
this.push(arr.shift());
} else {
this.push(null);
}
}
});
}
function writableToArray() {
const arr = [];
return new Writable({
objectMode: true,
write(chunk, encoding, callback) {
arr.push(chunk);
callback();
}
});
}
describe('stream-to-push-stream', () => {
describe('source', () => {
test('converts readable stream to push-source', () => {
const received = [];
const readable = readableFromArray([1, 2, 3, 4, 5]);
const pushSrc = source(readable);
const output = [];
pushSrc.pipe({
write(data) {
output.push(data);
},
end() {},
get paused() { return false; },
set paused(v) {}
});
return new Promise((resolve) => {
setTimeout(() => {
if (output.length === 5) {
resolve();
} else {
resolve(new Error('Expected 5 items'));
}
}, 50);
});
});
test('handles backpressure', () => {
const received = [];
const output = [];
let paused = true;
const readable = readableFromArray([1, 2, 3, 4, 5]);
const pushSrc = source(readable);
pushSrc.pipe({
write(data) {
output.push(data);
if (output.length === 1) {
paused = true;
this.paused = true;
} else if (output.length === 3) {
paused = false;
this.paused = false;
pushSrc.resume();
}
},
end() {},
get paused() { return paused; },
set paused(v) { paused = v; }
});
return new Promise((resolve) => {
setTimeout(() => {
if (output.length === 5) {
resolve();
} else {
resolve(new Error('Expected 5 items, got ' + output.length));
}
}, 50);
});
});
test('propagates errors', () => {
const readable = new Readable({
read() {
this.destroy(new Error('Test error'));
}
});
const pushSrc = source(readable);
let errorReceived = null;
pushSrc.pipe({
write(data) {},
end(err) {
errorReceived = err;
},
get paused() { return false; },
set paused(v) {}
});
return new Promise((resolve) => {
setTimeout(() => {
if (errorReceived && errorReceived.message === 'Test error') {
resolve();
} else {
resolve(new Error('Error not propagated'));
}
}, 50);
});
});
test('handles already-ended stream', () => {
const readable = new Readable({
read() {}
});
readable.push(null);
const pushSrc = source(readable);
let endCalled = false;
pushSrc.pipe({
write(data) {},
end() { endCalled = true; },
get paused() { return false; },
set paused(v) {}
});
return new Promise((resolve) => {
setTimeout(() => {
if (endCalled) {
resolve();
} else {
resolve(new Error('End not called for already-ended stream'));
}
}, 50);
});
});
});
describe('sink', () => {
test('converts writable stream to push-sink', () => {
const writable = new Writable({
objectMode: true,
write(chunk, encoding, callback) {
callback();
}
});
const received = [];
writable.on('data', (chunk) => received.push(chunk));
const pushSnk = sink(writable);
pushSnk.write(1);
pushSnk.write(2);
pushSnk.write(3);
pushSnk.end();
return new Promise((resolve) => {
setTimeout(() => {
if (received.length === 3) {
resolve();
} else {
resolve(new Error('Expected 3 items, got ' + received.length));
}
}, 50);
});
});
test('handles backpressure from writable', () => {
let canWrite = false;
const writable = new Writable({
objectMode: true,
write(chunk, encoding, callback) {
if (!canWrite) {
callback(false);
} else {
callback();
}
}
});
const pushSnk = sink(writable);
let drainCount = 0;
writable.on('drain', () => {
drainCount++;
canWrite = true;
pushSnk.resume();
});
pushSnk.write(1);
pushSnk.write(2);
return new Promise((resolve) => {
setTimeout(() => {
if (drainCount > 0) {
resolve();
} else {
resolve(new Error('Drain not emitted'));
}
}, 50);
});
});
test('propagates writable errors', () => {
const writable = new Writable({
objectMode: true,
write(chunk, encoding, callback) {
callback(new Error('Write error'));
}
});
const pushSnk = sink(writable);
let errorReceived = null;
pushSnk.write(1);
writable.on('error', (err) => {
errorReceived = err;
});
return new Promise((resolve) => {
setTimeout(() => {
if (errorReceived) {
resolve();
} else {
resolve(new Error('Error not propagated'));
}
}, 50);
});
});
});
describe('duplex', () => {
test('converts duplex stream to push-duplex', () => {
const duplexStream = new Duplex({
objectMode: true,
read() {},
write(chunk, encoding, callback) {
this.push(chunk);
callback();
}
});
const pushDup = duplex(duplexStream);
const output = [];
pushDup.pipe({
write(data) {
output.push(data);
},
end() {},
get paused() { return false; },
set paused(v) {}
});
duplexStream.write(1, () => {
duplexStream.write(2, () => {
duplexStream.end();
});
});
return new Promise((resolve) => {
setTimeout(() => {
if (output.length === 2) {
resolve();
} else {
resolve(new Error('Expected 2 items, got ' + output.length));
}
}, 50);
});
});
test('handles source and sink independently', () => {
const duplexStream = new Duplex({
objectMode: true,
read() {},
write(chunk, encoding, callback) {
this.push(chunk);
callback();
}
});
const pushDup = duplex(duplexStream);
const sourceOutput = [];
const sinkReceived = [];
pushDup.sink.pipe({
write(data) {
sinkReceived.push(data);
},
end() {},
get paused() { return false; },
set paused(v) {}
});
pushDup.source.pipe({
write(data) {
sourceOutput.push(data);
},
end() {},
get paused() { return false; },
set paused(v) {}
});
duplexStream.write('test');
duplexStream.push('response');
return new Promise((resolve) => {
setTimeout(() => {
if (sinkReceived.length === 1 && sourceOutput.length === 1) {
resolve();
} else {
resolve(new Error('Expected 1 item each'));
}
}, 50);
});
});
});
describe('error handling', () => {
test('source throws for non-Readable', () => {
try {
source({});
throw new Error('Expected error');
} catch (e) {
if (e.message !== 'Expected a Node.js Readable stream') {
throw new Error('Wrong error: ' + e.message);
}
}
});
test('sink throws for non-Writable', () => {
try {
sink({});
throw new Error('Expected error');
} catch (e) {
if (e.message !== 'Expected a Node.js Writable stream') {
throw new Error('Wrong error: ' + e.message);
}
}
});
test('duplex throws for non-Duplex', () => {
try {
duplex({});
throw new Error('Expected error');
} catch (e) {
if (e.message !== 'Expected a Node.js Duplex stream') {
throw new Error('Wrong error: ' + e.message);
}
}
});
});
});