release(v4.2.0): pull-mode streams for browser @shade/files

4.1.0's HTTP RPC for browsers capped at inline payloads (≤ 256 KiB). 4.2.0 unlocks streams: server queues outbound chunks + control envelopes per peer, browser long-polls the queue. Browser-to-server writes ride the existing /v1/transfer/<id>/chunk POST routes unchanged. For Dispatch this unlocks mod-jar uploads (50 MB) and world-backup downloads (100+ MB) — the actual reason browser-side @shade/files matters. ### New API @shade/sdk: - shade.transferQueueRoute(opts?) — Hono app with /queue + /v1/transfer/* routes. Auto-configures the queue transport. - shade.configureTransfers extended: transport + envelopeTransport override slots; resolveBaseUrl optional when both supplied. @shade/transfer: - OutboundQueue — per-peer monotonic event log with long-poll semantics, idle-eviction GC, ring-buffered to maxEventsPerPeer. - QueueTransferTransport — enqueues instead of POSTing. @shade/files: - httpClient({ outboundQueueUrl, transferBaseUrl }) — when set, starts a long-poll drainer + builds a streams-bridge. fs.read / fs.write of >256 KiB work end-to-end. - startQueueDrainer(shade, opts) — exported helper for advanced consumers driving their own drainer. ### Implementation notes - ClientStreamsBridge's TransformStream had HWM=0 by default which stalled the drainer's await chain at chunk 4 (writer.write pended before the consumer's reader was attached). Bumped to HWM=64 so the receive loop can buffer ahead of the consumer. ### Tests 3 new integration tests in tests/integration/http-rpc-streams.test.ts: 4 MiB streamed read round-trip, inline-only error path, idle-timeout long-poll behaviour. Wire-compatible. Source-compatible. Lockstep bump to 4.2.0. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-03 23:27:06 +02:00
parent da93b97cce
commit 7520b11b25
34 changed files with 1331 additions and 59 deletions
--- a/packages/shade-sdk/package.json
+++ b/packages/shade-sdk/package.json
@@ -1,6 +1,6 @@
 {
  "name": "@shade/sdk",
-  "version": "4.1.0",
+  "version": "4.2.0",
  "type": "module",
  "main": "src/index.ts",
  "types": "src/index.ts",
--- a/packages/shade-sdk/src/shade.ts
+++ b/packages/shade-sdk/src/shade.ts
@@ -31,6 +31,8 @@ import {
  type TransferHandle,
  type TransferOptions,
  type TransferSummary,
+  type OutboundQueue as OutboundQueueLike,
+  type QueuedEventInput,
 } from '@shade/transfer';
 import type { Hono } from 'hono';
 import { BackgroundTasks } from './background.js';
@@ -151,6 +153,8 @@ export class Shade {
  private controlChannel: ShadeControlChannel | null = null;
  private peerBaseUrlResolver: ((peerAddress: string) => Promise<string>) | null = null;
  private envelopeOutboxes: ControlEnvelopeTransport | null = null;
+  private transferTransportOverride: ITransferTransport | null = null;
+  private transferQueue: OutboundQueueLike | null = null;

  // `@shade/files` namespace, lazy + memoized.
  private filesNamespace: FilesNamespace | null = null;
@@ -746,12 +750,52 @@ export class Shade {
   * HTTP POSTs to `<base>/v1/transfer/control`).
   */
  configureTransfers(opts: {
-    resolveBaseUrl: (peerAddress: string) => Promise<string>;
+    /**
+     * Resolver for the peer's HTTP base URL (used by the default
+     * `ShadeTransferHttpTransport` to POST chunks). Optional when a
+     * custom `transport` and `envelopeTransport` are supplied — e.g.
+     * for pull-mode browser servers (`@shade/files transferQueueRoute`)
+     * which never POST chunks anywhere.
+     */
+    resolveBaseUrl?: (peerAddress: string) => Promise<string>;
+    /**
+     * Override the chunk-level transport. Defaults to
+     * `ShadeTransferHttpTransport` (HTTP POSTs per chunk) when
+     * `resolveBaseUrl` is supplied. Required when `resolveBaseUrl`
+     * is omitted.
+     */
+    transport?: ITransferTransport;
+    /**
+     * Override the control-envelope transport. Defaults to HTTP POSTs
+     * to `<base>/v1/transfer/control` when `resolveBaseUrl` is
+     * supplied. Required when `resolveBaseUrl` is omitted.
+     */
    envelopeTransport?: ControlEnvelopeTransport;
  }): void {
-    this.peerBaseUrlResolver = opts.resolveBaseUrl;
-    this.envelopeOutboxes =
-      opts.envelopeTransport ?? new HttpEnvelopeTransport(opts.resolveBaseUrl, this.address);
+    if (opts.resolveBaseUrl === undefined) {
+      if (opts.transport === undefined || opts.envelopeTransport === undefined) {
+        throw new Error(
+          'configureTransfers: resolveBaseUrl is required unless both `transport` and `envelopeTransport` are supplied (e.g. for pull-mode queue servers).',
+        );
+      }
+      this.peerBaseUrlResolver = async () => {
+        throw new Error(
+          'resolveBaseUrl was not configured — this Shade is in queue/pull mode and does not POST chunks. Configure a custom transport instead.',
+        );
+      };
+    } else {
+      this.peerBaseUrlResolver = opts.resolveBaseUrl;
+    }
+    this.transferTransportOverride = opts.transport ?? null;
+    if (opts.envelopeTransport !== undefined) {
+      this.envelopeOutboxes = opts.envelopeTransport;
+    } else if (opts.resolveBaseUrl !== undefined) {
+      this.envelopeOutboxes = new HttpEnvelopeTransport(opts.resolveBaseUrl, this.address);
+    } else {
+      throw new Error(
+        'configureTransfers: envelopeTransport is required when resolveBaseUrl is omitted.',
+      );
+    }
  }

  /**
@@ -898,6 +942,109 @@ export class Shade {
    return (await this.engine()).onIncomingTransfer(handler);
  }

+  /**
+   * Mount the **pull-mode** transfer routes on a Hono app. Mount under
+   * any base path: `app.route('/api/v1/shade-files', shade.transferQueueRoute())`.
+   *
+   * Configures this Shade instance to queue all outbound chunks +
+   * control envelopes per peer instead of POSTing them. Browser-style
+   * receivers drain the queue via long-polling — no inbound HTTP
+   * listener required on the receiver.
+   *
+   * Routes mounted (relative to the base path):
+   *   POST /queue                          — long-poll the per-peer outbound queue
+   *   POST /v1/transfer/:streamId/chunk    — receive incoming chunks (browser → server)
+   *   GET  /v1/transfer/:streamId/state    — resume-state lookup
+   *   POST /v1/transfer/control            — receive incoming control envelopes
+   *   GET  /v1/transfer/health             — peer reachability probe
+   *
+   * **Idempotent**: calling twice returns a fresh `Hono` app each
+   * time but reuses the underlying queue + transport (so the engine
+   * stays single).
+   *
+   * **Ordering**: must be called **before** `shade.files.serve(...)`
+   * (or any other path that builds the engine), because configuring
+   * the queue transport mutates the transfer stack. Calling after the
+   * engine is built throws.
+   */
+  async transferQueueRoute(opts: TransferQueueRouteOptions = {}): Promise<Hono> {
+    if (this.transferEngine !== null && this.transferTransportOverride === null) {
+      throw new Error(
+        'transferQueueRoute(): the transfer engine has already been built with the default HTTP transport. Call transferQueueRoute() before any upload()/onIncomingTransfer()/configureTransfers().',
+      );
+    }
+    const { OutboundQueue, QueueTransferTransport } = await import('@shade/transfer');
+    if (this.transferQueue === null) {
+      this.transferQueue = new OutboundQueue({
+        ...(opts.maxEventsPerPeer !== undefined ? { maxEventsPerPeer: opts.maxEventsPerPeer } : {}),
+        ...(opts.idleEvictionMs !== undefined ? { idleEvictionMs: opts.idleEvictionMs } : {}),
+      });
+    }
+    if (this.transferTransportOverride === null) {
+      const queueTransport = new QueueTransferTransport(this.transferQueue);
+      const queueEnvelopeTransport = new QueueEnvelopeTransport(this.transferQueue);
+      this.configureTransfers({
+        transport: queueTransport,
+        envelopeTransport: queueEnvelopeTransport,
+      });
+    }
+    const queue = this.transferQueue;
+    const blockMs = opts.blockMs ?? 30_000;
+    const maxBlockMs = opts.maxBlockMs ?? 55_000;
+    const engine = await this.engine();
+    const { createTransferRoutes, PermissiveAuthenticator } = await import('@shade/transfer');
+    const app = await createTransferRoutes(engine, {
+      authenticator: PermissiveAuthenticator,
+    });
+    app.post('/v1/transfer/control', async (c) => {
+      const senderAddress = c.req.header('X-Shade-Sender-Address');
+      if (senderAddress === undefined || senderAddress === '') {
+        return c.json({ error: 'missing X-Shade-Sender-Address' }, 400);
+      }
+      const ab = await c.req.arrayBuffer();
+      const bytes = new Uint8Array(ab);
+      try {
+        await this.acceptTransferEnvelope(senderAddress, bytes);
+      } catch (err) {
+        return c.json({ error: (err as Error).message }, 400);
+      }
+      return c.json({ ok: true });
+    });
+    // Long-poll endpoint.
+    app.post('/queue', async (c) => {
+      const senderAddress = c.req.header('X-Shade-Sender-Address');
+      if (senderAddress === undefined || senderAddress === '') {
+        return c.json({ error: 'missing X-Shade-Sender-Address' }, 400);
+      }
+      let body: { since?: unknown; blockMs?: unknown };
+      try {
+        body = (await c.req.json()) as { since?: unknown; blockMs?: unknown };
+      } catch {
+        return c.json({ error: 'invalid JSON body' }, 400);
+      }
+      const since = typeof body.since === 'number' && Number.isFinite(body.since) ? body.since : 0;
+      const requestedBlockMs =
+        typeof body.blockMs === 'number' && Number.isFinite(body.blockMs)
+          ? Math.max(0, Math.min(maxBlockMs, body.blockMs))
+          : blockMs;
+      // Bun-side short-circuit if the request was aborted while we
+      // were holding the long-poll. AbortSignal from the request body
+      // is already surfaced via `c.req.raw.signal` in Hono.
+      const events = await queue.drain(senderAddress, since, requestedBlockMs, c.req.raw.signal);
+      return c.json({
+        events: events.map((e) => ({
+          id: e.id,
+          timestampMs: e.timestampMs,
+          kind: e.kind,
+          bytesB64: bytesToBase64Std(e.bytes),
+          ...(e.kind === 'chunk' ? { meta: e.meta } : {}),
+        })),
+        nextSince: events.length > 0 ? events[events.length - 1]!.id : since,
+      });
+    });
+    return app;
+  }
+
  /**
   * Mount the receiver-side HTTP routes on a Hono app. Mount under any
   * base path: `app.route('/shade', await shade.transferRoute())`.
@@ -1019,16 +1166,23 @@ export class Shade {
      );
    }
    this.controlChannel = new ShadeControlChannel(this, this.envelopeOutboxes);
-    const httpTransport: ITransferTransport = new ShadeTransferHttpTransport({
-      resolveBaseUrl: this.peerBaseUrlResolver,
-      authenticator: await this.makeAuthenticator(),
-    });
-
-    let transport: ITransferTransport = httpTransport;
+    let transport: ITransferTransport;
    let webrtcRuntime: ShadeWebRtcRuntime | null = null;
-    if (this.webrtcConfig !== null) {
-      webrtcRuntime = await this.buildWebRtcRuntime(this.webrtcConfig, httpTransport);
-      transport = webrtcRuntime.fallback;
+    if (this.transferTransportOverride !== null) {
+      // Custom transport (queue, in-memory, custom adapter) — used as-is.
+      // WebRTC fallback only attaches when the default HTTP transport is
+      // active because WebRTC's `MultiTransportFallback` is HTTP-shaped.
+      transport = this.transferTransportOverride;
+    } else {
+      const httpTransport: ITransferTransport = new ShadeTransferHttpTransport({
+        resolveBaseUrl: this.peerBaseUrlResolver,
+        authenticator: await this.makeAuthenticator(),
+      });
+      transport = httpTransport;
+      if (this.webrtcConfig !== null) {
+        webrtcRuntime = await this.buildWebRtcRuntime(this.webrtcConfig, httpTransport);
+        transport = webrtcRuntime.fallback;
+      }
    }

    this.transferEngine = new TransferEngine({
@@ -1256,6 +1410,53 @@ function parseChunkHeader(bytes: Uint8Array): {
  return { streamId, laneId, seq };
 }

+// ─── Queue-mode (pull) envelope transport ─────────────────────
+
+/**
+ * Configuration for {@link Shade.transferQueueRoute}. All fields are
+ * optional with sensible production defaults.
+ */
+export interface TransferQueueRouteOptions {
+  /**
+   * Long-poll timeout in milliseconds. Server holds the request open
+   * up to this long before returning an empty `events` array. Default
+   * 30_000.
+   */
+  blockMs?: number;
+  /**
+   * Hard cap on long-poll timeout (clamps client-supplied `blockMs`).
+   * Default 55_000 — under typical reverse-proxy idle thresholds (60s
+   * on most CDNs).
+   */
+  maxBlockMs?: number;
+  /**
+   * Per-peer ring-buffer size. When the queue is full, oldest events
+   * are dropped on enqueue. Receivers detect the gap via missing
+   * sequence numbers and re-resume from `since=0`. Default 1000.
+   */
+  maxEventsPerPeer?: number;
+  /**
+   * Drop a peer's queue + reject pending pollers after this much
+   * silence. Default 10 minutes. Setting to `0` disables idle-eviction.
+   */
+  idleEvictionMs?: number;
+}
+
+/**
+ * `ControlEnvelopeTransport` that enqueues outbound envelopes into an
+ * `OutboundQueue` for browser-style receivers to long-poll. Mirrors
+ * `HttpEnvelopeTransport` shape (one `send(peer, envelope)` method);
+ * the difference is the destination — local queue, not remote HTTP.
+ */
+class QueueEnvelopeTransport implements ControlEnvelopeTransport {
+  constructor(private readonly queue: OutboundQueueLike) {}
+  async send(peerAddress: string, envelope: ShadeEnvelope): Promise<void> {
+    const bytes = encodeEnvelope(envelope);
+    const event: QueuedEventInput = { kind: 'envelope', bytes };
+    this.queue.enqueue(peerAddress, event);
+  }
+}
+
 // ─── Default HTTP envelope transport ──────────────────────────

 class HttpEnvelopeTransport implements ControlEnvelopeTransport {