release(v4.2.0): pull-mode streams for browser @shade/files

4.1.0's HTTP RPC for browsers capped at inline payloads (≤ 256 KiB).
4.2.0 unlocks streams: server queues outbound chunks + control
envelopes per peer, browser long-polls the queue. Browser-to-server
writes ride the existing /v1/transfer/<id>/chunk POST routes
unchanged.

For Dispatch this unlocks mod-jar uploads (50 MB) and world-backup
downloads (100+ MB) — the actual reason browser-side @shade/files
matters.

### New API

@shade/sdk:
- shade.transferQueueRoute(opts?) — Hono app with /queue +
  /v1/transfer/* routes. Auto-configures the queue transport.
- shade.configureTransfers extended: transport + envelopeTransport
  override slots; resolveBaseUrl optional when both supplied.

@shade/transfer:
- OutboundQueue — per-peer monotonic event log with long-poll
  semantics, idle-eviction GC, ring-buffered to maxEventsPerPeer.
- QueueTransferTransport — enqueues instead of POSTing.

@shade/files:
- httpClient({ outboundQueueUrl, transferBaseUrl }) — when set,
  starts a long-poll drainer + builds a streams-bridge. fs.read /
  fs.write of >256 KiB work end-to-end.
- startQueueDrainer(shade, opts) — exported helper for advanced
  consumers driving their own drainer.

### Implementation notes

- ClientStreamsBridge's TransformStream had HWM=0 by default which
  stalled the drainer's await chain at chunk 4 (writer.write pended
  before the consumer's reader was attached). Bumped to HWM=64 so
  the receive loop can buffer ahead of the consumer.

### Tests

3 new integration tests in tests/integration/http-rpc-streams.test.ts:
4 MiB streamed read round-trip, inline-only error path, idle-timeout
long-poll behaviour.

Wire-compatible. Source-compatible. Lockstep bump to 4.2.0.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

packages/shade-files/src/client/http-client.ts

@@ -69,6 +69,11 @@ import {
 import { buildRpcRequest } from '../protocol/rpc-builder.js';
 import { decideInline, INLINE_THRESHOLD, type WriteSource } from './inline-threshold.js';
 import { base64ToBytes, bytesToBase64 } from '../protocol/canonical.js';
+import { startQueueDrainer, type QueueDrainerHandle } from './queue-drainer.js';
+import {
+  createClientStreamsBridge,
+  type ClientStreamsBridge,
+} from './streams-bridge.js';
 import type {
   FileClient,
   ReadOpts,
@@ -80,7 +85,7 @@ import type {
 } from './client.js';
 
 export interface FilesHttpClientOptions
-  extends Omit<CreateFileClientOptions, 'streamsBridge' | 'ioTimeoutMs'> {
+  extends Omit<CreateFileClientOptions, 'streamsBridge'> {
   /**
    * Server endpoint that hosts `createFilesRpcRoute(...)`. Typically:
    * `https://server.example.com/api/v1/shade-files/rpc`.
@@ -98,6 +103,32 @@ export interface FilesHttpClientOptions
    * orthogonal to the ratchet authentication on the envelope itself.
    */
   headers?: Record<string, string>;
+  /**
+   * Server endpoint that hosts `transferQueueRoute()`'s long-poll
+   * endpoint. Typically:
+   * `https://server.example.com/api/v1/shade-files/queue`.
+   *
+   * When supplied, the client starts a background long-poll that
+   * drains queued envelopes + chunks from the server and dispatches
+   * them via `shade.acceptTransferEnvelope`. This unlocks
+   * **streamed reads** (>256 KiB) for browser-style consumers.
+   */
+  outboundQueueUrl?: string;
+  /**
+   * Base URL for outbound transfer routes (browser → server). Required
+   * alongside `outboundQueueUrl` to enable streamed writes. Typically:
+   * `https://server.example.com/api/v1/shade-files`.
+   *
+   * The client POSTs:
+   *   - chunks to `<base>/v1/transfer/<streamId>/chunk`
+   *   - control envelopes to `<base>/v1/transfer/control`
+   */
+  transferBaseUrl?: string;
+  /**
+   * Long-poll block timeout, milliseconds. Default 30_000. Server
+   * clamps to its own `maxBlockMs` (default 55_000).
+   */
+  queueBlockMs?: number;
 }
 
 interface RoundTripOpts {
@@ -112,6 +143,12 @@ interface RoundTripOpts {
  * (via `shade.initSessionFromBundle(peerAddress, bundle)` or an
  * incoming first-message). Otherwise the first RPC will fail with
  * "decrypt failed: no session for peer".
+ *
+ * When `outboundQueueUrl` + `transferBaseUrl` are supplied, the
+ * client also unlocks **streamed reads/writes** for files larger than
+ * the inline threshold (256 KiB). The browser polls the server's
+ * outbound queue for chunks/envelopes and POSTs its own outbound
+ * chunks to the server's transfer-receive routes.
  */
 export function createFilesHttpClient(
   shade: ShadeBridge,
@@ -122,9 +159,84 @@ export function createFilesHttpClient(
   const fetchFn = options.fetch ?? globalThis.fetch.bind(globalThis);
   const extraHeaders = options.headers ?? {};
   const defaultTimeoutMs = options.defaultTimeoutMs ?? 30_000;
+  const ioTimeoutMs = options.ioTimeoutMs ?? 60_000;
   const signRequest = options.signRequest;
   const senderAddress = shade.myAddress;
 
+  // ─── Streamed-mode bootstrap ─────────────────────────────────
+  //
+  // When `outboundQueueUrl` is supplied, the client:
+  //  1. Configures `shade.configureTransfers(...)` so outbound
+  //     chunks POST to `<transferBaseUrl>/v1/transfer/<streamId>/chunk`
+  //     and outbound control envelopes POST to
+  //     `<transferBaseUrl>/v1/transfer/control`.
+  //  2. Spawns a streams-bridge so streamed reads can be awaited.
+  //  3. Starts a long-poll drainer that pulls queued envelopes +
+  //     chunks from the server and dispatches via
+  //     `shade.acceptTransferEnvelope`.
+
+  let drainer: QueueDrainerHandle | null = null;
+  let streamsBridgePromise: Promise<ClientStreamsBridge> | null = null;
+  let streamsBridge: ClientStreamsBridge | null = null;
+
+  if (options.outboundQueueUrl !== undefined) {
+    const outboundQueueUrl = options.outboundQueueUrl;
+    if (options.transferBaseUrl === undefined) {
+      throw new Error(
+        'createFilesHttpClient: outboundQueueUrl was supplied without transferBaseUrl. Pass `transferBaseUrl` (the server prefix that hosts /v1/transfer/...) so outbound chunks have a destination.',
+      );
+    }
+    if (shade.configureTransfers === undefined) {
+      throw new Error(
+        'createFilesHttpClient: shade.configureTransfers is required for streamed mode (the underlying ShadeBridge must surface it).',
+      );
+    }
+    const transferBaseUrl = options.transferBaseUrl.replace(/\/$/, '');
+    shade.configureTransfers({
+      resolveBaseUrl: async (peer) => {
+        if (peer !== peerAddress) {
+          throw new Error(
+            `httpClient is bound to peer "${peerAddress}" — refusing to resolve outgoing chunks for "${peer}" without a multi-peer registry. Use shade.files.client(peer) for server-to-server multi-peer.`,
+          );
+        }
+        return transferBaseUrl;
+      },
+    });
+    // Build the streams-bridge eagerly. The engine's incoming-transfer
+    // subscription has to be in place BEFORE the drainer dispatches the
+    // first stream-init envelope, otherwise the engine emits the
+    // IncomingTransfer to zero handlers and the read silently never
+    // accepts. We kick off the drainer once the bridge has subscribed.
+    streamsBridgePromise = createClientStreamsBridge(shade).then((bridge) => {
+      streamsBridge = bridge;
+      drainer = startQueueDrainer(shade, {
+        outboundQueueUrl,
+        peerAddress,
+        senderAddress,
+        ...(options.fetch !== undefined ? { fetch: options.fetch } : {}),
+        ...(options.headers !== undefined ? { headers: options.headers } : {}),
+        ...(options.queueBlockMs !== undefined ? { blockMs: options.queueBlockMs } : {}),
+      });
+      return bridge;
+    });
+    // Surface bridge-construction failures eagerly via a rejected
+    // promise the next read/write picks up.
+    streamsBridgePromise.catch(() => {
+      /* observed via getStreamsBridge() */
+    });
+  }
+
+  async function getStreamsBridge(): Promise<ClientStreamsBridge> {
+    if (streamsBridge !== null) return streamsBridge;
+    if (streamsBridgePromise === null) {
+      throw new ConflictError(
+        `http RPC client supports inline writes/reads only (≤ ${INLINE_THRESHOLD} bytes) — pass { outboundQueueUrl, transferBaseUrl } to enable streamed transfers.`,
+      );
+    }
+    streamsBridge = await streamsBridgePromise;
+    return streamsBridge;
+  }
+
   /**
    * Encrypt + POST + decrypt + parse one RPC round-trip.
    *
@@ -321,20 +433,39 @@ export function createFilesHttpClient(
         ReadResultSchema,
         opts,
       );
-      if (wire.kind !== 'inline') {
-        // The HTTP RPC route does not service streamed reads — there is
-        // no place to stream from in pure request-response.
+      if (wire.kind === 'inline') {
+        const bytes = base64ToBytes(wire.bytesB64);
+        const out: ReadOutput = {
+          kind: 'inline',
+          bytes,
+          size: wire.size,
+          sha256: wire.sha256,
+          ...(wire.contentType !== undefined ? { contentType: wire.contentType } : {}),
+        };
+        return out;
+      }
+      // Streamed read — only supported when the queue drainer is wired.
+      if (drainer === null) {
         throw new InternalFileError(
-          `http RPC client received a streamed read (size ${wire.size}). Use shade.files.client(peer) on a server-to-server deployment, or pass { preferInline: true } when the file is known to fit inline.`,
+          `http RPC client received a streamed read (size ${wire.size}) but is in inline-only mode. Pass { outboundQueueUrl, transferBaseUrl } when constructing the client to enable streamed reads.`,
         );
       }
-      const bytes = base64ToBytes(wire.bytesB64);
+      const bridge = await getStreamsBridge();
+      const bridgeSignal = opts.signal ?? new AbortController().signal;
+      const parked = await bridge.awaitRead(wire.streamId, {
+        expectedFrom: peerAddress,
+        signal: bridgeSignal,
+        timeoutMs: ioTimeoutMs,
+      });
       const out: ReadOutput = {
-        kind: 'inline',
-        bytes,
+        kind: 'streams',
+        stream: parked.readable,
         size: wire.size,
         sha256: wire.sha256,
         ...(wire.contentType !== undefined ? { contentType: wire.contentType } : {}),
+        done: async () => {
+          await parked.done;
+        },
       };
       return out;
     },
@@ -344,25 +475,77 @@ export function createFilesHttpClient(
       const overwrite = opts.overwrite ?? false;
       const contentType = opts.contentType ?? decision.contentType;
 
-      if (decision.kind !== 'inline') {
-        throw new ConflictError(
-          `http RPC client supports inline writes only (≤ ${INLINE_THRESHOLD} bytes). The supplied input was promoted to streams (size ${decision.size ?? 'unknown'}). Use shade.files.client(peer) for streamed writes, or pre-buffer the input below the inline threshold.`,
+      if (decision.kind === 'inline' || opts.forceInline === true) {
+        const bytes = decision.kind === 'inline' ? decision.bytes : null;
+        if (bytes === null) {
+          // forceInline === true with a streams-typed decision —
+          // decideInline always produced a `streams` shape because the
+          // input was a bare ReadableStream. We can't drain a stream
+          // synchronously here without a streams-bridge.
+          throw new ConflictError(
+            'http RPC client cannot forceInline a streamed input — pass a Uint8Array / Blob, or pre-buffer the stream.',
+          );
+        }
+        if (bytes.byteLength > INLINE_THRESHOLD) {
+          throw new ConflictError(
+            `inline write exceeds ${INLINE_THRESHOLD}-byte threshold (got ${bytes.byteLength}); pass forceInline=true to override`,
+          );
+        }
+        const args = WriteArgsSchema.parse({
+          kind: 'inline',
+          path,
+          bytesB64: bytesToBase64(bytes),
+          ...(contentType !== undefined ? { contentType } : {}),
+          overwrite,
+        });
+        return await roundTrip<WriteResult>(
+          KIND_WRITE_V1,
+          'write',
+          args,
+          WriteResultSchema,
+          opts,
         );
       }
+
+      // Streamed write — requires the queue drainer + streams-bridge.
+      if (drainer === null) {
+        throw new ConflictError(
+          `http RPC client supports inline writes only (≤ ${INLINE_THRESHOLD} bytes). The supplied input was promoted to streams (size ${decision.size ?? 'unknown'}). Pass { outboundQueueUrl, transferBaseUrl } to enable streamed writes.`,
+        );
+      }
+      const bridge = await getStreamsBridge();
+      const size = decision.size;
+      if (size === undefined) {
+        throw new ConflictError(
+          'streams write requires a known plaintext size; pass `{ stream, size }` instead of a bare ReadableStream',
+        );
+      }
+      const { writeId, handle } = await bridge.initiateWrite({
+        peer: peerAddress,
+        stream: decision.stream,
+        size,
+        ...(contentType !== undefined ? { contentType } : {}),
+        name: path,
+        ...(opts.signal !== undefined ? { signal: opts.signal } : {}),
+      });
       const args = WriteArgsSchema.parse({
-        kind: 'inline',
+        kind: 'streams',
         path,
-        bytesB64: bytesToBase64(decision.bytes),
+        size,
         ...(contentType !== undefined ? { contentType } : {}),
         overwrite,
+        writeId,
       });
-      return await roundTrip<WriteResult>(
-        KIND_WRITE_V1,
-        'write',
-        args,
-        WriteResultSchema,
-        opts,
-      );
+      try {
+        const [result] = await Promise.all([
+          roundTrip<WriteResult>(KIND_WRITE_V1, 'write', args, WriteResultSchema, opts),
+          handle.done(),
+        ]);
+        return result;
+      } catch (err) {
+        await handle.abort('rpc-failed').catch(() => undefined);
+        throw err;
+      }
     },
 
     async getThumbnail(path, size: ThumbnailSize, opts): Promise<ThumbnailResult> {
@@ -393,7 +576,15 @@ export function createFilesHttpClient(
     },
 
     close(): void {
-      // Stateless — nothing to release. Exists for FileClient symmetry.
+      // Stop the long-poll drainer + tear down the streams-bridge if
+      // we built one. Idempotent — safe to call multiple times.
+      drainer?.stop();
+      drainer = null;
+      if (streamsBridge !== null) {
+        void streamsBridge.destroy().catch(() => undefined);
+        streamsBridge = null;
+      }
+      streamsBridgePromise = null;
     },
   } as FileClient;
 }

packages/shade-files/src/client/queue-drainer.ts

@@ -0,0 +1,172 @@
+/**
+ * Browser-side drainer for the pull-mode outbound queue.
+ *
+ * Background task that long-polls the server's `/queue` endpoint,
+ * decodes each event, and dispatches it into the consumer's Shade
+ * instance via `shade.acceptTransferEnvelope`. Same wire-shape as the
+ * server-to-server case where the engine receives chunks via direct
+ * HTTP POSTs — we just flip the direction so the browser pulls
+ * instead of accepts.
+ */
+import type { ShadeBridge } from '../integration/shade-bridge.js';
+
+export interface QueueDrainerOptions {
+  /**
+   * Server endpoint that hosts `transferQueueRoute()`. Typically:
+   * `https://server.example.com/api/v1/shade-files/queue`.
+   */
+  outboundQueueUrl: string;
+  /** Peer the queue is pulled FROM (the server's address). */
+  peerAddress: string;
+  /** Address we identify ourselves with on the long-poll. */
+  senderAddress: string;
+  /** Optional `fetch` override. Default `globalThis.fetch`. */
+  fetch?: typeof globalThis.fetch;
+  /** Extra headers applied to every poll request. */
+  headers?: Record<string, string>;
+  /**
+   * Long-poll request timeout (server-side block). Default 30_000.
+   * Server clamps to its own `maxBlockMs` (default 55_000).
+   */
+  blockMs?: number;
+  /**
+   * Backoff after a network error before re-polling. Default 2_000.
+   * Doubles up to `maxBackoffMs` on consecutive failures.
+   */
+  initialBackoffMs?: number;
+  maxBackoffMs?: number;
+  /**
+   * Called when a poll cycle fails. Defaults to logging via `console.error`.
+   * Throwing from this hook does NOT stop the drainer — it backs off
+   * and retries.
+   */
+  onError?: (err: unknown) => void;
+}
+
+export interface QueueDrainerHandle {
+  /** Stop the drainer. Pending fetch is aborted; the loop exits. */
+  stop(): void;
+  /** Promise that resolves once the drainer has fully stopped. */
+  stopped: Promise<void>;
+}
+
+interface PolledEvent {
+  id: number;
+  timestampMs: number;
+  kind: 'envelope' | 'chunk';
+  bytesB64: string;
+  meta?: { streamId: string; laneId: number; seq: number };
+}
+
+interface PollResponseBody {
+  events: PolledEvent[];
+  nextSince: number;
+}
+
+const DEFAULT_BLOCK_MS = 30_000;
+const DEFAULT_INITIAL_BACKOFF_MS = 2_000;
+const DEFAULT_MAX_BACKOFF_MS = 30_000;
+
+/**
+ * Start a long-poll loop that drains queued envelopes + chunks from
+ * the server and dispatches them into the local Shade instance.
+ *
+ * Returns a handle the caller can use to stop the drainer when the
+ * `httpClient` is closed (e.g. on tab unload).
+ */
+export function startQueueDrainer(
+  shade: ShadeBridge,
+  options: QueueDrainerOptions,
+): QueueDrainerHandle {
+  if (shade.acceptTransferEnvelope === undefined) {
+    throw new Error(
+      'startQueueDrainer: shade.acceptTransferEnvelope is required for pull-mode streams. The supplied ShadeBridge implementation must surface it.',
+    );
+  }
+  const accept = shade.acceptTransferEnvelope.bind(shade);
+  const fetchFn = options.fetch ?? globalThis.fetch.bind(globalThis);
+  const blockMs = options.blockMs ?? DEFAULT_BLOCK_MS;
+  const onError = options.onError ?? ((err: unknown) => console.error('[shade.files queue-drainer]', err));
+  const initialBackoffMs = options.initialBackoffMs ?? DEFAULT_INITIAL_BACKOFF_MS;
+  const maxBackoffMs = options.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS;
+  const ac = new AbortController();
+  let stopped = false;
+  let resolveStopped!: () => void;
+  const stoppedPromise = new Promise<void>((r) => {
+    resolveStopped = r;
+  });
+
+  void (async () => {
+    let since = 0;
+    let backoff = initialBackoffMs;
+    while (!stopped) {
+      try {
+        const response = await fetchFn(options.outboundQueueUrl, {
+          method: 'POST',
+          signal: ac.signal,
+          headers: {
+            'Content-Type': 'application/json',
+            'X-Shade-Sender-Address': options.senderAddress,
+            ...(options.headers ?? {}),
+          },
+          body: JSON.stringify({ since, blockMs }),
+        });
+        if (!response.ok) {
+          throw new Error(`queue poll → ${response.status} ${response.statusText}`);
+        }
+        const body = (await response.json()) as PollResponseBody;
+        if (Array.isArray(body.events) && body.events.length > 0) {
+          for (const event of body.events) {
+            if (stopped) break;
+            try {
+              const bytes = base64ToBytes(event.bytesB64);
+              await accept(options.peerAddress, bytes);
+            } catch (err) {
+              // Per-event dispatch failure should not kill the loop —
+              // resume picks up missing chunks via @shade/transfer's
+              // built-in lane-resume protocol.
+              onError(err);
+            }
+          }
+        }
+        since = typeof body.nextSince === 'number' ? body.nextSince : since;
+        backoff = initialBackoffMs;
+      } catch (err) {
+        if (stopped || ac.signal.aborted) break;
+        onError(err);
+        // Exponential backoff with jitter — caps at maxBackoffMs.
+        const jitter = Math.floor(Math.random() * Math.min(backoff, 1_000));
+        await new Promise<void>((r) => {
+          const t = setTimeout(r, backoff + jitter);
+          (t as unknown as { unref?: () => void }).unref?.();
+          ac.signal.addEventListener(
+            'abort',
+            () => {
+              clearTimeout(t);
+              r();
+            },
+            { once: true },
+          );
+        });
+        backoff = Math.min(maxBackoffMs, backoff * 2);
+      }
+    }
+    resolveStopped();
+  })();
+
+  return {
+    stop(): void {
+      if (stopped) return;
+      stopped = true;
+      ac.abort(new Error('queue drainer stopped'));
+    },
+    stopped: stoppedPromise,
+  };
+}
+
+function base64ToBytes(b64: string): Uint8Array {
+  const bin = atob(b64);
+  const out = new Uint8Array(bin.length);
+  for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i);
+  return out;
+}

packages/shade-files/src/client/streams-bridge.ts

@@ -102,7 +102,16 @@ export async function createClientStreamsBridge(
     const readStreamId = incoming.metadata.userMetadata?.[META_KEY_READ_STREAM_ID];
     if (readStreamId === undefined) return;
 
-    const ts = new TransformStream<Uint8Array, Uint8Array>();
+    // Generous HWM so the receiver-side write loop (drainer →
+    // engine.receiveChunk → sink.write) doesn't stall on backpressure
+    // before the consumer's reader is wired up. The reader still
+    // applies its own backpressure once it's consuming, but we no
+    // longer race fs.read's await on stream-init against the consumer
+    // attaching its reader.
+    const ts = new TransformStream<Uint8Array, Uint8Array>(undefined, undefined, {
+      highWaterMark: 64,
+      size: (chunk?: Uint8Array) => (chunk === undefined ? 0 : 1),
+    });
     let handle: TransferHandle;
     try {
       handle = await incoming.accept({

packages/shade-files/src/index.ts

@@ -190,6 +190,11 @@ export { createFilesRpcRoute } from './server/rpc-route.js';
 export type { FilesRpcRouteOptions } from './server/rpc-route.js';
 export { createFilesHttpClient } from './client/http-client.js';
 export type { FilesHttpClientOptions } from './client/http-client.js';
+export { startQueueDrainer } from './client/queue-drainer.js';
+export type {
+  QueueDrainerHandle,
+  QueueDrainerOptions,
+} from './client/queue-drainer.js';
 
 // Shared structural surface @shade/files needs from a Shade instance —
 // exposed so consumers building custom Shade-shaped bridges can verify

packages/shade-files/src/integration/shade-bridge.ts

@@ -70,4 +70,25 @@ export interface ShadeBridge {
 
   /** Optional control-envelope passthrough used by the WebRTC bridge. */
   deliverControlEnvelope?(peer: string, envelope: ShadeEnvelope): Promise<void>;
+
+  /**
+   * Hand a freshly-decoded wire envelope (control or chunk) to the
+   * transfer engine. Required by the pull-mode HTTP client when it
+   * drains queued events from the server: each polled chunk / control
+   * envelope is dispatched here so the engine sees it just as if it
+   * had arrived via an HTTP POST on `/v1/transfer/...`.
+   */
+  acceptTransferEnvelope?(from: string, env: ShadeEnvelope | Uint8Array): Promise<void>;
+
+  /**
+   * Configure the transfer stack. Called by the pull-mode HTTP client
+   * to point the browser's outgoing chunks + control envelopes at the
+   * server's transferQueueRoute mount. Optional because the
+   * server-to-server path uses a separate, app-driven configuration.
+   */
+  configureTransfers?(opts: {
+    resolveBaseUrl?: (peerAddress: string) => Promise<string>;
+    transport?: unknown;
+    envelopeTransport?: unknown;
+  }): void;
 }