release(v4.2.0): pull-mode streams for browser @shade/files

4.1.0's HTTP RPC for browsers capped at inline payloads (≤ 256 KiB). 4.2.0 unlocks streams: server queues outbound chunks + control envelopes per peer, browser long-polls the queue. Browser-to-server writes ride the existing /v1/transfer/<id>/chunk POST routes unchanged. For Dispatch this unlocks mod-jar uploads (50 MB) and world-backup downloads (100+ MB) — the actual reason browser-side @shade/files matters. ### New API @shade/sdk: - shade.transferQueueRoute(opts?) — Hono app with /queue + /v1/transfer/* routes. Auto-configures the queue transport. - shade.configureTransfers extended: transport + envelopeTransport override slots; resolveBaseUrl optional when both supplied. @shade/transfer: - OutboundQueue — per-peer monotonic event log with long-poll semantics, idle-eviction GC, ring-buffered to maxEventsPerPeer. - QueueTransferTransport — enqueues instead of POSTing. @shade/files: - httpClient({ outboundQueueUrl, transferBaseUrl }) — when set, starts a long-poll drainer + builds a streams-bridge. fs.read / fs.write of >256 KiB work end-to-end. - startQueueDrainer(shade, opts) — exported helper for advanced consumers driving their own drainer. ### Implementation notes - ClientStreamsBridge's TransformStream had HWM=0 by default which stalled the drainer's await chain at chunk 4 (writer.write pended before the consumer's reader was attached). Bumped to HWM=64 so the receive loop can buffer ahead of the consumer. ### Tests 3 new integration tests in tests/integration/http-rpc-streams.test.ts: 4 MiB streamed read round-trip, inline-only error path, idle-timeout long-poll behaviour. Wire-compatible. Source-compatible. Lockstep bump to 4.2.0. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-03 23:27:06 +02:00
parent da93b97cce
commit 7520b11b25
34 changed files with 1331 additions and 59 deletions
--- a/packages/shade-files/package.json
+++ b/packages/shade-files/package.json
@@ -1,6 +1,6 @@
 {
  "name": "@shade/files",
-  "version": "4.1.0",
+  "version": "4.2.0",
  "type": "module",
  "main": "src/index.ts",
  "types": "src/index.ts",
--- a/packages/shade-files/src/client/http-client.ts
+++ b/packages/shade-files/src/client/http-client.ts
@@ -69,6 +69,11 @@ import {
 import { buildRpcRequest } from '../protocol/rpc-builder.js';
 import { decideInline, INLINE_THRESHOLD, type WriteSource } from './inline-threshold.js';
 import { base64ToBytes, bytesToBase64 } from '../protocol/canonical.js';
+import { startQueueDrainer, type QueueDrainerHandle } from './queue-drainer.js';
+import {
+  createClientStreamsBridge,
+  type ClientStreamsBridge,
+} from './streams-bridge.js';
 import type {
  FileClient,
  ReadOpts,
@@ -80,7 +85,7 @@ import type {
 } from './client.js';

 export interface FilesHttpClientOptions
-  extends Omit<CreateFileClientOptions, 'streamsBridge' | 'ioTimeoutMs'> {
+  extends Omit<CreateFileClientOptions, 'streamsBridge'> {
  /**
   * Server endpoint that hosts `createFilesRpcRoute(...)`. Typically:
   * `https://server.example.com/api/v1/shade-files/rpc`.
@@ -98,6 +103,32 @@ export interface FilesHttpClientOptions
   * orthogonal to the ratchet authentication on the envelope itself.
   */
  headers?: Record<string, string>;
+  /**
+   * Server endpoint that hosts `transferQueueRoute()`'s long-poll
+   * endpoint. Typically:
+   * `https://server.example.com/api/v1/shade-files/queue`.
+   *
+   * When supplied, the client starts a background long-poll that
+   * drains queued envelopes + chunks from the server and dispatches
+   * them via `shade.acceptTransferEnvelope`. This unlocks
+   * **streamed reads** (>256 KiB) for browser-style consumers.
+   */
+  outboundQueueUrl?: string;
+  /**
+   * Base URL for outbound transfer routes (browser → server). Required
+   * alongside `outboundQueueUrl` to enable streamed writes. Typically:
+   * `https://server.example.com/api/v1/shade-files`.
+   *
+   * The client POSTs:
+   *   - chunks to `<base>/v1/transfer/<streamId>/chunk`
+   *   - control envelopes to `<base>/v1/transfer/control`
+   */
+  transferBaseUrl?: string;
+  /**
+   * Long-poll block timeout, milliseconds. Default 30_000. Server
+   * clamps to its own `maxBlockMs` (default 55_000).
+   */
+  queueBlockMs?: number;
 }

 interface RoundTripOpts {
@@ -112,6 +143,12 @@ interface RoundTripOpts {
 * (via `shade.initSessionFromBundle(peerAddress, bundle)` or an
 * incoming first-message). Otherwise the first RPC will fail with
 * "decrypt failed: no session for peer".
+ *
+ * When `outboundQueueUrl` + `transferBaseUrl` are supplied, the
+ * client also unlocks **streamed reads/writes** for files larger than
+ * the inline threshold (256 KiB). The browser polls the server's
+ * outbound queue for chunks/envelopes and POSTs its own outbound
+ * chunks to the server's transfer-receive routes.
 */
 export function createFilesHttpClient(
  shade: ShadeBridge,
@@ -122,9 +159,84 @@ export function createFilesHttpClient(
  const fetchFn = options.fetch ?? globalThis.fetch.bind(globalThis);
  const extraHeaders = options.headers ?? {};
  const defaultTimeoutMs = options.defaultTimeoutMs ?? 30_000;
+  const ioTimeoutMs = options.ioTimeoutMs ?? 60_000;
  const signRequest = options.signRequest;
  const senderAddress = shade.myAddress;

+  // ─── Streamed-mode bootstrap ─────────────────────────────────
+  //
+  // When `outboundQueueUrl` is supplied, the client:
+  //  1. Configures `shade.configureTransfers(...)` so outbound
+  //     chunks POST to `<transferBaseUrl>/v1/transfer/<streamId>/chunk`
+  //     and outbound control envelopes POST to
+  //     `<transferBaseUrl>/v1/transfer/control`.
+  //  2. Spawns a streams-bridge so streamed reads can be awaited.
+  //  3. Starts a long-poll drainer that pulls queued envelopes +
+  //     chunks from the server and dispatches via
+  //     `shade.acceptTransferEnvelope`.
+
+  let drainer: QueueDrainerHandle | null = null;
+  let streamsBridgePromise: Promise<ClientStreamsBridge> | null = null;
+  let streamsBridge: ClientStreamsBridge | null = null;
+
+  if (options.outboundQueueUrl !== undefined) {
+    const outboundQueueUrl = options.outboundQueueUrl;
+    if (options.transferBaseUrl === undefined) {
+      throw new Error(
+        'createFilesHttpClient: outboundQueueUrl was supplied without transferBaseUrl. Pass `transferBaseUrl` (the server prefix that hosts /v1/transfer/...) so outbound chunks have a destination.',
+      );
+    }
+    if (shade.configureTransfers === undefined) {
+      throw new Error(
+        'createFilesHttpClient: shade.configureTransfers is required for streamed mode (the underlying ShadeBridge must surface it).',
+      );
+    }
+    const transferBaseUrl = options.transferBaseUrl.replace(/\/$/, '');
+    shade.configureTransfers({
+      resolveBaseUrl: async (peer) => {
+        if (peer !== peerAddress) {
+          throw new Error(
+            `httpClient is bound to peer "${peerAddress}" — refusing to resolve outgoing chunks for "${peer}" without a multi-peer registry. Use shade.files.client(peer) for server-to-server multi-peer.`,
+          );
+        }
+        return transferBaseUrl;
+      },
+    });
+    // Build the streams-bridge eagerly. The engine's incoming-transfer
+    // subscription has to be in place BEFORE the drainer dispatches the
+    // first stream-init envelope, otherwise the engine emits the
+    // IncomingTransfer to zero handlers and the read silently never
+    // accepts. We kick off the drainer once the bridge has subscribed.
+    streamsBridgePromise = createClientStreamsBridge(shade).then((bridge) => {
+      streamsBridge = bridge;
+      drainer = startQueueDrainer(shade, {
+        outboundQueueUrl,
+        peerAddress,
+        senderAddress,
+        ...(options.fetch !== undefined ? { fetch: options.fetch } : {}),
+        ...(options.headers !== undefined ? { headers: options.headers } : {}),
+        ...(options.queueBlockMs !== undefined ? { blockMs: options.queueBlockMs } : {}),
+      });
+      return bridge;
+    });
+    // Surface bridge-construction failures eagerly via a rejected
+    // promise the next read/write picks up.
+    streamsBridgePromise.catch(() => {
+      /* observed via getStreamsBridge() */
+    });
+  }
+
+  async function getStreamsBridge(): Promise<ClientStreamsBridge> {
+    if (streamsBridge !== null) return streamsBridge;
+    if (streamsBridgePromise === null) {
+      throw new ConflictError(
+        `http RPC client supports inline writes/reads only (≤ ${INLINE_THRESHOLD} bytes) — pass { outboundQueueUrl, transferBaseUrl } to enable streamed transfers.`,
+      );
+    }
+    streamsBridge = await streamsBridgePromise;
+    return streamsBridge;
+  }
+
  /**
   * Encrypt + POST + decrypt + parse one RPC round-trip.
   *
@@ -321,20 +433,39 @@ export function createFilesHttpClient(
        ReadResultSchema,
        opts,
      );
-      if (wire.kind !== 'inline') {
-        // The HTTP RPC route does not service streamed reads — there is
-        // no place to stream from in pure request-response.
+      if (wire.kind === 'inline') {
+        const bytes = base64ToBytes(wire.bytesB64);
+        const out: ReadOutput = {
+          kind: 'inline',
+          bytes,
+          size: wire.size,
+          sha256: wire.sha256,
+          ...(wire.contentType !== undefined ? { contentType: wire.contentType } : {}),
+        };
+        return out;
+      }
+      // Streamed read — only supported when the queue drainer is wired.
+      if (drainer === null) {
        throw new InternalFileError(
-          `http RPC client received a streamed read (size ${wire.size}). Use shade.files.client(peer) on a server-to-server deployment, or pass { preferInline: true } when the file is known to fit inline.`,
+          `http RPC client received a streamed read (size ${wire.size}) but is in inline-only mode. Pass { outboundQueueUrl, transferBaseUrl } when constructing the client to enable streamed reads.`,
        );
      }
-      const bytes = base64ToBytes(wire.bytesB64);
+      const bridge = await getStreamsBridge();
+      const bridgeSignal = opts.signal ?? new AbortController().signal;
+      const parked = await bridge.awaitRead(wire.streamId, {
+        expectedFrom: peerAddress,
+        signal: bridgeSignal,
+        timeoutMs: ioTimeoutMs,
+      });
      const out: ReadOutput = {
-        kind: 'inline',
-        bytes,
+        kind: 'streams',
+        stream: parked.readable,
        size: wire.size,
        sha256: wire.sha256,
        ...(wire.contentType !== undefined ? { contentType: wire.contentType } : {}),
+        done: async () => {
+          await parked.done;
+        },
      };
      return out;
    },
@@ -344,25 +475,77 @@ export function createFilesHttpClient(
      const overwrite = opts.overwrite ?? false;
      const contentType = opts.contentType ?? decision.contentType;

-      if (decision.kind !== 'inline') {
-        throw new ConflictError(
-          `http RPC client supports inline writes only (≤ ${INLINE_THRESHOLD} bytes). The supplied input was promoted to streams (size ${decision.size ?? 'unknown'}). Use shade.files.client(peer) for streamed writes, or pre-buffer the input below the inline threshold.`,
+      if (decision.kind === 'inline' || opts.forceInline === true) {
+        const bytes = decision.kind === 'inline' ? decision.bytes : null;
+        if (bytes === null) {
+          // forceInline === true with a streams-typed decision —
+          // decideInline always produced a `streams` shape because the
+          // input was a bare ReadableStream. We can't drain a stream
+          // synchronously here without a streams-bridge.
+          throw new ConflictError(
+            'http RPC client cannot forceInline a streamed input — pass a Uint8Array / Blob, or pre-buffer the stream.',
+          );
+        }
+        if (bytes.byteLength > INLINE_THRESHOLD) {
+          throw new ConflictError(
+            `inline write exceeds ${INLINE_THRESHOLD}-byte threshold (got ${bytes.byteLength}); pass forceInline=true to override`,
+          );
+        }
+        const args = WriteArgsSchema.parse({
+          kind: 'inline',
+          path,
+          bytesB64: bytesToBase64(bytes),
+          ...(contentType !== undefined ? { contentType } : {}),
+          overwrite,
+        });
+        return await roundTrip<WriteResult>(
+          KIND_WRITE_V1,
+          'write',
+          args,
+          WriteResultSchema,
+          opts,
        );
      }
+
+      // Streamed write — requires the queue drainer + streams-bridge.
+      if (drainer === null) {
+        throw new ConflictError(
+          `http RPC client supports inline writes only (≤ ${INLINE_THRESHOLD} bytes). The supplied input was promoted to streams (size ${decision.size ?? 'unknown'}). Pass { outboundQueueUrl, transferBaseUrl } to enable streamed writes.`,
+        );
+      }
+      const bridge = await getStreamsBridge();
+      const size = decision.size;
+      if (size === undefined) {
+        throw new ConflictError(
+          'streams write requires a known plaintext size; pass `{ stream, size }` instead of a bare ReadableStream',
+        );
+      }
+      const { writeId, handle } = await bridge.initiateWrite({
+        peer: peerAddress,
+        stream: decision.stream,
+        size,
+        ...(contentType !== undefined ? { contentType } : {}),
+        name: path,
+        ...(opts.signal !== undefined ? { signal: opts.signal } : {}),
+      });
      const args = WriteArgsSchema.parse({
-        kind: 'inline',
+        kind: 'streams',
        path,
-        bytesB64: bytesToBase64(decision.bytes),
+        size,
        ...(contentType !== undefined ? { contentType } : {}),
        overwrite,
+        writeId,
      });
-      return await roundTrip<WriteResult>(
-        KIND_WRITE_V1,
-        'write',
-        args,
-        WriteResultSchema,
-        opts,
-      );
+      try {
+        const [result] = await Promise.all([
+          roundTrip<WriteResult>(KIND_WRITE_V1, 'write', args, WriteResultSchema, opts),
+          handle.done(),
+        ]);
+        return result;
+      } catch (err) {
+        await handle.abort('rpc-failed').catch(() => undefined);
+        throw err;
+      }
    },

    async getThumbnail(path, size: ThumbnailSize, opts): Promise<ThumbnailResult> {
@@ -393,7 +576,15 @@ export function createFilesHttpClient(
    },

    close(): void {
-      // Stateless — nothing to release. Exists for FileClient symmetry.
+      // Stop the long-poll drainer + tear down the streams-bridge if
+      // we built one. Idempotent — safe to call multiple times.
+      drainer?.stop();
+      drainer = null;
+      if (streamsBridge !== null) {
+        void streamsBridge.destroy().catch(() => undefined);
+        streamsBridge = null;
+      }
+      streamsBridgePromise = null;
    },
  } as FileClient;
 }
--- a/packages/shade-files/src/client/queue-drainer.ts
+++ b/packages/shade-files/src/client/queue-drainer.ts
@@ -0,0 +1,172 @@
+/**
+ * Browser-side drainer for the pull-mode outbound queue.
+ *
+ * Background task that long-polls the server's `/queue` endpoint,
+ * decodes each event, and dispatches it into the consumer's Shade
+ * instance via `shade.acceptTransferEnvelope`. Same wire-shape as the
+ * server-to-server case where the engine receives chunks via direct
+ * HTTP POSTs — we just flip the direction so the browser pulls
+ * instead of accepts.
+ */
+import type { ShadeBridge } from '../integration/shade-bridge.js';
+
+export interface QueueDrainerOptions {
+  /**
+   * Server endpoint that hosts `transferQueueRoute()`. Typically:
+   * `https://server.example.com/api/v1/shade-files/queue`.
+   */
+  outboundQueueUrl: string;
+  /** Peer the queue is pulled FROM (the server's address). */
+  peerAddress: string;
+  /** Address we identify ourselves with on the long-poll. */
+  senderAddress: string;
+  /** Optional `fetch` override. Default `globalThis.fetch`. */
+  fetch?: typeof globalThis.fetch;
+  /** Extra headers applied to every poll request. */
+  headers?: Record<string, string>;
+  /**
+   * Long-poll request timeout (server-side block). Default 30_000.
+   * Server clamps to its own `maxBlockMs` (default 55_000).
+   */
+  blockMs?: number;
+  /**
+   * Backoff after a network error before re-polling. Default 2_000.
+   * Doubles up to `maxBackoffMs` on consecutive failures.
+   */
+  initialBackoffMs?: number;
+  maxBackoffMs?: number;
+  /**
+   * Called when a poll cycle fails. Defaults to logging via `console.error`.
+   * Throwing from this hook does NOT stop the drainer — it backs off
+   * and retries.
+   */
+  onError?: (err: unknown) => void;
+}
+
+export interface QueueDrainerHandle {
+  /** Stop the drainer. Pending fetch is aborted; the loop exits. */
+  stop(): void;
+  /** Promise that resolves once the drainer has fully stopped. */
+  stopped: Promise<void>;
+}
+
+interface PolledEvent {
+  id: number;
+  timestampMs: number;
+  kind: 'envelope' | 'chunk';
+  bytesB64: string;
+  meta?: { streamId: string; laneId: number; seq: number };
+}
+
+interface PollResponseBody {
+  events: PolledEvent[];
+  nextSince: number;
+}
+
+const DEFAULT_BLOCK_MS = 30_000;
+const DEFAULT_INITIAL_BACKOFF_MS = 2_000;
+const DEFAULT_MAX_BACKOFF_MS = 30_000;
+
+/**
+ * Start a long-poll loop that drains queued envelopes + chunks from
+ * the server and dispatches them into the local Shade instance.
+ *
+ * Returns a handle the caller can use to stop the drainer when the
+ * `httpClient` is closed (e.g. on tab unload).
+ */
+export function startQueueDrainer(
+  shade: ShadeBridge,
+  options: QueueDrainerOptions,
+): QueueDrainerHandle {
+  if (shade.acceptTransferEnvelope === undefined) {
+    throw new Error(
+      'startQueueDrainer: shade.acceptTransferEnvelope is required for pull-mode streams. The supplied ShadeBridge implementation must surface it.',
+    );
+  }
+  const accept = shade.acceptTransferEnvelope.bind(shade);
+  const fetchFn = options.fetch ?? globalThis.fetch.bind(globalThis);
+  const blockMs = options.blockMs ?? DEFAULT_BLOCK_MS;
+  const onError = options.onError ?? ((err: unknown) => console.error('[shade.files queue-drainer]', err));
+  const initialBackoffMs = options.initialBackoffMs ?? DEFAULT_INITIAL_BACKOFF_MS;
+  const maxBackoffMs = options.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS;
+  const ac = new AbortController();
+  let stopped = false;
+  let resolveStopped!: () => void;
+  const stoppedPromise = new Promise<void>((r) => {
+    resolveStopped = r;
+  });
+
+  void (async () => {
+    let since = 0;
+    let backoff = initialBackoffMs;
+    while (!stopped) {
+      try {
+        const response = await fetchFn(options.outboundQueueUrl, {
+          method: 'POST',
+          signal: ac.signal,
+          headers: {
+            'Content-Type': 'application/json',
+            'X-Shade-Sender-Address': options.senderAddress,
+            ...(options.headers ?? {}),
+          },
+          body: JSON.stringify({ since, blockMs }),
+        });
+        if (!response.ok) {
+          throw new Error(`queue poll → ${response.status} ${response.statusText}`);
+        }
+        const body = (await response.json()) as PollResponseBody;
+        if (Array.isArray(body.events) && body.events.length > 0) {
+          for (const event of body.events) {
+            if (stopped) break;
+            try {
+              const bytes = base64ToBytes(event.bytesB64);
+              await accept(options.peerAddress, bytes);
+            } catch (err) {
+              // Per-event dispatch failure should not kill the loop —
+              // resume picks up missing chunks via @shade/transfer's
+              // built-in lane-resume protocol.
+              onError(err);
+            }
+          }
+        }
+        since = typeof body.nextSince === 'number' ? body.nextSince : since;
+        backoff = initialBackoffMs;
+      } catch (err) {
+        if (stopped || ac.signal.aborted) break;
+        onError(err);
+        // Exponential backoff with jitter — caps at maxBackoffMs.
+        const jitter = Math.floor(Math.random() * Math.min(backoff, 1_000));
+        await new Promise<void>((r) => {
+          const t = setTimeout(r, backoff + jitter);
+          (t as unknown as { unref?: () => void }).unref?.();
+          ac.signal.addEventListener(
+            'abort',
+            () => {
+              clearTimeout(t);
+              r();
+            },
+            { once: true },
+          );
+        });
+        backoff = Math.min(maxBackoffMs, backoff * 2);
+      }
+    }
+    resolveStopped();
+  })();
+
+  return {
+    stop(): void {
+      if (stopped) return;
+      stopped = true;
+      ac.abort(new Error('queue drainer stopped'));
+    },
+    stopped: stoppedPromise,
+  };
+}
+
+function base64ToBytes(b64: string): Uint8Array {
+  const bin = atob(b64);
+  const out = new Uint8Array(bin.length);
+  for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i);
+  return out;
+}
--- a/packages/shade-files/src/client/streams-bridge.ts
+++ b/packages/shade-files/src/client/streams-bridge.ts
@@ -102,7 +102,16 @@ export async function createClientStreamsBridge(
    const readStreamId = incoming.metadata.userMetadata?.[META_KEY_READ_STREAM_ID];
    if (readStreamId === undefined) return;

-    const ts = new TransformStream<Uint8Array, Uint8Array>();
+    // Generous HWM so the receiver-side write loop (drainer →
+    // engine.receiveChunk → sink.write) doesn't stall on backpressure
+    // before the consumer's reader is wired up. The reader still
+    // applies its own backpressure once it's consuming, but we no
+    // longer race fs.read's await on stream-init against the consumer
+    // attaching its reader.
+    const ts = new TransformStream<Uint8Array, Uint8Array>(undefined, undefined, {
+      highWaterMark: 64,
+      size: (chunk?: Uint8Array) => (chunk === undefined ? 0 : 1),
+    });
    let handle: TransferHandle;
    try {
      handle = await incoming.accept({
--- a/packages/shade-files/src/index.ts
+++ b/packages/shade-files/src/index.ts
@@ -190,6 +190,11 @@ export { createFilesRpcRoute } from './server/rpc-route.js';
 export type { FilesRpcRouteOptions } from './server/rpc-route.js';
 export { createFilesHttpClient } from './client/http-client.js';
 export type { FilesHttpClientOptions } from './client/http-client.js';
+export { startQueueDrainer } from './client/queue-drainer.js';
+export type {
+  QueueDrainerHandle,
+  QueueDrainerOptions,
+} from './client/queue-drainer.js';

 // Shared structural surface @shade/files needs from a Shade instance —
 // exposed so consumers building custom Shade-shaped bridges can verify
--- a/packages/shade-files/src/integration/shade-bridge.ts
+++ b/packages/shade-files/src/integration/shade-bridge.ts
@@ -70,4 +70,25 @@ export interface ShadeBridge {

  /** Optional control-envelope passthrough used by the WebRTC bridge. */
  deliverControlEnvelope?(peer: string, envelope: ShadeEnvelope): Promise<void>;
+
+  /**
+   * Hand a freshly-decoded wire envelope (control or chunk) to the
+   * transfer engine. Required by the pull-mode HTTP client when it
+   * drains queued events from the server: each polled chunk / control
+   * envelope is dispatched here so the engine sees it just as if it
+   * had arrived via an HTTP POST on `/v1/transfer/...`.
+   */
+  acceptTransferEnvelope?(from: string, env: ShadeEnvelope | Uint8Array): Promise<void>;
+
+  /**
+   * Configure the transfer stack. Called by the pull-mode HTTP client
+   * to point the browser's outgoing chunks + control envelopes at the
+   * server's transferQueueRoute mount. Optional because the
+   * server-to-server path uses a separate, app-driven configuration.
+   */
+  configureTransfers?(opts: {
+    resolveBaseUrl?: (peerAddress: string) => Promise<string>;
+    transport?: unknown;
+    envelopeTransport?: unknown;
+  }): void;
 }
--- a/packages/shade-files/tests/integration/http-rpc-streams.test.ts
+++ b/packages/shade-files/tests/integration/http-rpc-streams.test.ts
@@ -0,0 +1,208 @@
+import { describe, expect, test } from 'bun:test';
+import { createShade } from '@shade/sdk';
+import {
+  createPrekeyServer,
+  MemoryPrekeyStore,
+  PrekeyServerEvents,
+} from '@shade/server';
+import { SubtleCryptoProvider } from '@shade/crypto-web';
+import { Hono } from 'hono';
+
+const crypto = new SubtleCryptoProvider();
+
+/**
+ * Stand up the full pull-mode rig:
+ *   - Prekey server (for X3DH)
+ *   - Bob: file handler + rpcRoute + transferQueueRoute, all on one server
+ *   - Alice: httpClient with outboundQueueUrl + transferBaseUrl wired
+ *
+ * Returns Alice's `FileClient`, which speaks browser-style: ONE base URL,
+ * no inbound listener, streams supported via long-poll.
+ */
+async function setupPullRig(opts: {
+  bobHandler: Parameters<NonNullable<Awaited<ReturnType<typeof createShade>>['files']>['serve']>[0];
+}) {
+  const prekey = createPrekeyServer({
+    crypto,
+    store: new MemoryPrekeyStore(),
+    disableRateLimit: true,
+    events: new PrekeyServerEvents(),
+  });
+  const prekeyServer = Bun.serve({ port: 0, fetch: prekey.fetch });
+  const prekeyUrl = `http://localhost:${prekeyServer.port}`;
+
+  const alice = await createShade({ prekeyServer: prekeyUrl, address: 'alice' });
+  const bob = await createShade({ prekeyServer: prekeyUrl, address: 'bob' });
+
+  // Bob: queue-route FIRST (configures bob's transports), then files.serve.
+  const queueRoute = await bob.transferQueueRoute({ blockMs: 1_500 });
+  await bob.files.serve(opts.bobHandler);
+  const rpcRoute = bob.files.rpcRoute({ acceptFirstMessage: true });
+
+  const app = new Hono();
+  app.route('/', queueRoute);
+  app.route('/', rpcRoute);
+
+  const bobServer = Bun.serve({ port: 0, fetch: app.fetch });
+  const baseUrl = `http://localhost:${bobServer.port}`;
+
+  const fs = alice.files.httpClient('bob', {
+    rpcUrl: `${baseUrl}/rpc`,
+    outboundQueueUrl: `${baseUrl}/queue`,
+    transferBaseUrl: baseUrl,
+    defaultTimeoutMs: 10_000,
+    queueBlockMs: 1_000,
+  });
+
+  return {
+    alice,
+    bob,
+    fs,
+    baseUrl,
+    teardown: async () => {
+      fs.close();
+      await alice.shutdown();
+      await bob.shutdown();
+      bobServer.stop();
+      prekeyServer.stop();
+    },
+  };
+}
+
+describe('@shade/files HTTP RPC — pull-mode streams', () => {
+  test('streamed read (4 MiB) via long-poll queue', async () => {
+    const payload = new Uint8Array(4 * 1024 * 1024);
+    for (let i = 0; i < payload.length; i++) payload[i] = (i * 97) & 0xff;
+
+    const rig = await setupPullRig({
+      bobHandler: {
+        read: async () => {
+          // Return the payload as a streamed read so the rpc-handler
+          // promotes it via the streams-bridge into a transfer.
+          const stream = new ReadableStream<Uint8Array>({
+            start(controller) {
+              const CHUNK = 256 * 1024;
+              for (let off = 0; off < payload.byteLength; off += CHUNK) {
+                controller.enqueue(payload.slice(off, Math.min(off + CHUNK, payload.byteLength)));
+              }
+              controller.close();
+            },
+          });
+          // Need a precomputed sha256 for streamed reads. Use the
+          // crypto provider's sha256 directly.
+          const digest = new Uint8Array(await globalThis.crypto.subtle.digest('SHA-256', payload));
+          const sha256Hex = Array.from(digest, (b) => b.toString(16).padStart(2, '0')).join('');
+          return {
+            kind: 'streams' as const,
+            stream,
+            size: payload.byteLength,
+            sha256: sha256Hex,
+            contentType: 'application/octet-stream',
+          };
+        },
+      },
+    });
+
+    try {
+      const result = await rig.fs.read('/big.bin');
+      expect(result.kind).toBe('streams');
+      if (result.kind !== 'streams') return;
+
+      // Drain the stream and compare.
+      const reader = result.stream.getReader();
+      const got = new Uint8Array(payload.byteLength);
+      let offset = 0;
+      while (true) {
+        const { value, done } = await reader.read();
+        if (done) break;
+        if (value !== undefined) {
+          got.set(value, offset);
+          offset += value.byteLength;
+        }
+      }
+      reader.releaseLock();
+      await result.done();
+
+      expect(offset).toBe(payload.byteLength);
+      // Compare in 64KiB strides for speed.
+      let mismatch = -1;
+      for (let i = 0; i < payload.byteLength; i++) {
+        if (got[i] !== payload[i]) {
+          mismatch = i;
+          break;
+        }
+      }
+      expect(mismatch).toBe(-1);
+    } finally {
+      await rig.teardown();
+    }
+  }, 30_000);
+
+  test('streamed read fails with clear error when outboundQueueUrl is omitted', async () => {
+    const rig = await setupPullRig({
+      bobHandler: {
+        read: async () => {
+          const stream = new ReadableStream<Uint8Array>({
+            start(c) {
+              c.enqueue(new Uint8Array(512 * 1024));
+              c.close();
+            },
+          });
+          const digest = new Uint8Array(await globalThis.crypto.subtle.digest('SHA-256', new Uint8Array(512 * 1024)));
+          const sha256Hex = Array.from(digest, (b) => b.toString(16).padStart(2, '0')).join('');
+          return {
+            kind: 'streams' as const,
+            stream,
+            size: 512 * 1024,
+            sha256: sha256Hex,
+          };
+        },
+      },
+    });
+    // Tear down the rig's drainer so we can construct an inline-only client
+    rig.fs.close();
+
+    const inlineOnly = rig.alice.files.httpClient('bob', {
+      rpcUrl: `${rig.baseUrl}/rpc`,
+      defaultTimeoutMs: 10_000,
+    });
+    try {
+      await expect(inlineOnly.read('/big.bin')).rejects.toThrow(/streamed read/);
+    } finally {
+      inlineOnly.close();
+      await rig.teardown();
+    }
+  }, 15_000);
+
+  test('long-poll returns empty events on idle timeout', async () => {
+    const rig = await setupPullRig({
+      bobHandler: {
+        stat: async () => ({
+          name: '_',
+          kind: 'dir' as const,
+          size: 0,
+          mtime: 0,
+          metadata: {},
+        }),
+      },
+    });
+    try {
+      // Direct poll without any pending events — should return after blockMs.
+      const start = Date.now();
+      const res = await fetch(`${rig.baseUrl}/queue`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'X-Shade-Sender-Address': 'alice',
+        },
+        body: JSON.stringify({ since: 0, blockMs: 500 }),
+      });
+      expect(res.status).toBe(200);
+      const body = (await res.json()) as { events: unknown[]; nextSince: number };
+      expect(body.events).toHaveLength(0);
+      expect(Date.now() - start).toBeGreaterThanOrEqual(400);
+    } finally {
+      await rig.teardown();
+    }
+  }, 10_000);
+});