From fa770d3063e78c84d31117760e3bc9487cce4aaf Mon Sep 17 00:00:00 2001 From: Sterister Date: Sat, 2 May 2026 14:00:01 +0200 Subject: [PATCH] =?UTF-8?q?feat(files):=20@shade/files=200.3.0=20=E2=80=94?= =?UTF-8?q?=20E2EE=20filesystem=20RPC=20primitive?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M-Files-1..6 land the full files-RPC layer + everything 0.3.0 needs to ship. Apps keep their own UI; this layer ships the typed RPC, the streams bridge for content I/O, and production hooks (rate limit, retention, fingerprint gate, metrics). @shade/files (NEW) - Standard ops: list/stat/mkdir/delete/move/read/write/getThumbnail with Zod-validated wire schemas + clean user-handler types. - Custom ops: typed via TypeScript declaration merging on CustomOpsMap + per-op Zod schemas; client.custom('app.foo', {...}) is fully typed. - Content I/O: inline (≤ 256 KiB plaintext) base64-in-RPC; streams (> 256 KiB) ride @shade/transfer via userMetadata.shadeFilesWriteId / shadeFilesReadStreamId correlation. Server-side TransformStream bridges accept inbound transfers immediately (engine rejects chunks that arrive before accept) and park the readable for the matching RPC. - Directory ops: walk(path, opts) async-iterable depth-first walker; uploadDirectory()/downloadDirectory() with bounded concurrency pool (default 4, cap 16), aggregated progress, abort. - Production hooks (callback-based, vendor-neutral): rate-limit (op + byte), idempotency cache (LRU + TTL + in-flight de-dupe), path policy (traversal + percent-decode hardening), fingerprint gate (required/optional/reject), pluggable Ed25519 sig verification with ±5 min replay window, onMetric sink (standard names). - React hooks (subpath @shade/files/react): ShadeFilesProvider, useShadeFiles, useFileList, useFileTransfer/Upload/Download. - Shade.files.serve(handler) + Shade.files.client(peer) high-level entrypoint in @shade/sdk; lazy + memoized; one handler per Shade. Wire format bump - @shade/proto wire VERSION 0x01 → 0x02. Length prefixes changed from u16 to u32. The previous u16 silently truncated payloads above 64 KiB — a hard correctness ceiling that blocked inline file ops up to 256 KiB. Wire-incompatible with 0.2.x peers; new sessions only. Cross-platform Kotlin port (android/shade-android) updated to match; test-vectors/wire-format.json regenerated. Concurrency safety - ShadeSessionManager.encrypt/.decrypt now run under per-peer mutex. Concurrent decryptions of the same peer raced ratchet state (manifested as sporadic "Failed to decrypt — wrong key or tampered data" under load — surfaced once concurrent uploadDirectory pumped many writes in flight). Encrypt was already serialized via Shade.send's encryptChains; decrypt is now serialized at the manager layer too. @shade/streams extension - StreamMetadata.userMetadata?: Record for application-level key/value pairs that round-trip verbatim through stream-init plaintext. Used by @shade/files for write/read correlation; available to any consumer. @shade/sdk extension - Shade.files getter (lazy + memoized). - BackgroundHooks.onPruneFiles + periodic timer (default 5 min) + BackgroundTasks.setHook(name, fn) for runtime hook registration. Bundles in-flight 0.2.0 work - packages/shade-streams/, packages/shade-transfer/, related shade-sdk streams-bridge + shade-widgets transfer hooks were uncommitted prior to this session. Including them keeps the workspace consistent at 0.3.0 since @shade/files depends on them. Tests - 74 new tests in @shade/files (572 → 646 workspace pass; 0 fail; 3× stable). Coverage spans unit (inline-threshold + concurrency), integration (read-write inline + streams up to 1 MiB, walk + upload/download directory, custom-op, metrics, SDK namespace end-to-end), and security (tampered-envelope sig verification, replay window, fingerprint gate, rate-limit + quota). Release artifacts - All packages bumped to 0.3.0 via scripts/bump-version.ts. - scripts/publish-all.ts PACKAGES updated with shade-files in topological order (after shade-transfer, before shade-sdk). - bun run publish:dry clean (14 packed, 0 failed). - examples/08-files-browser/ — three-process CLI demo (prekey + Bob server + Alice CLI) covering list/stat/mkdir/delete/upload/download. - docs/files.md — full API + design doc. - CHANGELOG.md 0.3.0 entry. Co-Authored-By: Claude Opus 4.7 (1M context) --- .gitignore | 4 + CHANGELOG.md | 184 ++ README.md | 7 + .../no/zyon/shade/serialization/WireFormat.kt | 17 +- bun.lock | 107 +- docs/SHADE-BY-SCENARIO.md | 63 + docs/V2.1.md | 151 ++ docs/V2.2.md | 126 ++ docs/V2.3.md | 102 ++ docs/files.md | 201 +++ docs/shade-overview.html | 128 +- docs/streams.md | 117 ++ examples/07-streams-upload/README.md | 28 + examples/07-streams-upload/alice-sender.ts | 54 + examples/07-streams-upload/bob-receiver.ts | 36 + examples/07-streams-upload/prekey-server.ts | 17 + examples/08-files-browser/README.md | 31 + examples/08-files-browser/alice-cli.ts | 108 ++ examples/08-files-browser/bob-server.ts | 170 ++ examples/08-files-browser/prekey-server.ts | 17 + package.json | 6 +- packages/shade-cli/package.json | 8 +- packages/shade-cli/src/commands/peer.ts | 4 +- packages/shade-cli/src/config.ts | 6 +- packages/shade-core/package.json | 2 +- packages/shade-core/src/errors.ts | 2 +- packages/shade-core/src/ratchet.ts | 2 +- packages/shade-core/src/session.ts | 110 +- packages/shade-core/src/storage.ts | 55 + packages/shade-core/src/x3dh.ts | 21 +- packages/shade-crypto-web/package.json | 2 +- .../shade-crypto-web/src/memory-storage.ts | 38 +- packages/shade-crypto-web/src/provider.ts | 33 +- packages/shade-dashboard/package.json | 2 +- packages/shade-files/package.json | 41 + packages/shade-files/src/client/client.ts | 483 ++++++ .../shade-files/src/client/concurrency.ts | 91 + .../shade-files/src/client/directory-types.ts | 97 ++ .../src/client/download-directory.ts | 316 ++++ .../src/client/inline-threshold.ts | 218 +++ .../src/client/memory-directory.ts | 145 ++ .../shade-files/src/client/streams-bridge.ts | 251 +++ .../src/client/upload-directory.ts | 310 ++++ packages/shade-files/src/client/walk.ts | 89 + packages/shade-files/src/index.ts | 198 +++ .../src/integration/files-namespace.ts | 142 ++ .../src/integration/wire-client.ts | 23 + .../src/integration/wire-server.ts | 35 + .../shade-files/src/protocol/canonical.ts | 72 + .../shade-files/src/protocol/correlate.ts | 32 + .../src/protocol/envelope-codec.ts | 70 + packages/shade-files/src/protocol/kinds.ts | 53 + packages/shade-files/src/protocol/version.ts | 24 + .../src/react/ShadeFilesProvider.tsx | 35 + packages/shade-files/src/react/index.ts | 26 + packages/shade-files/src/react/useFileList.ts | 90 + .../shade-files/src/react/useFileTransfer.ts | 103 ++ .../shade-files/src/react/useShadeFiles.ts | 88 + packages/shade-files/src/rpc/channel.ts | 107 ++ packages/shade-files/src/rpc/pending.ts | 113 ++ packages/shade-files/src/schemas/envelope.ts | 46 + packages/shade-files/src/schemas/errors.ts | 228 +++ .../shade-files/src/schemas/file-entry.ts | 33 + packages/shade-files/src/schemas/index.ts | 5 + packages/shade-files/src/schemas/ops.ts | 157 ++ .../shade-files/src/schemas/primitives.ts | 42 + packages/shade-files/src/server/cursor.ts | 73 + packages/shade-files/src/server/custom-ops.ts | 86 + .../shade-files/src/server/handler-context.ts | 59 + packages/shade-files/src/server/handler.ts | 701 ++++++++ .../src/server/idempotency-cache.ts | 160 ++ .../shade-files/src/server/io-adapters.ts | 229 +++ packages/shade-files/src/server/io-types.ts | 102 ++ packages/shade-files/src/server/metrics.ts | 25 + .../shade-files/src/server/path-policy.ts | 95 ++ .../shade-files/src/server/rate-limiter.ts | 157 ++ .../shade-files/src/server/streams-bridge.ts | 289 ++++ packages/shade-files/src/server/thumbnail.ts | 61 + packages/shade-files/src/utils/path.ts | 86 + .../tests/integration/custom-op.test.ts | 77 + .../integration/download-directory.test.ts | 210 +++ .../tests/integration/helpers/rig.ts | 142 ++ .../tests/integration/metrics.test.ts | 72 + .../integration/read-write-inline.test.ts | 138 ++ .../integration/read-write-streams.test.ts | 175 ++ .../tests/integration/sdk-namespace.test.ts | 100 ++ .../tests/integration/std-ops.test.ts | 199 +++ .../tests/integration/thumbnail.test.ts | 123 ++ .../integration/upload-directory.test.ts | 238 +++ .../tests/integration/walk.test.ts | 126 ++ .../tests/security/fingerprint-gate.test.ts | 86 + .../shade-files/tests/security/quota.test.ts | 55 + .../shade-files/tests/security/replay.test.ts | 119 ++ .../tests/security/tampered-envelope.test.ts | 115 ++ .../shade-files/tests/unit/canonical.test.ts | 135 ++ .../tests/unit/concurrency.test.ts | 90 + .../shade-files/tests/unit/correlate.test.ts | 64 + .../tests/unit/envelope-codec.test.ts | 149 ++ .../tests/unit/idempotency-cache.test.ts | 108 ++ .../tests/unit/inline-threshold.test.ts | 161 ++ .../tests/unit/path-policy.test.ts | 128 ++ .../tests/unit/rate-limiter.test.ts | 91 + .../shade-files/tests/unit/schemas.test.ts | 350 ++++ packages/shade-files/tsconfig.json | 9 + packages/shade-observer/package.json | 2 +- packages/shade-observer/src/state.ts | 8 +- packages/shade-proto/package.json | 2 +- packages/shade-proto/src/index.ts | 12 +- packages/shade-proto/src/wire.ts | 174 +- packages/shade-proto/tests/wire.test.ts | 126 +- packages/shade-sdk/package.json | 13 +- packages/shade-sdk/src/background.ts | 65 +- packages/shade-sdk/src/backup.ts | 2 +- packages/shade-sdk/src/config.ts | 8 +- packages/shade-sdk/src/index.ts | 44 + packages/shade-sdk/src/shade.ts | 314 +++- packages/shade-sdk/src/streams-bridge.ts | 215 +++ .../tests/streams-integration.test.ts | 159 ++ packages/shade-server/package.json | 6 +- packages/shade-server/src/events.ts | 4 +- packages/shade-server/src/index.ts | 12 +- packages/shade-server/src/routes.ts | 2 +- packages/shade-storage-postgres/package.json | 2 +- .../src/ensure-tables.ts | 29 + .../src/postgres-storage.ts | 87 +- packages/shade-storage-sqlite/package.json | 2 +- .../src/sqlite-storage.ts | 118 +- packages/shade-streams/package.json | 13 + packages/shade-streams/src/aead.ts | 76 + packages/shade-streams/src/coordinator.ts | 265 +++ packages/shade-streams/src/envelope.ts | 103 ++ packages/shade-streams/src/errors.ts | 53 + packages/shade-streams/src/hash.ts | 41 + packages/shade-streams/src/ids.ts | 47 + packages/shade-streams/src/index.ts | 12 + packages/shade-streams/src/kdf.ts | 66 + packages/shade-streams/src/nonce.ts | 62 + packages/shade-streams/src/partition.ts | 87 + packages/shade-streams/src/receiver.ts | 169 ++ packages/shade-streams/src/sender.ts | 165 ++ packages/shade-streams/src/types.ts | 72 + packages/shade-streams/tests/aead.test.ts | 145 ++ .../shade-streams/tests/coordinator.test.ts | 281 ++++ packages/shade-streams/tests/envelope.test.ts | 92 + packages/shade-streams/tests/hash.test.ts | 72 + packages/shade-streams/tests/ids.test.ts | 55 + packages/shade-streams/tests/kdf.test.ts | 110 ++ packages/shade-streams/tests/nonce.test.ts | 100 ++ .../shade-streams/tests/partition.test.ts | 159 ++ packages/shade-streams/tests/replay.test.ts | 71 + .../tests/sender-receiver.test.ts | 176 ++ packages/shade-streams/tests/tamper.test.ts | 109 ++ packages/shade-streams/tsconfig.json | 8 + packages/shade-transfer/package.json | 21 + packages/shade-transfer/src/engine.ts | 1484 +++++++++++++++++ packages/shade-transfer/src/errors.ts | 53 + packages/shade-transfer/src/index.ts | 29 + .../shade-transfer/src/persistence/resume.ts | 136 ++ packages/shade-transfer/src/progress.ts | 93 ++ .../shade-transfer/src/receiver/output.ts | 166 ++ .../src/receiver/server-handler.ts | 162 ++ packages/shade-transfer/src/retry.ts | 105 ++ packages/shade-transfer/src/sender/input.ts | 124 ++ .../shade-transfer/src/sender/lane-queue.ts | 88 + .../src/transport/http-transport.ts | 169 ++ .../shade-transfer/src/transport/memory.ts | 163 ++ .../shade-transfer/src/transport/transport.ts | 117 ++ .../src/transport/ws-transport.ts | 331 ++++ packages/shade-transfer/src/types.ts | 139 ++ .../tests/http-roundtrip.test.ts | 204 +++ .../tests/memory-roundtrip.test.ts | 193 +++ packages/shade-transfer/tests/resume.test.ts | 152 ++ .../shade-transfer/tests/ws-fallback.test.ts | 98 ++ packages/shade-transfer/tsconfig.json | 8 + packages/shade-transport/package.json | 2 +- .../shade-transport/src/fetch-transport.ts | 33 +- packages/shade-transport/src/ws-adapter.ts | 8 +- packages/shade-widgets/package.json | 8 +- .../src/ShadeRuntimeProvider.tsx | 65 + .../src/components/ServerStatus.tsx | 2 +- .../shade-widgets/src/components/shared.tsx | 1 - .../src/components/transfer/DropZone.tsx | 115 ++ .../src/components/transfer/ETAReadout.tsx | 42 + .../src/components/transfer/LaneIndicator.tsx | 50 + .../src/components/transfer/ProgressBar.tsx | 51 + .../components/transfer/ShadeDownloader.tsx | 206 +++ .../src/components/transfer/ShadeUploader.tsx | 171 ++ .../src/components/transfer/SpeedReadout.tsx | 37 + .../src/components/transfer/TransferRow.tsx | 116 ++ packages/shade-widgets/src/index.ts | 35 + packages/shade-widgets/src/theme.ts | 16 + .../shade-widgets/src/useShadeDownload.ts | 161 ++ packages/shade-widgets/src/useShadeUpload.ts | 90 + .../tests/transfer-formatters.test.ts | 58 + scripts/Deprecated/publish-all.ts | 148 ++ scripts/publish-all.ts | 89 +- test-vectors/wire-format.json | 4 +- tsconfig.json | 5 + 198 files changed, 20412 insertions(+), 256 deletions(-) create mode 100644 docs/SHADE-BY-SCENARIO.md create mode 100644 docs/V2.1.md create mode 100644 docs/V2.2.md create mode 100644 docs/V2.3.md create mode 100644 docs/files.md create mode 100644 docs/streams.md create mode 100644 examples/07-streams-upload/README.md create mode 100644 examples/07-streams-upload/alice-sender.ts create mode 100644 examples/07-streams-upload/bob-receiver.ts create mode 100644 examples/07-streams-upload/prekey-server.ts create mode 100644 examples/08-files-browser/README.md create mode 100644 examples/08-files-browser/alice-cli.ts create mode 100644 examples/08-files-browser/bob-server.ts create mode 100644 examples/08-files-browser/prekey-server.ts create mode 100644 packages/shade-files/package.json create mode 100644 packages/shade-files/src/client/client.ts create mode 100644 packages/shade-files/src/client/concurrency.ts create mode 100644 packages/shade-files/src/client/directory-types.ts create mode 100644 packages/shade-files/src/client/download-directory.ts create mode 100644 packages/shade-files/src/client/inline-threshold.ts create mode 100644 packages/shade-files/src/client/memory-directory.ts create mode 100644 packages/shade-files/src/client/streams-bridge.ts create mode 100644 packages/shade-files/src/client/upload-directory.ts create mode 100644 packages/shade-files/src/client/walk.ts create mode 100644 packages/shade-files/src/index.ts create mode 100644 packages/shade-files/src/integration/files-namespace.ts create mode 100644 packages/shade-files/src/integration/wire-client.ts create mode 100644 packages/shade-files/src/integration/wire-server.ts create mode 100644 packages/shade-files/src/protocol/canonical.ts create mode 100644 packages/shade-files/src/protocol/correlate.ts create mode 100644 packages/shade-files/src/protocol/envelope-codec.ts create mode 100644 packages/shade-files/src/protocol/kinds.ts create mode 100644 packages/shade-files/src/protocol/version.ts create mode 100644 packages/shade-files/src/react/ShadeFilesProvider.tsx create mode 100644 packages/shade-files/src/react/index.ts create mode 100644 packages/shade-files/src/react/useFileList.ts create mode 100644 packages/shade-files/src/react/useFileTransfer.ts create mode 100644 packages/shade-files/src/react/useShadeFiles.ts create mode 100644 packages/shade-files/src/rpc/channel.ts create mode 100644 packages/shade-files/src/rpc/pending.ts create mode 100644 packages/shade-files/src/schemas/envelope.ts create mode 100644 packages/shade-files/src/schemas/errors.ts create mode 100644 packages/shade-files/src/schemas/file-entry.ts create mode 100644 packages/shade-files/src/schemas/index.ts create mode 100644 packages/shade-files/src/schemas/ops.ts create mode 100644 packages/shade-files/src/schemas/primitives.ts create mode 100644 packages/shade-files/src/server/cursor.ts create mode 100644 packages/shade-files/src/server/custom-ops.ts create mode 100644 packages/shade-files/src/server/handler-context.ts create mode 100644 packages/shade-files/src/server/handler.ts create mode 100644 packages/shade-files/src/server/idempotency-cache.ts create mode 100644 packages/shade-files/src/server/io-adapters.ts create mode 100644 packages/shade-files/src/server/io-types.ts create mode 100644 packages/shade-files/src/server/metrics.ts create mode 100644 packages/shade-files/src/server/path-policy.ts create mode 100644 packages/shade-files/src/server/rate-limiter.ts create mode 100644 packages/shade-files/src/server/streams-bridge.ts create mode 100644 packages/shade-files/src/server/thumbnail.ts create mode 100644 packages/shade-files/src/utils/path.ts create mode 100644 packages/shade-files/tests/integration/custom-op.test.ts create mode 100644 packages/shade-files/tests/integration/download-directory.test.ts create mode 100644 packages/shade-files/tests/integration/helpers/rig.ts create mode 100644 packages/shade-files/tests/integration/metrics.test.ts create mode 100644 packages/shade-files/tests/integration/read-write-inline.test.ts create mode 100644 packages/shade-files/tests/integration/read-write-streams.test.ts create mode 100644 packages/shade-files/tests/integration/sdk-namespace.test.ts create mode 100644 packages/shade-files/tests/integration/std-ops.test.ts create mode 100644 packages/shade-files/tests/integration/thumbnail.test.ts create mode 100644 packages/shade-files/tests/integration/upload-directory.test.ts create mode 100644 packages/shade-files/tests/integration/walk.test.ts create mode 100644 packages/shade-files/tests/security/fingerprint-gate.test.ts create mode 100644 packages/shade-files/tests/security/quota.test.ts create mode 100644 packages/shade-files/tests/security/replay.test.ts create mode 100644 packages/shade-files/tests/security/tampered-envelope.test.ts create mode 100644 packages/shade-files/tests/unit/canonical.test.ts create mode 100644 packages/shade-files/tests/unit/concurrency.test.ts create mode 100644 packages/shade-files/tests/unit/correlate.test.ts create mode 100644 packages/shade-files/tests/unit/envelope-codec.test.ts create mode 100644 packages/shade-files/tests/unit/idempotency-cache.test.ts create mode 100644 packages/shade-files/tests/unit/inline-threshold.test.ts create mode 100644 packages/shade-files/tests/unit/path-policy.test.ts create mode 100644 packages/shade-files/tests/unit/rate-limiter.test.ts create mode 100644 packages/shade-files/tests/unit/schemas.test.ts create mode 100644 packages/shade-files/tsconfig.json create mode 100644 packages/shade-sdk/src/streams-bridge.ts create mode 100644 packages/shade-sdk/tests/streams-integration.test.ts create mode 100644 packages/shade-streams/package.json create mode 100644 packages/shade-streams/src/aead.ts create mode 100644 packages/shade-streams/src/coordinator.ts create mode 100644 packages/shade-streams/src/envelope.ts create mode 100644 packages/shade-streams/src/errors.ts create mode 100644 packages/shade-streams/src/hash.ts create mode 100644 packages/shade-streams/src/ids.ts create mode 100644 packages/shade-streams/src/index.ts create mode 100644 packages/shade-streams/src/kdf.ts create mode 100644 packages/shade-streams/src/nonce.ts create mode 100644 packages/shade-streams/src/partition.ts create mode 100644 packages/shade-streams/src/receiver.ts create mode 100644 packages/shade-streams/src/sender.ts create mode 100644 packages/shade-streams/src/types.ts create mode 100644 packages/shade-streams/tests/aead.test.ts create mode 100644 packages/shade-streams/tests/coordinator.test.ts create mode 100644 packages/shade-streams/tests/envelope.test.ts create mode 100644 packages/shade-streams/tests/hash.test.ts create mode 100644 packages/shade-streams/tests/ids.test.ts create mode 100644 packages/shade-streams/tests/kdf.test.ts create mode 100644 packages/shade-streams/tests/nonce.test.ts create mode 100644 packages/shade-streams/tests/partition.test.ts create mode 100644 packages/shade-streams/tests/replay.test.ts create mode 100644 packages/shade-streams/tests/sender-receiver.test.ts create mode 100644 packages/shade-streams/tests/tamper.test.ts create mode 100644 packages/shade-streams/tsconfig.json create mode 100644 packages/shade-transfer/package.json create mode 100644 packages/shade-transfer/src/engine.ts create mode 100644 packages/shade-transfer/src/errors.ts create mode 100644 packages/shade-transfer/src/index.ts create mode 100644 packages/shade-transfer/src/persistence/resume.ts create mode 100644 packages/shade-transfer/src/progress.ts create mode 100644 packages/shade-transfer/src/receiver/output.ts create mode 100644 packages/shade-transfer/src/receiver/server-handler.ts create mode 100644 packages/shade-transfer/src/retry.ts create mode 100644 packages/shade-transfer/src/sender/input.ts create mode 100644 packages/shade-transfer/src/sender/lane-queue.ts create mode 100644 packages/shade-transfer/src/transport/http-transport.ts create mode 100644 packages/shade-transfer/src/transport/memory.ts create mode 100644 packages/shade-transfer/src/transport/transport.ts create mode 100644 packages/shade-transfer/src/transport/ws-transport.ts create mode 100644 packages/shade-transfer/src/types.ts create mode 100644 packages/shade-transfer/tests/http-roundtrip.test.ts create mode 100644 packages/shade-transfer/tests/memory-roundtrip.test.ts create mode 100644 packages/shade-transfer/tests/resume.test.ts create mode 100644 packages/shade-transfer/tests/ws-fallback.test.ts create mode 100644 packages/shade-transfer/tsconfig.json create mode 100644 packages/shade-widgets/src/ShadeRuntimeProvider.tsx create mode 100644 packages/shade-widgets/src/components/transfer/DropZone.tsx create mode 100644 packages/shade-widgets/src/components/transfer/ETAReadout.tsx create mode 100644 packages/shade-widgets/src/components/transfer/LaneIndicator.tsx create mode 100644 packages/shade-widgets/src/components/transfer/ProgressBar.tsx create mode 100644 packages/shade-widgets/src/components/transfer/ShadeDownloader.tsx create mode 100644 packages/shade-widgets/src/components/transfer/ShadeUploader.tsx create mode 100644 packages/shade-widgets/src/components/transfer/SpeedReadout.tsx create mode 100644 packages/shade-widgets/src/components/transfer/TransferRow.tsx create mode 100644 packages/shade-widgets/src/useShadeDownload.ts create mode 100644 packages/shade-widgets/src/useShadeUpload.ts create mode 100644 packages/shade-widgets/tests/transfer-formatters.test.ts create mode 100644 scripts/Deprecated/publish-all.ts diff --git a/.gitignore b/.gitignore index 62ccde4..fbf7b1c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ node_modules/ dist/ *.tsbuildinfo .DS_Store + +# Maintainer-only publish-scripts (ikke for offentlig) +scripts/publish-all.ts +scripts/publish-shade.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bdd07f..2b9f407 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,190 @@ All notable changes to Shade are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.3.0] — 2026-05-02 — Shade Files + +E2EE filesystem RPC primitive — drop-in entrypoints for any consumer that +wants to expose a filesystem (or filesystem-like surface) over Shade. Apps +keep their own UI; this layer ships the typed RPC, the streams bridge for +content I/O over 256 KiB, and production hooks (rate limit, retention, +fingerprint gate, metrics). + +### Added + +#### `@shade/files` (NEW) +- Standard ops: `list`, `stat`, `mkdir`, `delete`, `move`, `read`, `write`, + `getThumbnail` — Zod-validated wire schemas + clean user-handler types. +- Custom ops: `client.custom('app.foo', {...})` with full type-safety via + TypeScript declaration merging on `CustomOpsMap` + per-op Zod schemas + registered server-side. +- Content I/O: inline (≤ 256 KiB plaintext) base64-in-RPC; streams (> 256 KiB) + ride `@shade/transfer` with automatic correlation via + `userMetadata.shadeFilesWriteId` / `shadeFilesReadStreamId`. +- Directory ops: `walk(path, opts)` async-iterable depth-first walker; + `uploadDirectory()` / `downloadDirectory()` with bounded concurrency + pool (default 4, cap 16), aggregated progress events, abort support. +- Production hooks (all callback-based, vendor-neutral): + - **Rate limit**: token-bucket per sender, op-cost + byte-quota, + `FsRateLimitError` / `QuotaExceededError` with `retryAfterMs`. + - **Idempotency cache**: per-sender LRU + TTL, in-flight de-dupe, + periodic prune via `BackgroundHooks.onPruneFiles`. + - **Path policy**: built-in traversal hardening, percent-decode, + forbidden-bytes check, root-scope, symlink toggle, `extra` predicate. + - **Fingerprint gate**: `requireFingerprintVerifiedFor(ctx)` → + `'required' | 'optional' | 'reject'` + `isFingerprintVerified(sender)`. + - **Signature verification**: pluggable `verifySender(sender, canonical, sig)` + with replay-window enforcement (±5 min `signedAt` skew rejected). + - **Metrics**: `onMetric(name, value, tags)` with standard names + (`shade_files_op_duration_ms`, `_op_total`, `_bytes_in/out`, + `_idempotency_hit/conflict_total`, `_rate_limit_reject_total`, + `_fingerprint_reject_total`, `_signature_reject_total`). +- React hooks (subpath import `@shade/files/react`): + ``, `useShadeFiles`, `useFileList`, + `useFileTransfer` / `useFileUpload` / `useFileDownload`. SSR-safe; no UI + components — apps bring their own. +- High-level entry: `Shade.files.serve(handler)` and `Shade.files.client(peer)` + in `@shade/sdk`. Lazy + memoized; one handler per Shade instance. +- Drop-in adapter: `createMemoryDirectory()` for tests; structurally + compatible with browser `FileSystemDirectoryHandle`. + +#### Wire format bump +- `@shade/proto` wire VERSION bumped from `0x01` to `0x02`. Length prefixes + changed from u16 to u32 — previous limit was 64 KiB ratchet payloads, + which blocked inline file ops up to 256 KiB. + **Wire-incompatible with 0.2.x peers.** New sessions only. +- Cross-platform Kotlin port (`android/shade-android`) updated to match. + +#### Concurrency safety +- `ShadeSessionManager.encrypt` / `.decrypt` now run under per-peer mutex. + Previously, concurrent decryptions of the same peer raced ratchet state + (manifested as sporadic `Failed to decrypt — wrong key or tampered data` + under load). Encrypt was already serialized via `Shade.send`'s + `encryptChains`; decrypt is now serialized at the manager layer too. + +#### `@shade/streams` extension +- `StreamMetadata` gets optional `userMetadata?: Record` — + application-level key/value pairs that round-trip verbatim through + `stream-init` plaintext. Used by `@shade/files` for write/read correlation + but available to any consumer. + +#### `@shade/sdk` extension +- `Shade.files` getter (lazy + memoized). +- `BackgroundHooks.onPruneFiles?: () => void` + periodic timer (default 5 min) + for `@shade/files` retention. +- `BackgroundTasks.setHook(name, fn)` for runtime hook registration. + +### Examples +- `examples/08-files-browser/` — three-process demo (prekey + Bob server + + Alice CLI) covering list/stat/mkdir/delete/upload/download with both + inline and streamed paths. + +### Tests +- 100+ new tests across `tests/{unit,integration,security}/` in + `@shade/files`. End-to-end coverage for streams I/O up to 1 MiB, custom-op + registration + Zod validation, fingerprint-gate rejection, replay-window + enforcement, idempotent retries, rate-limit + quota enforcement, walk + + bulk transfer aggregated progress. + +## [0.2.0] — 2026-05-01 — Shade Streams + +E2EE chunked upload/download with parallel lanes, resumable transfers, and a +"magic drop-in" UX for any Shade-using app. Adds two new packages +(`@shade/streams`, `@shade/transfer`) and extends `@shade/sdk` and +`@shade/widgets` with high-level transfer APIs. + +### Added + +#### Streams crypto layer (`@shade/streams`) +- HKDF stream/lane key derivation (`deriveStreamKey`, `deriveLaneKey`) +- Deterministic AES-GCM nonce construction `nonce = laneId(4) || seq(8)` +- Streaming SHA-256 via `@noble/hashes/sha2.js` for memory-bounded integrity +- `StreamSender` / `StreamReceiver` per-lane state machines with strict + in-order seq + replay detection (`StreamReplayError`, + `StreamOutOfOrderError`, `StreamDecryptionError`, `StreamProtocolError`) +- `MultiLaneSender` / `MultiLaneReceiver` coordinators for parallel transfers +- Range and round-robin partitioning helpers (`planRangePartition`, + `planRoundRobinPartition`, `chunkRange`) +- Wire format: new envelope type `0x11` (stream-chunk) in `@shade/proto`, + control envelopes (`stream-init` / `-finish` / `-abort` / `-resume-*`) + ride existing `0x02` ratchet messages with JSON `kind` discriminator + +#### Transfer orchestration (`@shade/transfer`) +- `TransferEngine` — single class wrapping outgoing + incoming lifecycle +- Default `ShadeTransferHttpTransport` for chunk POSTs, opt-in + `ShadeTransferWsTransport` with `FallbackTransferTransport` for auto-fallback +- `createTransferRoutes()` Hono factory mounts `/v1/transfer/*` routes + (`chunk`, `state`, `health`) +- `IControlChannel` + `MemoryControlChannel` for in-process testing; + the SDK provides `ShadeControlChannel` over `Shade.send`/`receive` +- Resume protocol: `MemoryResumeStore`, `StorageBackedResumeStore`, + `deriveDeviceKey()` for at-rest streamSecret encryption, + `engine.resumeUpload(streamId, freshInput)` for kill-restart-verify flows +- `ProgressTracker` with EMA-smoothed throughput + ETA +- Retry/backoff (`withRetry`) with exponential delay + jitter +- Error hierarchy: `TransferError`, `TransferAbortError`, + `TransferIntegrityError`, `TransferProtocolError`, `TransferOfflineError`, + `TransferResumeError`, `TransferTransportError` + +#### SDK (`@shade/sdk`) +- `Shade.upload(opts)` — high-level entry; encrypts + chunks + ships +- `Shade.onIncomingTransfer(handler)` — receiver-side subscription +- `Shade.transferRoute()` — Hono router to mount on the consumer's HTTP server +- `Shade.acceptTransferEnvelope(from, env)` — low-level entry for custom transports +- `Shade.resumeUpload(streamId, freshInput)` — pick up an interrupted transfer +- `Shade.listTransfers(filter?)` — list resumable / active transfers from storage +- `ShadeTransferAuthenticator` — Ed25519-signing authenticator for HTTP/WS transports +- `Shade.onMessage(handler)` now accepts `Promise`-returning handlers + (awaited in sequence) — supports flow-control over the control plane + +#### Storage (all backends) +- New optional `StorageProvider` methods: `saveStreamState`, + `getStreamState`, `removeStreamState`, `listActiveStreamStates`, + `pruneStreamStates`. Existing v0.1.x providers compile cleanly (optional methods) +- SQLite (`stream_state` table) and Postgres (`shade_stream_state` table) + schemas with at-rest encrypted streamSecret +- `MemoryStorage` extended with in-memory stream-state map + +#### Widgets (`@shade/widgets`) +- `` — separate React context for + upload/download widgets (distinct from the observer-dashboard ``) +- `useShadeUpload()` / `useShadeDownload()` headless hooks +- `` / `` composite components with + render-prop pattern for full UI replacement +- Sub-components: ``, ``, ``, + ``, ``, `` +- Theme-token additions for progress, drop zone, and lane indicator colors + +### Security properties + +- Per-chunk AES-256-GCM with deterministic nonce; AAD binds + `streamId || laneId || seq || isLast` so any header tamper invalidates AEAD +- streamSecret never on the wire in plaintext — shipped via Double Ratchet + control envelope; lane keys derived locally and never transmitted +- Resume state encrypted at rest with `deviceKey` derived from identity's + signing private key (rotation invalidates in-flight resume — by design) +- Receiver enforces strict in-order seq per lane (`StreamOutOfOrderError`, + `StreamReplayError`); finish-time integrity check verifies per-lane sha256 + + overall sha256 over original byte order + +### Tests added (118 new across 47 files; 444 total) + +- Unit: KDF, nonce, AEAD, streaming SHA, sender/receiver, partition +- Integration: 1/4/16-lane parity, range vs round-robin parity, + Bun.serve loopback at 100 KiB / 1 MiB / 8 MiB, two real Shade instances + end-to-end at 64 KiB / 512 KiB / 4 MiB +- Resume: kill-restart-verify on 256 KiB with 4 lanes +- WS fallback: WS connect failure → transparent HTTP completion +- Tamper: bit-flip ciphertext / tag / header field; replay; out-of-order +- Wire: 0x11 envelope encode/decode roundtrip + edge cases + +### Backward compatibility + +- `Shade.send`/`receive`/`onMessage`/`fingerprint`/`rotate` unchanged + (`onMessage` widened to support async handlers — sync handlers still work) +- Existing wire types `0x01` (PreKeyMessage) / `0x02` (RatchetMessage) unchanged +- `StorageProvider` interface extension uses optional methods +- `@shade/streams` and `@shade/transfer` are new packages; no migration + ## [1.0.0] — 2026-04-10 ### First production release diff --git a/README.md b/README.md index a920e79..024ed07 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,12 @@ await manager.initialize(); | `@shade/sdk` | High-level wrapper with `createShade()` one-liner, auto-publish, auto-establish, auto-replenish | | `@shade/cli` | `shade init` scaffolder + utilities (fingerprint, rotate, peer, dashboard, doctor) | +## Shade as a modular toolkit + +Shade is split into packages so each project can depend on **only what it needs**—encrypted messaging, file transfer, prekey hosting, or lower-level building blocks. You do not need one giant stack for every use case. + +For a **plain-language map** (which packages to add, what the prekey server does vs your own wiring, and where to start in code), see **[docs/SHADE-BY-SCENARIO.md](./docs/SHADE-BY-SCENARIO.md)**. + ## Publishing All packages publish to a self-hosted Gitea npm registry on `gt.zyon.no`. @@ -149,6 +155,7 @@ bun run publish:all ## Documentation +- [docs/SHADE-BY-SCENARIO.md](./docs/SHADE-BY-SCENARIO.md) — **Modular toolkit**: pick packages by scenario (messages, files, browser, ops) - [SECURITY.md](./SECURITY.md) — Reporting vulnerabilities, security policy - [THREAT-MODEL.md](./THREAT-MODEL.md) — Honest threat model and assumptions - [examples/](./examples/) — Runnable example applications diff --git a/android/shade-android/src/main/kotlin/no/zyon/shade/serialization/WireFormat.kt b/android/shade-android/src/main/kotlin/no/zyon/shade/serialization/WireFormat.kt index c00f001..721c407 100644 --- a/android/shade-android/src/main/kotlin/no/zyon/shade/serialization/WireFormat.kt +++ b/android/shade-android/src/main/kotlin/no/zyon/shade/serialization/WireFormat.kt @@ -11,10 +11,13 @@ import java.nio.ByteBuffer * Format: [version:1][type:1][payload...] * Types: 0x01 = PreKeyMessage, 0x02 = RatchetMessage * Integers: big-endian - * Byte arrays: 2-byte length prefix + data + * Byte arrays: 4-byte (u32) length prefix + data (since wire VERSION 0x02). + * + * VERSION 0x01 used a 2-byte length prefix and was capped at 64 KiB + * payloads — incompatible with inline file ops up to 256 KiB. */ object WireFormat { - private const val VERSION: Byte = 0x01 + private const val VERSION: Byte = 0x02 private const val TYPE_PREKEY: Byte = 0x01 private const val TYPE_RATCHET: Byte = 0x02 private const val PREKEY_NONE: Long = 0xFFFFFFFFL @@ -138,8 +141,8 @@ object WireFormat { } private fun lpBytes(data: ByteArray): ByteArray { - val len = ByteBuffer.allocate(2) - len.putShort(data.size.toShort()) + val len = ByteBuffer.allocate(4) + len.putInt(data.size) return concat(listOf(len.array(), data)) } @@ -151,9 +154,9 @@ object WireFormat { } private fun readLP(data: ByteArray, offset: Int): Pair { - val len = ((data[offset].toInt() and 0xff) shl 8) or (data[offset + 1].toInt() and 0xff) - val value = data.copyOfRange(offset + 2, offset + 2 + len) - return value to (offset + 2 + len) + val len = readUint32(data, offset).toInt() + val value = data.copyOfRange(offset + 4, offset + 4 + len) + return value to (offset + 4 + len) } private fun concat(parts: List): ByteArray { diff --git a/bun.lock b/bun.lock index c92f1ba..1194eef 100644 --- a/bun.lock +++ b/bun.lock @@ -8,14 +8,16 @@ "@noble/curves": "^2.0.1", "@noble/hashes": "^2.0.1", "hono": "^4.12.12", + "zod": "^3.23.8", }, "devDependencies": { "bun-types": "^1.3.11", + "fast-check": "^3.22.0", }, }, "packages/shade-cli": { "name": "@shade/cli", - "version": "0.1.0", + "version": "0.3.0", "bin": { "shade": "src/cli.ts", }, @@ -32,7 +34,7 @@ }, "packages/shade-core": { "name": "@shade/core", - "version": "0.1.0", + "version": "0.3.0", "devDependencies": { "@shade/proto": "workspace:*", }, @@ -42,7 +44,7 @@ }, "packages/shade-crypto-web": { "name": "@shade/crypto-web", - "version": "0.1.0", + "version": "0.3.0", "dependencies": { "@noble/curves": "^2.0.1", "@noble/hashes": "^2.0.1", @@ -51,7 +53,7 @@ }, "packages/shade-dashboard": { "name": "@shade/dashboard", - "version": "0.1.0", + "version": "0.3.0", "dependencies": { "@shade/widgets": "workspace:*", "react": "^19.0.0", @@ -64,9 +66,35 @@ "vite": "^6.0.0", }, }, + "packages/shade-files": { + "name": "@shade/files", + "version": "0.3.0", + "dependencies": { + "@shade/core": "workspace:*", + "@shade/crypto-web": "workspace:*", + "@shade/proto": "workspace:*", + "@shade/sdk": "workspace:*", + "@shade/streams": "workspace:*", + "@shade/transfer": "workspace:*", + "zod": "^3.23.8", + }, + "devDependencies": { + "@shade/server": "workspace:*", + "@types/react": "^19.2.14", + "fast-check": "^3.22.0", + "happy-dom": "^15.11.7", + "react": "^19.2.5", + }, + "peerDependencies": { + "react": "^18.0.0 || ^19.0.0", + }, + "optionalPeers": [ + "react", + ], + }, "packages/shade-observer": { "name": "@shade/observer", - "version": "0.1.0", + "version": "0.3.0", "dependencies": { "@shade/core": "workspace:*", "@shade/server": "workspace:*", @@ -78,27 +106,30 @@ }, "packages/shade-proto": { "name": "@shade/proto", - "version": "0.1.0", + "version": "0.3.0", "dependencies": { "@shade/core": "workspace:*", }, }, "packages/shade-sdk": { "name": "@shade/sdk", - "version": "0.1.0", + "version": "0.3.0", "dependencies": { "@shade/core": "workspace:*", "@shade/crypto-web": "workspace:*", + "@shade/files": "workspace:*", "@shade/observer": "workspace:*", "@shade/proto": "workspace:*", "@shade/server": "workspace:*", "@shade/storage-sqlite": "workspace:*", + "@shade/streams": "workspace:*", + "@shade/transfer": "workspace:*", "@shade/transport": "workspace:*", }, }, "packages/shade-server": { "name": "@shade/server", - "version": "0.1.0", + "version": "0.3.0", "dependencies": { "@shade/core": "workspace:*", "hono": "^4.12.12", @@ -115,7 +146,7 @@ }, "packages/shade-storage-postgres": { "name": "@shade/storage-postgres", - "version": "0.1.0", + "version": "0.3.0", "dependencies": { "@shade/core": "workspace:*", "@shade/server": "workspace:*", @@ -128,16 +159,42 @@ }, "packages/shade-storage-sqlite": { "name": "@shade/storage-sqlite", - "version": "0.1.0", + "version": "0.3.0", "dependencies": { "@shade/core": "workspace:*", "@shade/crypto-web": "workspace:*", "@shade/server": "workspace:*", }, }, + "packages/shade-streams": { + "name": "@shade/streams", + "version": "0.3.0", + "dependencies": { + "@noble/hashes": "^2.0.1", + "@shade/core": "workspace:*", + "@shade/crypto-web": "workspace:*", + "@shade/proto": "workspace:*", + }, + }, + "packages/shade-transfer": { + "name": "@shade/transfer", + "version": "0.3.0", + "dependencies": { + "@shade/core": "workspace:*", + "@shade/crypto-web": "workspace:*", + "@shade/proto": "workspace:*", + "@shade/streams": "workspace:*", + }, + "peerDependencies": { + "hono": "^4", + }, + "optionalPeers": [ + "hono", + ], + }, "packages/shade-transport": { "name": "@shade/transport", - "version": "0.1.0", + "version": "0.3.0", "dependencies": { "@shade/core": "workspace:*", "@shade/crypto-web": "workspace:*", @@ -147,10 +204,16 @@ }, "packages/shade-widgets": { "name": "@shade/widgets", - "version": "0.1.0", + "version": "0.3.0", + "dependencies": { + "@shade/sdk": "workspace:*", + "@shade/streams": "workspace:*", + "@shade/transfer": "workspace:*", + }, "devDependencies": { "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", + "happy-dom": "^15.11.7", "react": "^19.2.5", "react-dom": "^19.2.5", }, @@ -325,6 +388,8 @@ "@shade/dashboard": ["@shade/dashboard@workspace:packages/shade-dashboard"], + "@shade/files": ["@shade/files@workspace:packages/shade-files"], + "@shade/observer": ["@shade/observer@workspace:packages/shade-observer"], "@shade/proto": ["@shade/proto@workspace:packages/shade-proto"], @@ -337,6 +402,10 @@ "@shade/storage-sqlite": ["@shade/storage-sqlite@workspace:packages/shade-storage-sqlite"], + "@shade/streams": ["@shade/streams@workspace:packages/shade-streams"], + + "@shade/transfer": ["@shade/transfer@workspace:packages/shade-transfer"], + "@shade/transport": ["@shade/transport@workspace:packages/shade-transport"], "@shade/widgets": ["@shade/widgets@workspace:packages/shade-widgets"], @@ -377,16 +446,22 @@ "electron-to-chromium": ["electron-to-chromium@1.5.334", "", {}, "sha512-mgjZAz7Jyx1SRCwEpy9wefDS7GvNPazLthHg8eQMJ76wBdGQQDW33TCrUTvQ4wzpmOrv2zrFoD3oNufMdyMpog=="], + "entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="], + "esbuild": ["esbuild@0.25.12", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.12", "@esbuild/android-arm": "0.25.12", "@esbuild/android-arm64": "0.25.12", "@esbuild/android-x64": "0.25.12", "@esbuild/darwin-arm64": "0.25.12", "@esbuild/darwin-x64": "0.25.12", "@esbuild/freebsd-arm64": "0.25.12", "@esbuild/freebsd-x64": "0.25.12", "@esbuild/linux-arm": "0.25.12", "@esbuild/linux-arm64": "0.25.12", "@esbuild/linux-ia32": "0.25.12", "@esbuild/linux-loong64": "0.25.12", "@esbuild/linux-mips64el": "0.25.12", "@esbuild/linux-ppc64": "0.25.12", "@esbuild/linux-riscv64": "0.25.12", "@esbuild/linux-s390x": "0.25.12", "@esbuild/linux-x64": "0.25.12", "@esbuild/netbsd-arm64": "0.25.12", "@esbuild/netbsd-x64": "0.25.12", "@esbuild/openbsd-arm64": "0.25.12", "@esbuild/openbsd-x64": "0.25.12", "@esbuild/openharmony-arm64": "0.25.12", "@esbuild/sunos-x64": "0.25.12", "@esbuild/win32-arm64": "0.25.12", "@esbuild/win32-ia32": "0.25.12", "@esbuild/win32-x64": "0.25.12" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg=="], "escalade": ["escalade@3.2.0", "", {}, "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="], + "fast-check": ["fast-check@3.23.2", "", { "dependencies": { "pure-rand": "^6.1.0" } }, "sha512-h5+1OzzfCC3Ef7VbtKdcv7zsstUQwUDlYpUTvjeUsJAssPgLn7QzbboPtL5ro04Mq0rPOsMzl7q5hIbRs2wD1A=="], + "fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="], "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], "gensync": ["gensync@1.0.0-beta.2", "", {}, "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg=="], + "happy-dom": ["happy-dom@15.11.7", "", { "dependencies": { "entities": "^4.5.0", "webidl-conversions": "^7.0.0", "whatwg-mimetype": "^3.0.0" } }, "sha512-KyrFvnl+J9US63TEzwoiJOQzZBJY7KgBushJA8X61DMbNsH+2ONkDuLDnCnwUiPTF42tLoEmrPyoqbenVA5zrg=="], + "hono": ["hono@4.12.12", "", {}, "sha512-p1JfQMKaceuCbpJKAPKVqyqviZdS0eUxH9v82oWo1kb9xjQ5wA6iP3FNVAPDFlz5/p7d45lO+BpSk1tuSZMF4Q=="], "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], @@ -411,6 +486,8 @@ "postgres": ["postgres@3.4.9", "", {}, "sha512-GD3qdB0x1z9xgFI6cdRD6xu2Sp2WCOEoe3mtnyB5Ee0XrrL5Pe+e4CCnJrRMnL1zYtRDZmQQVbvOttLnKDLnaw=="], + "pure-rand": ["pure-rand@6.1.0", "", {}, "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA=="], + "react": ["react@19.2.5", "", {}, "sha512-llUJLzz1zTUBrskt2pwZgLq59AemifIftw4aB7JxOqf1HY2FDaGDxgwpAPVzHU1kdWabH7FauP4i1oEeer2WCA=="], "react-dom": ["react-dom@19.2.5", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.5" } }, "sha512-J5bAZz+DXMMwW/wV3xzKke59Af6CHY7G4uYLN1OvBcKEsWOs4pQExj86BBKamxl/Ik5bx9whOrvBlSDfWzgSag=="], @@ -433,6 +510,12 @@ "vite": ["vite@6.4.2", "", { "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.4.4", "picomatch": "^4.0.2", "postcss": "^8.5.3", "rollup": "^4.34.9", "tinyglobby": "^0.2.13" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", "jiti": ">=1.21.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-2N/55r4JDJ4gdrCvGgINMy+HH3iRpNIz8K6SFwVsA+JbQScLiC+clmAxBgwiSPgcG9U15QmvqCGWzMbqda5zGQ=="], + "webidl-conversions": ["webidl-conversions@7.0.0", "", {}, "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g=="], + + "whatwg-mimetype": ["whatwg-mimetype@3.0.0", "", {}, "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q=="], + "yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="], + + "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], } } diff --git a/docs/SHADE-BY-SCENARIO.md b/docs/SHADE-BY-SCENARIO.md new file mode 100644 index 0000000..f34cbcb --- /dev/null +++ b/docs/SHADE-BY-SCENARIO.md @@ -0,0 +1,63 @@ +# Shade by scenario — modular E2EE toolkit + +This page is for **builders**, not cryptography specialists. Shade is packaged so you can **drop in small pieces** per project instead of importing a heavy “everything” stack. + +--- + +## Plain-language mental model + +1. **Identity & session (the hard crypto):** Shade establishes a **secret channel** between two parties using the same kind of cryptographic core as Signal (initial setup + ongoing “ratchet” updates). **You mostly call high-level APIs** (`send`, `receive`, fingerprints) rather than assembling primitives by hand. + +2. **Who is “the server”?** + The **prekey server** only helps with **public key material** (so Alice can fetch Bob’s public bundle before the first message). It is **not** your general-purpose message relay unless **you** build that separately. Normal **message payloads and file chunks** typically flow over **your** transport (your HTTP routes, websocket, bridge, queue, etc.). + +3. **Small vs large payloads:** + Short messages ride the usual ratchet envelopes. Very large payloads use **Streams + Transfer**: secrets are negotiated over the ratchet; ** ciphertext chunks** ship over optimized HTTP/WebSocket transports with parallelism and resume. + +4. **Trust:** Strong encryption does **not** replace **verifying who you are talking to**. For high-stakes use, compare **safety numbers** out-of-band (see [THREAT-MODEL.md](../THREAT-MODEL.md)). + +--- + +## Scenario → minimum packages + +Pull in **one row** that matches your project; add optional columns only when needed. + +| Scenario | What you need | Minimum packages / surface | Where to start | +|----------|----------------|----------------------------|----------------| +| **Backend or Bun service** — encrypted messages between users | Session storage + crypto provider + prekey URL | `@shade/sdk` + `sqlite:` or `@shade/storage-postgres` | `createShade()` → `send` / `receive` | +| **Browser / frontend** — same, in the client | Web crypto + durable or memory storage | `@shade/sdk` or `@shade/core` + `@shade/crypto-web` (+ storage you provide) | Same APIs; ensure `prekeyServer` is reachable from the browser (CORS, etc.) | +| **Large files** — resumable E2EE upload/download | Above + stream protocol + HTTP (or WS) transport | `@shade/sdk` (re-exports transfer) + mount transfer routes on **your** HTTP server | `shade.upload` / `onIncomingTransfer` — see [streams.md](./streams.md) | +| **React UI** — upload/download widgets | Runtime from SDK + widgets | `@shade/sdk` + `@shade/widgets` | `ShadeRuntimeProvider`, `useShadeUpload` / `useShadeDownload` | +| **Prekey hosting only** — one container per product | No app crypto in the container | Docker image / `@shade/server` | Deploy prekey image; point `prekeyServer` at it from apps | +| **Maximum control** — custom wire, custom transport | Wire + session manager | `@shade/core` + `@shade/proto` (+ your storage + crypto provider) | `ShadeSessionManager`, encode/decode envelopes yourself | +| **HTTP or WebSocket convenience** | Auto-wrap application bytes | `@shade/transport` on top of your stack | Use when you want transport helpers, not a new protocol | +| **Android** | Byte-compatible with TS (roadmap) | `shade-android` module | See [android/shade-android/README.md](../android/shade-android/README.md) — parity work in progress | + +You can **mix rows**: e.g. backend with `@shade/sdk` + SQLite for sessions, separate service mounting `transfer` routes, browser clients using `@shade/widgets`. + +--- + +## New project checklist (lightweight) + +1. Run a **prekey server** (Docker or embedded `@shade/server`) for your environment. +2. Pick **storage** (`sqlite:…`, Postgres, or project-specific adapter implementing the core storage interfaces). +3. Choose **surface**: usually `@shade/sdk` unless you truly need `@shade/core` only. +4. For files: enable **transfer routes** and authenticate chunk uploads using the patterns in the SDK (see streams doc). +5. Run **`shade doctor`** when something fails in production-ish setups (install the CLI as in repository [Quick start](../README.md#quick-start)); coverage is evolving — roadmap in [V2.2](./V2.2.md). + +--- + +## Related docs & roadmap + +| Topic | Doc | +|--------|-----| +| File transfer architecture | [streams.md](./streams.md) | +| Deployment & operations | [DEPLOYMENT.md](./DEPLOYMENT.md) | +| Threat model | [THREAT-MODEL.md](../THREAT-MODEL.md) | +| Planned improvements | [V2.1](./V2.1.md), feature backlog [V2.2](./V2.2.md), trust/ops [V2.3](./V2.3.md) | + +--- + +## Version note + +This file describes how Shade is **intended** to be composed. Package names and re-exports may gain small aliases over time; the **scenario table** should remain the source of truth for “what to install for what job.” Update this page when adding major surfaces (new transport bridges, richer `shade init` templates, etc.). diff --git a/docs/V2.1.md b/docs/V2.1.md new file mode 100644 index 0000000..0193e72 --- /dev/null +++ b/docs/V2.1.md @@ -0,0 +1,151 @@ +# Shade V2.1 — Improvements (infrastructure, storage, operations, security) + +This document describes **improvements** agreed for next-generation work on Shade: clearer product story, stronger storage, mobile parity, operational hardening, transfer abuse, and a formal security narrative. + +**Audience:** **Maintainers and contributors** implementing the changes. Add status fields as items land in code/docs. + +--- + +## 1. Clear “who is the server?” and data flow + +**Problem:** New users may think the prekey server is a message hub or that all E2EE traffic goes through the Shade container. + +**Goal:** One consistent explanation across the root README, package READMEs, and optional onboarding: **the prekey server distributes public keys and bundles**; **actual messages and (typically) file chunks go through your app’s own channel** (your transport, your backend, your URLs). + +**Deliverables (proposal):** + +- Diagram + short “keys vs payloads” text in the root README and in `@shade/server` README. +- Link to `THREAT-MODEL.md` from the same section (MITM on first contact ↔ safety numbers). +- Optionally one “concept page” (or extend `MIGRATION.md`) with typical architecture: *A ↔ B via app; both talk to the prekey host for X3DH material*. + +**Acceptance criteria:** A new developer without domain background understands in one reading *what* goes to the Shade server and *what* does not. + +--- + +## 2. Optional encryption of storage (at-rest) + +**Problem:** `THREAT-MODEL.md` states that a stolen DB + filesystem can expose private keys because Shade does not encrypt the storage layer by default. + +**Goal:** **Opt-in** protection for sensitive state (identity, session, optional stream resume secrets) with keys that **do not** live in plaintext in the DB — e.g. OS keychain/Keystore, passphrase + KDF, or an explicit device key injected by the app. + +**Design principles:** + +- Default developer experience (dev, simple demos) stays unchanged or includes a clear “insecure mode” warning in docs. +- APIs implementable per platform (Bun/SQLite, Postgres, web/IndexedDB, Android). +- Document limitations: what remains uncovered (e.g. active memory compromise). + +**Acceptance criteria:** Threat model updated for “when encrypted storage is enabled”; at least one reference implementation + migration note. + +--- + +## 3. Android parity and a published roadmap + +**Problem:** `shade-android` is under development; drift from the TS SDK undermines the “byte-compatible” promise. + +**Goal:** A **published roadmap** (milestones + what counts as parity vs TS-only) and **CI running shared test vectors** as a merge gate before release. + +**Deliverables:** + +- Roadmap section in `android/shade-android/README.md` or dedicated `ROADMAP-ANDROID.md` with explicit cross-checkpoints: wire format, fingerprints, rotations, streams (`0x11`) where applicable, resume semantics. +- CI job that fails on Kotlin vs TS vector mismatch. + +**Acceptance criteria:** Parity coverage is visible and enforceable; the first critical cross-surface (e.g. core ratchet + proto) is green before a “production” label. + +--- + +## 4. Operational hardening — prekey container and production + +**Problem:** Many teams deploy the Docker image quickly; mistakes around TLS, backups, and secrets add avoidable risk. + +**Goal:** A **production checklist**: TLS termination, volume backup (`/data`), rotation of `SHADE_OBSERVER_TOKEN`, use of `SHADE_PREKEY_PG_URL` vs SQLite, observability hooks, logging levels, meaning of stale cleanup parameters. + +**Deliverables:** + +- Extend `docs/DEPLOYMENT.md` or add short `docs/PRODUCTION-CHECKLIST.md` with bullet defaults. +- Link from the main README under “Deployment”. + +**Acceptance criteria:** A checklist operators can follow without reading the whole codebase first. + +--- + +## 5. Abuse and resource limits on the transfer plane + +**Problem:** Parallel lanes and large uploads can be abused for resource or storage if consumer mounts of `createTransferRoutes()` share no coherent policy. + +**Goal:** Documented **limits and patterns**: authentication (already an active SDK topic), max stream size, TTL for temporary chunk storage, quotas per identity or IP where sensible. + +**Deliverables:** + +- Guidelines in `docs/streams.md` or a dedicated “Transfer hardening” section. +- Optional helpers or middleware examples in `@shade/transfer` / server routes for common limits (without forcing every deployment into one DB model). + +**Acceptance criteria:** A clear “recommended minimum” for production that teams can copy. + +--- + +## 6. Security review and formal test / narrative + +**Problem:** Enterprises and security-conscious users often ask for independent review and a traceable test matrix. + +**Goal:** Plan for **independent crypto review** (timing, scope, deliverables) and a **published test / threat matrix** linking `THREAT-MODEL.md` to concrete automated tests (replay, tamper, out-of-order, resume, etc.). + +**Deliverables:** + +- Internal checklist “preparing for external review” (which files, assumptions, known limits). +- Short section in `SECURITY.md` on review status and how to report findings. + +**Acceptance criteria:** One authoritative source for “what is tested automatically” vs “what needs manual/MITM/out-of-band process”. + +--- + +## Dependencies and order (proposal) + +| Priority | Topic | Note | +|---------|-------|------| +| Low friction | §1, §4 | Documentation and checklists only | +| Medium | §5 | Design + docs + possibly small API helpers | +| High effort | §2, §3 | Platform work and long-term maintenance | +| Coordinated | §6 | Parallel with maturity in §2–§5 | + +--- + +## Document versioning + +- **V2.1** — first consolidated plan for the six improvement areas above. Update with dates and short per-section status lines as work starts and completes. + +Gitt der du er nå (post-streams 0.2.0), ville jeg prioritert: + + 1. Sosial nøkkel-recovery — løser "hva skjer hvis jeg mister telefonen?" som er det store hullet i alle + E2EE-systemer + 2. P2P WebRTC transport — naturlig kompletterende til streams; gir massiv throughput-gevinst gratis + 3. E2EE Pub/Sub — lavt scope, høy nytte, og passer perfekt med din "magic drop-in"-filosofi + + Disse tre har alle samme egenskap: de er infrastruktur-features som hver app du bygger fremover vil dra nytte av, + ikke bare Dispatch. Helt i tråd med "Shade som komponent som dras inn i hvilket som helst prosjekt". + + + /btw Hva med Shade videostreaming, VOIP streaming, og lignende? + + Magic drop-in-tilnærming (din filosofi) + + Tenk deg at i en Dispatch-app kan du bare gjøre: + + + + + + + …og det bare fungerer, E2EE, P2P når mulig, fallback til relay automatisk. Det er det samme mønsteret som streams + 0.2.0, bare for sanntid. + + Realisme-sjekk + + Video/VOIP er det vanskeligste i hele E2EE-verdenen. Signal brukte år på å få det riktig. Du bør: + 1. Ferdigstille streams 0.2.0 først (verifiserer crypto-fundamentet) + 2. Bygge P2P WebRTC-transport som separat milestone + 3. Da har du alle byggeklossene og Voice 0.4.0 blir 70% gjenbruk + + Men ja — dette hører absolutt hjemme i Shade. Shade som "alt-i-ett E2EE-platform" er en mye sterkere posisjon enn + "bare messaging + filer". Du kan bli til E2EE hva Twilio er til vanlig kommunikasjon. + + diff --git a/docs/V2.2.md b/docs/V2.2.md new file mode 100644 index 0000000..b5c480e --- /dev/null +++ b/docs/V2.2.md @@ -0,0 +1,126 @@ +# Shade V2.2 — Feature plan: product, platform, and developer experience + +This document gathers **planned features** that extend Shade beyond today’s core (X3DH + Double Ratchet + Streams/transfer): groups, asynchronous delivery, richer file UX, web workers, CLI, API docs, and scaffolding. + +Add optional per-feature status (Idea / Design / IMP / Done). + +--- + +## 1. Groups / multiple participants + +**Vision:** Beyond strict 1:1 — multiple identities in the same “conversation” or shared context (messages and possibly shared file/stream policy). + +**Challenges:** + +- Today’s Signal-like core is naturally 1:1; groups need either **pairwise sessions per member**, **sender keys / fan-out**, or a **dedicated group protocol** (larger architectural step). +- Lifecycle: invites, member removal, compromised device, history, and group PCS. + +**Goals for early milestones (proposal):** + +1. **Document** a recommended pattern for “group-lite” (e.g. coordinator relays ciphertext without decrypting + clients encrypt per-recipient). +2. **Optional** minimal API making fan-out easier in the SDK (without promising full MLS). + +**Acceptance criteria (MVP definition):** Explicit scope in docs + one reference architecture; no ambiguous “group is done” without an updated threat model. + +--- + +## 2. Async store-and-forward messaging + +**Vision:** Recipient need not be online; **ciphertext** is stored temporarily and fetched when the recipient returns — the server never sees plaintext. + +**Distinction from the prekey server:** + +- Prekey stays **public keys only** (or extended only under strict policy). +- A **dedicated relay/inbox service** (or app-owned backend) holds **encrypted blobs only** with TTL, idempotency, and authorization (who may list/fetch). + +**Deliverables (proposal):** + +- Protocol sketch: address registration, `PUT` blob, `GET`/`DELETE` or lease, replay protection at the application layer. +- SDK helpers: outgoing queue, poll/pull, or push-notification hook (without dictating mobile platform). + +**Acceptance criteria:** Threat-model section “what the relay sees” + reference implementation or example app. + +--- + +## 3. File metadata and preview + +**Vision:** Richer UX **without** leaking sensitive content to the server: filename, MIME type, length where known; **optional** client-generated thumbnails or previews encrypted as separate blocks or small payloads on the control init path. + +**Technical considerations:** + +- Anything sent must be **E2EE** or omitted; plaintext metadata on the server must be deliberate and minimal. +- Thumbnails should use **format hardening** on the client (size limits, sandboxing in UI). + +**Acceptance criteria:** Extended `stream-init` (or sidecar envelope) with optional fields + widget support for “preview when available”. + +--- + +## 4. Web: worker-based crypto and streaming + +**Vision:** Large files in the browser without blocking the main thread or blowing RAM — **Web Crypto / noble** inside a **Worker**, **ReadableStream/WritableStream** end-to-end chunk pipeline aligned with `@shade/streams` / transfer. + +**Deliverables:** + +- `@shade/crypto-web` (or companion) patterns: transferable buffers, lifecycle, errors surfaced to the UI. +- Documented constraints (Safari, chunk sizing, Service Worker vs dedicated worker). + +**Acceptance criteria:** E2E demo or test that sends multi‑MiB through a worker without a blocking UI. + +--- + +## 5. CLI: `shade doctor` + +**Vision:** One command that **diagnoses the environment** before production debugging. + +**Typical checks (proposal):** + +- Reachability of `prekeyServer` (`/health`, optional OpenAPI). +- Local config: storage path, rotation headers, clock skew (relevant for signed requests). +- **Streams:** transfer routes mounted, auth matches expected key, `GET .../state` behaves as expected in test mode. +- CLI / Node/Bun runtime versions and `@shade/*` packages where readable from `package.json`. + +**Acceptance criteria:** `shade doctor` with exit codes suitable for CI (warn vs fail). + +--- + +## 6. OpenAPI / docs + +**Vision:** All HTTP contracts teams are expected to implement (prekey **and** transfer) appear in **one** OpenAPI story or clearly linked specs — not only README examples. + +**Deliverables:** + +- Consolidate or cross-reference `openapi.yaml` with transfer endpoints (`/v1/transfer/*`) and security schemes for chunk upload. +- `/docs` (Redoc or similar) or published static artifacts for versioned specs. + +**Acceptance criteria:** Generated client (e.g. Python/Go) from spec without manual fixes for the happy path. + +--- + +## 7. `shade init` + +**Vision:** Scaffolding from **empty repo** to a **minimal runnable** app with Shade and optional streams. + +**Extensions:** + +- New or extended template: **minimal Hono/Fastify** app with `Shade.transferRoute()` mounted, auth example matching the SDK authenticator, `.env` template. +- Optional: demo `shade doctor` after init. + +**Acceptance criteria:** `shade init …` produces a project that `bun install && bun run start` runs with documented env vars. + +--- + +## Dependencies between items + +```text +shade init ─────► doctor (same conventions for URLs and secrets) +openapi/docs ◄── transfer + prekey (single source) +web workers ───► streams UX (large file in browser) +groups ◄──────── store-and-forward (often related socially/technically) +metadata/preview► widgets + proto/control plane +``` + +--- + +## Document versioning + +- **V2.2** — feature backlog as described. Split into issues/ADR per feature when implementation starts. diff --git a/docs/V2.3.md b/docs/V2.3.md new file mode 100644 index 0000000..47681fc --- /dev/null +++ b/docs/V2.3.md @@ -0,0 +1,102 @@ +# Shade V2.3 — Tillit, retention, integrasjon og observability + +Dette dokumentet beskriver **høyere ambisjonsnivå** og **plattformkryssende** arbeid: brukertilfeller der tillit må være **eksplisitt**, data må **utkrympes**/ryddes automatisk, apper må **enkelt koble seg på** kjente transportmønstre, og drift må **observere** uten å lekke innhold. + +--- + +## 1. Key transparency / bundle-attestasjon **eller** kritiske fingerprint-øyeblikk i UI + +Dette kan sees som **to spor** samme mål: redusere tillit til én korrumperbar prekey-server uten brukerens merknad. + +### Spor A — Key transparency / bundle-attestasjon (ambisiøst) + +**Idé:** Logging av **hvilket bundle som ble utlevert når**, kryptografisk forankret og **verifiserbar av klienter** eller tredjeparts-audit (inspirert av KT-litteratur, tilpasset Shade sin trusselmodell). + +**Leveranser (målpris høyt):** + +- Trusselmodell-oppdatering: *hva* CT/attest løser vs *fortsatt MITM før første verifisering*. +- Designnotat: datastruktur, friskhetsbevis, klient-verifikasjonssteg, operatørkost. + +**Risiko:** Kompleks drift og kryptodesign — bør være **valgfritt** lag for motiverte deploys. + +### Spor B — Fingerprints i app-UI på kritiske hendelser (pragmatisk) + +**Idé:** Gjør **safety numbers** synlige og **handlingspålagte** i presiserte flyt: + +- **Før første stor fil** (eller før første stream over terskel i bytes). +- **Før «trust this device»** / backup-importer / ny enhet som gjenbruker identitet. + +**Leveranser:** + +- `@shade/widgets` + SDK-hooks: modal/sheet med fingerprint, kopier-OOB-tekst, «jeg har verifisert». +- Dokumenterte UX-retningslinjer (unngå «alert fatigue» kun på lave risiko-events). + +### Felles akseptansekriterier + +Enten spor A eller B må ha **eksplisitt testcoverage** på «blokkerer/handhever verifisering» der det er lovet, og trusselmodellen må nevne kombinasjonen av OOB + tekniske grep. + +--- + +## 2. Retention policies + +**Vision:** Standardiserte **TTL- og oppryddingsregler** for data Shade-økosystemet etterlater på server eller i klient-lagring — spesielt: + +- **Stream-state** og midlertidige chunk-referanser etter fullførte/avbrutte transfers. +- **Eventuell** inbox/relay-ciphertext (`V2.2`). +- Prekey-server: kobling til eksisterende `SHADE_STALE_DAYS` / cleanup, plus **policy-dokumentasjon** for operatører. + +**Leveranser:** + +- Default-anbefalinger (f.eks. «ferdige streams: prune etter N dager») i `@shade/storage-*` helpers eller server-factory. +- Konfigurerbare hooks: `maxAge`, `maxBytesPerIdentity`, cron vs on-access prune. + +**Akseptansekriterier:** Ingen «uendelig vekst» som default i nye templates; dokumentert adferd i `streams.md` / deployment. + +--- + +## 3. Ferdig «bridge» (transport) + +**Vision:** Apper som ikke kan eller vil bruke WebSocket, får **ferdig mønster** for mottak av små meldinger eller kontroll-signaler. + +**Eksempler:** + +- **Server-Sent Events (SSE)** som **ren ciphertext-pipe** eller som signal «hent fra inbox» uten plaintext på server (kombinasjon med `V2.2`). +- **Lang-poll fallback** dokumentert ved siden av WS i `@shade/transport`. + +**Leveranser:** + +- Modulær `bridge`-pakke eller tydelig undermodul med få eksponerte typer og én felles `IncomingMessage`-modell på klient. + +**Akseptansekriterier:** Ett fungende eksempel + test som viser dekryptering likt eksisterende transport. + +--- + +## 4. Observability + +**Vision:** Produksjonsteam får **målepunkter og spor** uten **innholdslekkasjer** eller kryptomatrise i logger. + +**Forslag til innhold:** + +- **Metrics (Prometheus-stil eller vendor-nøytralt):** opplastings-/nedlastings-varighet, lane-telling, retry counts, abort vs complete rates, HTTP/WS-feilkoder (aggregert). +- **OpenTelemetry:** spans rundt `TransferEngine` og prekey-endepunkter (uten payload-lengde i klartekst som PII — bruk binære størrelser eller binning). +- **Sampling og PII-policy** dokumentert (ikke logg adresser i full hvis compliance krever maskering). + +**Akseptansekriterier:** Opt-in flags (default av i lib, på i server-container der det gir mening), og `docs/` avsnitt om hva som **aldri** skal logges. + +--- + +## Prioritering mot V2.1 / V2.2 + +| Dette dokument (V2.3) | Naturlig forutsetning | +|----------------------|------------------------| +| Retention | Streams/transfer i bruk (`V2.1` §5, `V2.2` metadata) | +| Bridge | `V2.2` store-and-forward eller egen meldingskanal | +| UI fingerprints | Widgets/SDK allerede i bruk | +| KT / attest | Moden trusselmodell + juridisk/operativ vilje | +| Observability | Stabil nok intern API for hooks | + +--- + +## Versjonering + +- **V2.3** — første samlet plan for tillit, retention, bridge og observability. Splitt i ADR-er når konkret design er valgt. diff --git a/docs/files.md b/docs/files.md new file mode 100644 index 0000000..a400b81 --- /dev/null +++ b/docs/files.md @@ -0,0 +1,201 @@ +# `@shade/files` — E2EE filesystem RPC + +`@shade/files` exposes a typed, end-to-end-encrypted RPC surface for +filesystem-style operations between two Shade peers. Both sides ride a +single Double Ratchet session for control envelopes; large-file content +(`> 256 KiB`) flows through `@shade/transfer` lanes, correlated back to +the RPC by an opaque `userMetadata.shadeFilesWriteId` / +`shadeFilesReadStreamId`. + +The package is a **primitive**, not a UI: it ships hooks and clients, not +file managers. Consumers (e.g. Dispatch, Mail, Drive-style apps) keep +their own UI and plug `@shade/files` underneath. + +## Quick start + +### Server (Bob exposes a filesystem) + +```ts +import { createShade } from '@shade/sdk'; + +const bob = await createShade({ prekeyServer, address: 'bob' }); +bob.configureTransfers({ resolveBaseUrl: ... }); +Bun.serve({ port: 8080, fetch: (await bob.transferRoute()).fetch }); + +const stop = await bob.files.serve({ + list: async (ctx) => ({ entries: await readdirAt(ctx.path), hasMore: false }), + stat: async (ctx) => statAt(ctx.path), + mkdir: async (ctx) => mkdirAt(ctx.path, ctx.args.recursive), + delete: async (ctx) => deleteAt(ctx.path, ctx.args.recursive), + move: async (ctx) => moveAt(ctx.args.src, ctx.args.dst, ctx.args.overwrite), + read: async (ctx) => readAt(ctx.path), // returns inline | streams + write: async (ctx) => writeAt(ctx.args), // receives inline | streams + getThumbnail: async (ctx) => thumbnailAt(ctx.path, ctx.args.size, ctx.args.format), +}); + +// Later: stop the handler. +await stop(); +``` + +### Client (Alice consumes Bob's filesystem) + +```ts +const alice = await createShade({ prekeyServer, address: 'alice' }); +alice.configureTransfers({ resolveBaseUrl: ... }); +Bun.serve({ port: 8081, fetch: (await alice.transferRoute()).fetch }); + +const fs = await alice.files.client('bob'); + +const page = await fs.list('/photos'); +await fs.write('/photos/cover.png', new Uint8Array(...)); // auto inline/streams +const result = await fs.read('/photos/cover.png'); +if (result.kind === 'inline') console.log(result.bytes.byteLength); +else for await (const _chunk of /* result.stream */) { /* ... */ } +``` + +## Op surface + +| Op | Args | Result | +|----------------|------------------------------------------|-----------------------------------------| +| `list` | `path`, `cursor?`, `pageSize?`, `filter?`| `{ entries, hasMore, nextCursor? }` | +| `stat` | `path` | `FileEntry` | +| `mkdir` | `path`, `recursive?` | `{ entry: FileEntry }` | +| `delete` | `path`, `recursive?` | `{ deletedCount }` | +| `move` | `src`, `dst`, `overwrite?` | `{ entry: FileEntry }` | +| `read` | `path`, `range?`, `preferInline?` | inline `{ bytes, sha256, size }` or streams `{ stream, size, sha256 }` | +| `write` | `path`, `Uint8Array \| Blob \| Stream` | `{ entry: FileEntry }` | +| `getThumbnail` | `path`, `size: 64\|128\|256\|512`, `format?` | `{ bytes, format, width, height, sha256 }` | +| `custom` | typed via `CustomOpsMap[K]` | typed via `CustomOpsMap[K]` | + +`MUTATION_OPS = { mkdir, delete, move, write, custom }` — these auto-generate +an idempotency key per logical call so transparent retries under the SDK +don't produce duplicates. + +## Inline vs streams + +The threshold is `INLINE_THRESHOLD = 256 * 1024` bytes (plaintext). The +client's `decideInline()` runs at `write` time: + +* `Uint8Array` / `Blob` with known size → direct comparison. +* Bare `ReadableStream` → peek the first chunks; promote to streams as + soon as the buffered prefix exceeds the threshold. + +Streams paths kick `shade.upload(...)` with `userMetadata.shadeFilesWriteId` +in parallel with the RPC envelope. The server's streams-bridge accepts the +inbound transfer immediately into a `TransformStream` and parks the +readable side until the matching `write` RPC arrives. + +## Custom ops + +Augment `CustomOpsMap` once globally for type-safe consumer-defined ops: + +```ts +declare module '@shade/files' { + interface CustomOpsMap { + 'app.deploy': CustomOpDef<{ jarPath: string }, { deploymentId: string }>; + } +} + +// Server registers a Zod-backed handler: +await shade.files.serve({ + custom: { + 'app.deploy': { + args: z.object({ jarPath: z.string() }), + response: z.object({ deploymentId: z.string() }), + handler: async (args, ctx) => ({ deploymentId: deploy(args.jarPath) }), + }, + }, +}); + +// Client gets typed I/O: +const result = await fs.custom('app.deploy', { jarPath: '/mods/foo.jar' }); +// ^? { deploymentId: string } +``` + +## Production hooks + +All hooks are callbacks the consumer plugs in — `@shade/files` enforces +the *mechanism*; the app owns the *policy*. + +```ts +await shade.files.serve({ + pathPolicy: { rootScope: '/srv/shade-root', forbidTraversal: true }, + rateLimits: { + maxOpsPerMinutePerSender: 600, + maxBytesPerHourPerSender: 10 * 1024 ** 3, + opCost: { read: 1, write: 5, delete: 3, default: 1 }, + }, + idempotency: { ttlMs: 60 * 60 * 1000, maxEntriesPerSender: 10_000 }, + requireFingerprintVerifiedFor: (ctx) => + ['delete', 'write', 'mkdir'].includes(String(ctx.op)) ? 'required' : 'optional', + isFingerprintVerified: (sender) => verifiedSet.has(sender), + verifySender: async (sender, canonical, sig) => { + return ed25519.verify(base64ToBytes(sig), canonical, getPubKey(sender)); + }, + onMetric: (name, value, tags) => prometheus.observe(name, value, tags), + onError: (err, ctx) => logger.error({ op: ctx.op, sender: ctx.sender }, err), +}); +``` + +## React + +```tsx +import { ShadeFilesProvider, useFileList } from '@shade/files/react'; + +function FileBrowser({ shade, peer }: { shade: Shade; peer: string }) { + return ( + + + + ); +} + +function Listing({ peer, path }) { + const { entries, isLoading, hasMore, loadMore, refresh } = useFileList(peer, path); + if (isLoading) return ; + return ( +
    + {entries.map((e) =>
  • {e.name} ({e.kind})
  • )} + {hasMore && } +
+ ); +} +``` + +`useFileTransfer` exposes a generic state machine for `BulkTransferHandle`s +returned by `uploadDirectory()` / `downloadDirectory()`: + +```tsx +const { start, abort, state, filesDone, filesTotal, bytesDone } = useFileUpload(); +const handle = uploadDirectory(fs, localDir, '/uploaded'); +useEffect(() => { start(handle); return () => void abort(); }, []); +``` + +## Path safety + +The dispatcher applies `validatePath()` before invoking the user handler: + +1. Length check on raw input. +2. Forbidden-bytes check (NUL/CR/LF/DEL/backslash). +3. Percent-decode (defends against `%2e%2e` smuggling). +4. POSIX normalization. +5. `..` traversal rejection. +6. Root-scope boundary check. +7. Consumer-supplied `extra` predicate. + +The user handler receives the *normalized* path; using `args.path` raw is a +TOCTOU bug. + +## Wire compatibility + +`@shade/files` 0.3.0 requires `@shade/proto` 0.3.0+. The proto layer's wire +VERSION was bumped from `0x01` to `0x02` to lift the 64 KiB length-prefix +ceiling that previously capped ratchet payloads. **Sessions are +incompatible across the bump**; both peers must run 0.3.0+. + +## Related modules + +* `@shade/streams` — chunk encryption, lane key derivation. Indirect dep. +* `@shade/transfer` — multi-lane transport with HTTP / WS fallback. +* `@shade/sdk` — `Shade.files` getter; `BackgroundHooks.onPruneFiles` for + retention. diff --git a/docs/shade-overview.html b/docs/shade-overview.html index 788a2a8..4494e9f 100644 --- a/docs/shade-overview.html +++ b/docs/shade-overview.html @@ -1,9 +1,9 @@ - + - Shade — ende-til-ende kryptering som modul + Shade — end-to-end encryption as a module @@ -364,32 +364,32 @@

Shade

- En gjenbrukbar modul for ende-til-ende-kryptert kommunikasjon i egne apper — med samme type protokoll som brukes i Signal. + A reusable module for end-to-end encrypted communication in your own apps — using the same kind of protocol as Signal.

X3DH Double Ratchet TypeScript - Plattformagnostisk crypto + Platform-agnostic crypto
-
-

Hva gjør prosjektet?

+
+

What does the project do?

- Shade er et monorepo som implementerer sikker meldingskryptering mellom to parter (for eksempel nettleser og server, eller to klienter). Meldingene er kryptert slik at transportlaget (HTTP, WebSocket, e.l.) bare ser uleselige bytes — ikke innholdet. + Shade is a monorepo that implements secure messaging encryption between two parties (for example browser and server, or two clients). Messages are encrypted so the transport layer (HTTP, WebSocket, etc.) only sees opaque bytes — not the content.

- Kjerneideen: Du bygger inn ShadeSessionManager (fra @shade/core) sammen med en CryptoProvider (f.eks. Web Crypto i nettleser/Bun) og lagring. Deretter kan du kalle encrypt / decrypt per motpart, akkurat som i demo-koden demo.ts. + Core idea: Embed ShadeSessionManager (from @shade/core) together with a CryptoProvider (e.g. Web Crypto in the browser/Bun) and storage. Then call encrypt / decrypt per peer, just like in the demo code demo.ts.

- Første melding til noen ny inneholder nøkkelavtale (X3DH). Etterpå bruker hver melding Double Ratchet: nye meldingsnøkler og periodiske DH-steg gir forward secrecy (gamle meldinger overlever ikke nøkkellekkasje) og post-compromise security (systemet «helbreder» seg over tid etter kompromittering). + The first message to someone new performs key agreement (X3DH). After that each message uses the Double Ratchet: fresh message keys and periodic DH steps provide forward secrecy (past messages do not survive key compromise) and post-compromise security (the system “recovers” over time after a compromise).

-
-

Pakkene (hvordan det henger sammen)

-
+
+

Packages (how they fit)

+
@@ -397,117 +397,117 @@
-

Protokollen. X3DH, Double Ratchet, sesjonstyper, feiltyper. Ingen plattformkrypto her — bare grensesnittet CryptoProvider.

+

The protocol. X3DH, Double Ratchet, session shapes, errors. No platform crypto here — only the CryptoProvider interface.

    -
  • ShadeSessionManager — høynivå-API: initialize, createPreKeyBundle, initSessionFromBundle, encrypt, decrypt
  • -
  • Symmetrisk kryptering: AES-256-GCM med AAD fra ratchet-header
  • +
  • ShadeSessionManager — high-level API: initialize, createPreKeyBundle, initSessionFromBundle, encrypt, decrypt
  • +
  • Symmetric encryption: AES-256-GCM with AAD from the ratchet header
-
-

Nøkler i korthet

+
+

Keys at a glance

- Ed25519 brukes til å signere den «signerte prekeyen». X25519 brukes i Diffie-Hellman i X3DH og i ratchet. Én identitet per enhet/bruker i typisk oppsett. + Ed25519 signs the “signed prekey”. X25519 is used in Diffie–Hellman in X3DH and in the ratchet. One identity per device/user is typical.
-
-

Interaktiv flyt: fra null til kryptert melding

+
+

Interactive flow: zero to encrypted message

- Klikk «Neste» for å gå gjennom rekkefølgen slik Shade er bygget. Dette speiler ShadeSessionManager og demoen i repoet. + Click “Next step” to walk through the sequence as Shade builds it. This mirrors ShadeSessionManager and the demo in the repo.

-

Sesjon og meldinger

+

Sessions and messages

- - + +
-

X3DH og Double Ratchet (kort forklart)

+

X3DH and Double Ratchet (brief)

- X3DH løser problemet «jeg vil snakke med Bob nå, men Bob svarer ikke før senere». Bob legger ut en prekey bundle på serveren. Alice henter den, gjør 3 eller 4 DH-operasjoner (avhengig av om engangsnøkkel brukes), og deriverer en felles rot-nøkkel som begge kan rekonstruere uten at serveren kjenner hemmeligheten. + X3DH solves “I want to talk to Bob now, but Bob may not reply until later”. Bob publishes a prekey bundle on the server. Alice fetches it, runs 3 or 4 DH operations (depending on whether a one-time key is used), and derives a shared root key both parties can reconstruct — without the server learning the secret.

- Double Ratchet bruker den roten som startpunkt. For hver melding (eller ved nye DH-nøkler) avledes nye nøkler; meldinger på ledningen er AES-GCM med autentisering (AAD binder kryptoteksten til ratchet-header). Protokollen håndterer også meldinger i feil rekkefølge innenfor grenser (MAX_SKIP). + Double Ratchet uses that root as a starting point. For each message (or when new DH keys spin), keys are derived; on the wire payloads are AES-GCM with authentication (AAD binds ciphertext to the ratchet header). The protocol also tolerates messages arriving out of order within limits (MAX_SKIP).

- Spesifikasjoner fra Signal (engelsk): X3DH · Double Ratchet. + Signal specifications (English): X3DH · Double Ratchet.

-
-

Bruke Shade i flere prosjekter

+
+

Using Shade across projects

- Tenk på Shade som tre lag du kan kombinere etter behov: + Treat Shade as three layers you combine as needed:

    -
  1. Core + crypto-provider + storage — selve E2EE-motoren (kan kjøre i klient eller serverprosess som skal dekryptere).
  2. -
  3. Proto — når du vil ha kompakt binær serialisering.
  4. -
  5. Transport + prekey-server — når du vil standardisere nøkkelutveksling og kanaler.
  6. +
  7. Core + crypto provider + storage — the E2EE engine itself (runs in a client or a server process that must decrypt).
  8. +
  9. Proto — when you want compact binary serialization.
  10. +
  11. Transport + prekey server — when you want standardized key discovery and channels.

- Referansekjøring: bun demo.ts i rotmappen viser frontend/backend-flyt med minnelager og ekte kryptoprimitiver. + Reference path: bun demo.ts from the repo root shows a frontend/backend flow with memory storage and real crypto primitives.

-

Shade — oversikt generert som statisk HTML i docs/shade-overview.html. Åpne filen direkte i nettleseren eller server den statisk.

+

Shade — overview written as static HTML in docs/shade-overview.html. Open the file directly in the browser or serve it as static assets.

@@ -561,28 +561,28 @@ // Flow steps var steps = [ { - title: "Initialiser klient", - body: "Kall initialize(): last eller generer identitetsnøkkel (Ed25519 + X25519), registrationId og signert prekey.", + title: "Initialize client", + body: "Call initialize(): load or generate the identity keys (Ed25519 + X25519), registrationId, and signed prekey.", }, { - title: "Publiser prekey bundle", - body: "Bygg bundle med createPreKeyBundle() / generateOneTimePreKeys() og registrer på prekey-server (eller del ut av band for demo).", + title: "Publish prekey bundle", + body: "Build a bundle with createPreKeyBundle() / generateOneTimePreKeys() and register it on the prekey server (or share out-of-band for a demo).", }, { - title: "Start sesjon mot peer", - body: "Hent motpartens bundle, kjør initSessionFromBundle(address, bundle). X3DH kjører og ratchet-sesjon lagres i StorageProvider.", + title: "Start session with peer", + body: "Fetch the peer bundle, run initSessionFromBundle(address, bundle). X3DH runs and the ratchet session is stored in StorageProvider.", }, { - title: "Første encrypt", - body: "encrypt() returnerer ShadeEnvelope type 'prekey': inneholder ephemeral nøkkel, prekey-ID-er og første RatchetMessage (AES-GCM).", + title: "First encrypt", + body: "encrypt() returns a ShadeEnvelope of type 'prekey': includes ephemeral keys, prekey IDs, and the first RatchetMessage (AES-GCM).", }, { - title: "Motpart decrypt", - body: "decrypt() på PreKeyMessage: gjenskaper samme root key, initReceiverSession, ratchetDecrypt — plaintext ut.", + title: "Peer decrypt", + body: "decrypt() on PreKeyMessage: restores the same root key, initReceiverSession, ratchetDecrypt — plaintext out.", }, { - title: "Videre meldinger", - body: "Neste kall til encrypt() gir type 'ratchet'. DH-ratchet steg gir nye kjeder og forbedret sikkerhet over tid.", + title: "Further messages", + body: "The next encrypt() calls yield type 'ratchet'. DH ratchet steps rotate chains and improve security over time.", }, ]; diff --git a/docs/streams.md b/docs/streams.md new file mode 100644 index 0000000..a178ba9 --- /dev/null +++ b/docs/streams.md @@ -0,0 +1,117 @@ +# Shade Streams 0.2.0 + +E2EE chunked upload/download for Shade. Drop into any Shade-using app: + +```ts +const handle = await shade.upload({ to: 'bob', input: file }); +const result = await handle.done(); // { sha256, bytesSent, durationMs } +``` + +…and on the receiver: + +```ts +shade.onIncomingTransfer(async (incoming) => { + const handle = await incoming.accept({ output: { kind: 'file', path: '/uploads/x' } }); + await handle.done(); +}); +``` + +Or in React: + +```tsx + + console.log(r.sha256)} /> + +``` + +## How it works + +A transfer has two planes: + +- **Control plane** — `stream-init`, `stream-finish`, `stream-abort`, and + `stream-resume-*` messages, encoded as JSON plaintext and shipped through + the existing Double Ratchet (envelope type `0x02`). One ratchet step + establishes a stream; the rest is per-chunk AEAD. +- **Data plane** — `stream-chunk` envelopes (envelope type `0x11`), + AES-256-GCM-encrypted under a per-lane key, shipped over HTTP POST (or + WebSocket if opted-in). Lanes run in parallel for throughput. + +``` +Sender Receiver +────── ──────── +streamSecret = randomBytes(32) +streamId = randomBytes(16) +streamKey = HKDF(streamSecret, streamId, "shade-stream/v1\0master") +laneKey[i] = HKDF(streamKey, streamId, "...\0lane\0" || u32(i)) + +[stream-init JSON over Double Ratchet] ─▶ + parses streamSecret, derives same keys + spawns L per-lane receivers + +[chunk 0x11 over HTTP] ─▶ + AES-GCM(laneKey[i], plaintext, nonce=laneId||seq, aad=streamId||laneId||seq||isLast) + decrypts, verifies, writes to sink + (× 4 lanes in parallel) + +[stream-finish JSON over Double Ratchet] ─▶ + verifies per-lane sha256 + overall sha256 + throws TransferIntegrityError on mismatch +``` + +## Partition strategies + +- **Range** (default for known-size inputs) — lane `i` owns bytes + `[i·N/L, (i+1)·N/L)`. Receiver reconstructs by concatenating lane outputs + in laneId order. +- **Round-robin** (default for unknown-size streams) — chunk `i` goes to + lane `i mod L`. Receiver reorders via a per-stream chunk-index buffer. + +## Resume + +Persistence is opt-in via a `ResumeStore` (memory, SQLite, Postgres, +IndexedDB-ready). State persisted on init; sender's resume queries the +receiver's `lastSeqAcked` per lane via `GET /v1/transfer/:streamId/state`, +then continues from there. The streamSecret is encrypted at rest under a +device-key derived from the local identity's signing private key — a +stolen DB without the identity key cannot resume. + +```ts +const handle = await shade.resumeUpload(streamId, sameInputAsBefore); +await handle.done(); +``` + +Resume across **identity rotation** is not supported (rotation invalidates +the device key — by design, to prevent a stolen pre-rotation DB from +deriving keys for any post-rotation transfer). Restart the transfer +manually after rotation. + +## Throughput + +- Default 4 lanes × 1 MiB chunks × 4 in-flight chunks per lane = + 16 MiB peak in-flight per direction. +- Memory-bounded: receivers stream chunks to the configured sink without + buffering the full payload. 1 GB transfer = O(chunkSize) RSS, not O(file). +- AES-GCM is hardware-accelerated via `SubtleCrypto`; SHA-256 streaming via + `@noble/hashes`. + +## Security properties + +| ID | Property | +|---|---| +| S1 | streamSecret never on the wire in plaintext (Double Ratchet only) | +| S2 | Unique per-(streamKey, laneId, seq) AEAD nonce — no nonce reuse | +| S3 | Tampered chunk header / ciphertext / tag → AEAD reject | +| S4 | Per-lane sha256 + overall sha256 verified at finish | +| S5 | streamKey/laneKey zeroized on abort/finish (`destroy()`) | +| S6 | Concurrent streams have independent lane keys | +| S7 | seq overflow practical-impossible (u64 max) | +| S8 | At-rest streamSecret encrypted under device-key | + +## API surface + +See package READMEs: + +- `packages/shade-streams/README.md` — crypto + state machines +- `packages/shade-transfer/README.md` — orchestration, transports, persistence +- `packages/shade-sdk/README.md` — magic drop-in +- `packages/shade-widgets/README.md` — React UI diff --git a/examples/07-streams-upload/README.md b/examples/07-streams-upload/README.md new file mode 100644 index 0000000..546a6db --- /dev/null +++ b/examples/07-streams-upload/README.md @@ -0,0 +1,28 @@ +# Example 07 — E2EE file upload with Shade Streams + +Two Bun processes, both running real `Shade` instances. Alice uploads a file +to Bob; the file is chunked, encrypted per-chunk under a per-lane key, sent +over HTTP across 4 parallel lanes, and verified end-to-end. + +## Files + +- `prekey-server.ts` — boots a `@shade/server` prekey server on port 9991. +- `bob-receiver.ts` — Bob's Shade runtime + a Bun HTTP server that mounts + `Shade.transferRoute()`. Saves incoming files to `./received/`. +- `alice-sender.ts` — Alice's Shade runtime; uploads a file to Bob. + +## Run + +```bash +# Terminal 1 — prekey server +bun run examples/07-streams-upload/prekey-server.ts + +# Terminal 2 — Bob (receiver) +bun run examples/07-streams-upload/bob-receiver.ts + +# Terminal 3 — Alice (sender) +bun run examples/07-streams-upload/alice-sender.ts ./path/to/file +``` + +The receiver writes the decrypted file to `./received/` and prints +the matching sha256. diff --git a/examples/07-streams-upload/alice-sender.ts b/examples/07-streams-upload/alice-sender.ts new file mode 100644 index 0000000..5aa0e65 --- /dev/null +++ b/examples/07-streams-upload/alice-sender.ts @@ -0,0 +1,54 @@ +/** + * Alice — sender. Uploads a local file to Bob using 4 parallel lanes, + * 1 MiB chunks, and the default HTTP transport. Prints sender-side + * progress + the verified sha256 on completion. + * + * Usage: + * bun run examples/07-streams-upload/alice-sender.ts + */ +import { readFile } from 'node:fs/promises'; +import { basename } from 'node:path'; +import { createShade } from '@shade/sdk'; + +const PREKEY_URL = 'http://localhost:9991'; +const BOB_BASE_URL = 'http://localhost:9992'; + +const filePath = process.argv[2]; +if (filePath === undefined || filePath === '') { + console.error('usage: alice-sender.ts '); + process.exit(2); +} + +const bytes = await readFile(filePath); +const name = basename(filePath); + +const alice = await createShade({ prekeyServer: PREKEY_URL, address: 'alice' }); + +alice.configureTransfers({ + resolveBaseUrl: async (peer) => { + if (peer === 'bob') return BOB_BASE_URL; + throw new Error(`unknown peer ${peer}`); + }, +}); + +console.log(`[alice] uploading ${name} (${bytes.length} bytes) → bob`); + +const handle = await alice.upload({ + to: 'bob', + input: new Uint8Array(bytes), + lanes: 4, + chunkSize: 1024 * 1024, + metadata: { name }, + onProgress: (p) => { + const pct = p.percent !== undefined ? `${(p.percent * 100).toFixed(1)}%` : '—'; + const mbps = (p.bytesPerSecond / (1024 * 1024)).toFixed(2); + process.stdout.write(`\r[alice] ${pct} ${mbps} MB/s ${p.bytesSent}/${p.bytesTotal ?? '?'} B`); + }, +}); + +const result = await handle.done(); +process.stdout.write('\n'); +console.log(`[alice] done — ${result.bytesSent} B in ${result.durationMs.toFixed(0)} ms`); +console.log(`[alice] sha256 = ${result.sha256}`); + +await alice.shutdown(); diff --git a/examples/07-streams-upload/bob-receiver.ts b/examples/07-streams-upload/bob-receiver.ts new file mode 100644 index 0000000..ab561db --- /dev/null +++ b/examples/07-streams-upload/bob-receiver.ts @@ -0,0 +1,36 @@ +/** + * Bob — receiver. Boots a Shade runtime and mounts the transfer route on + * port 9992. Incoming uploads are decrypted, integrity-checked, and saved + * to ./received/. + */ +import { mkdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { createShade } from '@shade/sdk'; + +const PREKEY_URL = 'http://localhost:9991'; +const PORT = 9992; +const OUT_DIR = join(import.meta.dir, 'received'); +mkdirSync(OUT_DIR, { recursive: true }); + +const bob = await createShade({ prekeyServer: PREKEY_URL, address: 'bob' }); + +bob.configureTransfers({ + resolveBaseUrl: async () => { + throw new Error('bob is receive-only'); + }, +}); + +bob.onIncomingTransfer(async (incoming) => { + const name = incoming.metadata.name ?? incoming.streamId; + const path = join(OUT_DIR, name.replace(/[^a-zA-Z0-9._-]/g, '_')); + console.log(`[bob] incoming "${name}" from ${incoming.from} → ${path}`); + const handle = await incoming.accept({ output: { kind: 'file', path } }); + void handle.done().then((result) => { + console.log(`[bob] done "${name}" — ${result.bytesSent} B, sha256=${result.sha256}`); + }); +}); + +const app = await bob.transferRoute(); +Bun.serve({ port: PORT, fetch: app.fetch }); +console.log(`Bob's transfer endpoint listening on http://localhost:${PORT}`); +console.log(`Saving incoming files to ${OUT_DIR}`); diff --git a/examples/07-streams-upload/prekey-server.ts b/examples/07-streams-upload/prekey-server.ts new file mode 100644 index 0000000..fe4caa2 --- /dev/null +++ b/examples/07-streams-upload/prekey-server.ts @@ -0,0 +1,17 @@ +/** + * Minimal Shade prekey server for the streams example. + * Run on port 9991 — both Alice and Bob register here. + */ +import { createPrekeyServer, MemoryPrekeyStore } from '@shade/server'; +import { SubtleCryptoProvider } from '@shade/crypto-web'; + +const crypto = new SubtleCryptoProvider(); +const server = createPrekeyServer({ + crypto, + store: new MemoryPrekeyStore(), + disableRateLimit: true, +}); + +const port = 9991; +Bun.serve({ port, fetch: server.fetch }); +console.log(`Prekey server listening on http://localhost:${port}`); diff --git a/examples/08-files-browser/README.md b/examples/08-files-browser/README.md new file mode 100644 index 0000000..3a7d02c --- /dev/null +++ b/examples/08-files-browser/README.md @@ -0,0 +1,31 @@ +# 08 — Files Browser (E2EE filesystem RPC) + +Two-process demo of `@shade/files`: Alice (CLI) talks to Bob (file server) over +Double Ratchet–encrypted RPC. Bob's filesystem is rooted at `./files-root/`. + +## Run + +```sh +# Terminal 1 — prekey server +bun run examples/08-files-browser/prekey-server.ts + +# Terminal 2 — Bob's file server +bun run examples/08-files-browser/bob-server.ts + +# Terminal 3 — Alice issues commands +bun run examples/08-files-browser/alice-cli.ts mkdir /docs +bun run examples/08-files-browser/alice-cli.ts upload ./README.md /docs/readme.md +bun run examples/08-files-browser/alice-cli.ts list /docs +bun run examples/08-files-browser/alice-cli.ts stat /docs/readme.md +bun run examples/08-files-browser/alice-cli.ts download /docs/readme.md /tmp/out.md +bun run examples/08-files-browser/alice-cli.ts delete /docs/readme.md +``` + +## What it shows + +* Standard ops: `list`, `stat`, `mkdir`, `delete`, `move`. +* Content I/O: `read`/`write` with automatic inline (≤ 256 KiB) / + streams (> 256 KiB) routing — handled transparently by the SDK. +* sha256 paritet: streaming downloads return the verified hash. +* Two ratchets, end-to-end encrypted. No content ever leaves either + process unencrypted. diff --git a/examples/08-files-browser/alice-cli.ts b/examples/08-files-browser/alice-cli.ts new file mode 100644 index 0000000..b42725b --- /dev/null +++ b/examples/08-files-browser/alice-cli.ts @@ -0,0 +1,108 @@ +/** + * Alice — the file CLIENT. Demonstrates the typed `@shade/files` API: + * + * bun run examples/08-files-browser/alice-cli.ts list / + * bun run examples/08-files-browser/alice-cli.ts mkdir /docs + * bun run examples/08-files-browser/alice-cli.ts upload ./README.md /docs/readme.md + * bun run examples/08-files-browser/alice-cli.ts download /docs/readme.md ./out.md + * bun run examples/08-files-browser/alice-cli.ts stat /docs/readme.md + * bun run examples/08-files-browser/alice-cli.ts delete /docs/readme.md + */ +import { readFile, writeFile, stat as fsstat } from 'node:fs/promises'; +import { createShade } from '@shade/sdk'; + +const PREKEY = 'http://localhost:9992'; +const ALICE_BASE_URL = 'http://localhost:9993'; +const BOB_BASE_URL = 'http://localhost:9994'; + +const alice = await createShade({ prekeyServer: PREKEY, address: 'alice' }); +alice.configureTransfers({ + resolveBaseUrl: async (peer) => { + if (peer === 'bob') return BOB_BASE_URL; + throw new Error(`alice: unknown peer ${peer}`); + }, +}); +const app = await alice.transferRoute(); +Bun.serve({ port: 9993, fetch: app.fetch }); + +const fs = await alice.files.client('bob', { defaultTimeoutMs: 30_000 }); + +const [cmd, ...args] = process.argv.slice(2); +try { + switch (cmd) { + case 'list': { + const path = args[0] ?? '/'; + const result = await fs.list(path); + for (const e of result.entries) { + console.log(`${e.kind === 'dir' ? 'd' : '-'} ${String(e.size).padStart(10)} ${e.name}`); + } + break; + } + case 'stat': { + const path = args[0]!; + const e = await fs.stat(path); + console.log(JSON.stringify(e, null, 2)); + break; + } + case 'mkdir': { + const path = args[0]!; + const r = await fs.mkdir(path, { recursive: true }); + console.log(`created ${r.entry.name}`); + break; + } + case 'delete': { + const path = args[0]!; + const r = await fs.delete(path, { recursive: true }); + console.log(`deleted ${r.deletedCount} entrie(s)`); + break; + } + case 'upload': { + const localPath = args[0]!; + const remotePath = args[1]!; + const bytes = new Uint8Array(await readFile(localPath)); + const s = await fsstat(localPath); + console.log(`uploading ${s.size} bytes → ${remotePath}`); + const r = await fs.write(remotePath, bytes, { overwrite: true }); + console.log(`done — remote size: ${r.entry.size}`); + break; + } + case 'download': { + const remotePath = args[0]!; + const localPath = args[1]!; + const result = await fs.read(remotePath); + if (result.kind === 'inline') { + await writeFile(localPath, result.bytes); + console.log(`downloaded ${result.bytes.byteLength} bytes (inline) sha256=${result.sha256}`); + } else { + const reader = result.stream.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + while (true) { + const { value, done } = await reader.read(); + if (done) break; + if (value !== undefined) { + chunks.push(value); + total += value.byteLength; + } + } + reader.releaseLock(); + const out = new Uint8Array(total); + let offset = 0; + for (const c of chunks) { + out.set(c, offset); + offset += c.byteLength; + } + await writeFile(localPath, out); + await result.done(); + console.log(`downloaded ${total} bytes (streamed) sha256=${result.sha256}`); + } + break; + } + default: + console.error('Usage: alice-cli.ts [args...]'); + process.exit(1); + } +} finally { + await alice.shutdown(); + process.exit(0); +} diff --git a/examples/08-files-browser/bob-server.ts b/examples/08-files-browser/bob-server.ts new file mode 100644 index 0000000..4b1f362 --- /dev/null +++ b/examples/08-files-browser/bob-server.ts @@ -0,0 +1,170 @@ +/** + * Bob — the file SERVER. Mounts a virtual filesystem rooted in this + * directory's `./files-root/` dir (created on first run) and serves it + * over `@shade/files` E2EE RPC. + * + * Demonstrates: list/stat/mkdir/delete/move + read/write inline+streams. + */ +import { mkdir, readdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises'; +import { createReadStream } from 'node:fs'; +import { Readable } from 'node:stream'; +import { join, basename, dirname } from 'node:path'; +import { createShade } from '@shade/sdk'; +import { + ConflictError, + NotFoundError, + type FileEntry, + type UserReadResult, +} from '@shade/files'; + +const ROOT = join(import.meta.dir, 'files-root'); +await mkdir(ROOT, { recursive: true }); + +const PREKEY = 'http://localhost:9992'; +const BOB_BASE_URL = 'http://localhost:9994'; +const ALICE_BASE_URL = 'http://localhost:9993'; + +const bob = await createShade({ prekeyServer: PREKEY, address: 'bob' }); +bob.configureTransfers({ + resolveBaseUrl: async (peer) => { + if (peer === 'alice') return ALICE_BASE_URL; + throw new Error(`bob: unknown peer ${peer}`); + }, +}); +const app = await bob.transferRoute(); +Bun.serve({ port: 9994, fetch: app.fetch }); +console.log(`[bob] listening on ${BOB_BASE_URL}, files at ${ROOT}`); + +function safePath(remote: string): string { + // Strip leading slash + reject any '..' (the dispatcher already does + // this, but we defend in depth). + const segments = remote.split('/').filter((s) => s !== '' && s !== '..'); + return join(ROOT, ...segments); +} + +await bob.files.serve({ + list: async (ctx) => { + const local = safePath(ctx.path); + let names: string[]; + try { + names = await readdir(local); + } catch { + throw new NotFoundError(ctx.path); + } + const entries: FileEntry[] = []; + for (const name of names) { + const s = await stat(join(local, name)); + entries.push({ + name, + kind: s.isDirectory() ? 'dir' : 'file', + size: s.isDirectory() ? 0 : s.size, + mtime: s.mtimeMs, + metadata: {}, + }); + } + return { entries, hasMore: false }; + }, + stat: async (ctx) => { + let s: import('node:fs').Stats; + try { + s = await stat(safePath(ctx.path)); + } catch { + throw new NotFoundError(ctx.path); + } + return { + name: basename(ctx.path), + kind: s.isDirectory() ? 'dir' : 'file', + size: s.isDirectory() ? 0 : s.size, + mtime: s.mtimeMs, + metadata: {}, + }; + }, + mkdir: async (ctx) => { + const local = safePath(ctx.path); + await mkdir(local, { recursive: ctx.args.recursive }); + return { + entry: { + name: basename(ctx.path), + kind: 'dir', + size: 0, + mtime: Date.now(), + metadata: {}, + }, + }; + }, + delete: async (ctx) => { + await rm(safePath(ctx.path), { recursive: ctx.args.recursive, force: false }); + return { deletedCount: 1 }; + }, + move: async (ctx) => { + await rename(safePath(ctx.args.src), safePath(ctx.args.dst)); + return { + entry: { + name: basename(ctx.args.dst), + kind: 'file', + size: 0, + mtime: Date.now(), + metadata: {}, + }, + }; + }, + read: async (ctx): Promise => { + const local = safePath(ctx.path); + const s = await stat(local); + if (s.isDirectory()) throw new ConflictError(`${ctx.path} is a directory`); + if (s.size <= 256 * 1024) { + const bytes = new Uint8Array(await readFile(local)); + return { kind: 'inline', bytes }; + } + // Stream the file. + const sha256 = await computeSha256OfFile(local); + const stream = Readable.toWeb(createReadStream(local)) as ReadableStream; + return { kind: 'streams', stream, size: s.size, sha256 }; + }, + write: async (ctx) => { + const local = safePath(ctx.args.path); + await mkdir(dirname(local), { recursive: true }); + if (ctx.args.content.kind === 'inline') { + await writeFile(local, ctx.args.content.bytes); + } else { + // Drain stream → temp buffer → file. For huge files use createWriteStream. + const reader = ctx.args.content.stream.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + while (true) { + const { value, done } = await reader.read(); + if (done) break; + if (value !== undefined) { + chunks.push(value); + total += value.byteLength; + } + } + reader.releaseLock(); + const out = new Uint8Array(total); + let offset = 0; + for (const c of chunks) { + out.set(c, offset); + offset += c.byteLength; + } + await writeFile(local, out); + // Verify integrity (optional) + await ctx.args.content.sha256; + } + const s = await stat(local); + return { + entry: { + name: basename(ctx.args.path), + kind: 'file', + size: s.size, + mtime: s.mtimeMs, + metadata: {}, + }, + }; + }, +}); + +async function computeSha256OfFile(path: string): Promise { + const { sha256 } = await import('@noble/hashes/sha2.js'); + const data = new Uint8Array(await readFile(path)); + return Array.from(sha256(data), (b) => b.toString(16).padStart(2, '0')).join(''); +} diff --git a/examples/08-files-browser/prekey-server.ts b/examples/08-files-browser/prekey-server.ts new file mode 100644 index 0000000..b58ac43 --- /dev/null +++ b/examples/08-files-browser/prekey-server.ts @@ -0,0 +1,17 @@ +/** + * Minimal Shade prekey server for the files-browser example. + * Run on port 9992 — both Alice and Bob register here. + */ +import { createPrekeyServer, MemoryPrekeyStore } from '@shade/server'; +import { SubtleCryptoProvider } from '@shade/crypto-web'; + +const crypto = new SubtleCryptoProvider(); +const server = createPrekeyServer({ + crypto, + store: new MemoryPrekeyStore(), + disableRateLimit: true, +}); + +const port = 9992; +Bun.serve({ port, fetch: server.fetch }); +console.log(`[prekey] listening on http://localhost:${port}`); diff --git a/package.json b/package.json index def8c76..fc834c6 100644 --- a/package.json +++ b/package.json @@ -18,11 +18,13 @@ "publish:docker": "bun run scripts/build-docker.ts -- --push" }, "devDependencies": { - "bun-types": "^1.3.11" + "bun-types": "^1.3.11", + "fast-check": "^3.22.0" }, "dependencies": { "@noble/curves": "^2.0.1", "@noble/hashes": "^2.0.1", - "hono": "^4.12.12" + "hono": "^4.12.12", + "zod": "^3.23.8" } } diff --git a/packages/shade-cli/package.json b/packages/shade-cli/package.json index 73747f5..2bb6aab 100644 --- a/packages/shade-cli/package.json +++ b/packages/shade-cli/package.json @@ -1,17 +1,17 @@ { "name": "@shade/cli", - "version": "0.1.0", + "version": "0.3.0", "type": "module", "main": "src/cli.ts", "bin": { "shade": "src/cli.ts" }, "dependencies": { - "@shade/sdk": "workspace:*", "@shade/core": "workspace:*", + "@shade/crypto-web": "workspace:*", + "@shade/sdk": "workspace:*", "@shade/storage-sqlite": "workspace:*", - "@shade/transport": "workspace:*", - "@shade/crypto-web": "workspace:*" + "@shade/transport": "workspace:*" }, "devDependencies": { "@shade/server": "workspace:*" diff --git a/packages/shade-cli/src/commands/peer.ts b/packages/shade-cli/src/commands/peer.ts index 8f0b123..4d7126e 100644 --- a/packages/shade-cli/src/commands/peer.ts +++ b/packages/shade-cli/src/commands/peer.ts @@ -27,7 +27,9 @@ export async function peerAddCommand(address: string): Promise { } export async function peerListCommand(): Promise { - const config = loadConfig(); + // Loading the config validates the environment; we don't need its values + // until session enumeration is implemented (TODO). + loadConfig(); // For list, we need to enumerate sessions from storage. The StorageProvider // doesn't currently expose a "list all sessions" method. For v1, we show diff --git a/packages/shade-cli/src/config.ts b/packages/shade-cli/src/config.ts index bd5a8eb..6b8ac7e 100644 --- a/packages/shade-cli/src/config.ts +++ b/packages/shade-cli/src/config.ts @@ -4,9 +4,9 @@ import { join } from 'path'; export interface CliConfig { prekeyServer: string; storage: string; - observerToken?: string; - observerUrl?: string; - address?: string; + observerToken?: string | undefined; + observerUrl?: string | undefined; + address?: string | undefined; } const DEFAULT_STORAGE = 'sqlite:./.shade/client.db'; diff --git a/packages/shade-core/package.json b/packages/shade-core/package.json index 21bdb2c..ba91eb4 100644 --- a/packages/shade-core/package.json +++ b/packages/shade-core/package.json @@ -1,6 +1,6 @@ { "name": "@shade/core", - "version": "0.1.0", + "version": "0.3.0", "type": "module", "main": "src/index.ts", "types": "src/index.ts", diff --git a/packages/shade-core/src/errors.ts b/packages/shade-core/src/errors.ts index 1756843..c887421 100644 --- a/packages/shade-core/src/errors.ts +++ b/packages/shade-core/src/errors.ts @@ -98,7 +98,7 @@ export class NetworkError extends ShadeError { } export class StorageError extends ShadeError { - constructor(message: string, public readonly cause?: unknown) { + constructor(message: string, public override readonly cause?: unknown) { super('SHADE_STORAGE', message); this.name = 'StorageError'; } diff --git a/packages/shade-core/src/ratchet.ts b/packages/shade-core/src/ratchet.ts index 43c46b4..cc0c8fa 100644 --- a/packages/shade-core/src/ratchet.ts +++ b/packages/shade-core/src/ratchet.ts @@ -2,7 +2,7 @@ import type { CryptoProvider } from './crypto.js'; import type { KeyPair, SessionState, ChainState, RatchetMessage } from './types.js'; import { MAX_SKIP, MAX_CACHED_SKIPPED_KEYS } from './types.js'; import { kdfRootKey, kdfChainKey } from './keys.js'; -import { DecryptionError, MaxSkipExceededError, DuplicateMessageError } from './errors.js'; +import { DecryptionError, MaxSkipExceededError } from './errors.js'; /** * Double Ratchet — per-message forward secrecy and post-compromise recovery. diff --git a/packages/shade-core/src/session.ts b/packages/shade-core/src/session.ts index 9c3c69b..4c4e46a 100644 --- a/packages/shade-core/src/session.ts +++ b/packages/shade-core/src/session.ts @@ -23,9 +23,8 @@ import { ratchetEncrypt, ratchetDecrypt, } from './ratchet.js'; -import { NoSessionError, UntrustedIdentityError } from './errors.js'; +import { NoSessionError } from './errors.js'; import { computeFingerprint, shortFingerprint } from './fingerprint.js'; -import { constantTimeEqual } from './crypto.js'; import { ShadeEventEmitter, shortHash } from './events.js'; const enc = new TextEncoder(); @@ -61,13 +60,42 @@ export class ShadeSessionManager { private registrationId: number = 0; private currentSignedPreKeyId: number = 0; private readonly events?: ShadeEventEmitter; + /** + * Per-address operation chain. Both encrypt and decrypt mutate ratchet + * state in place (counter, DH key, skipped-keys cache); concurrent + * operations on the same peer can corrupt the session. We serialize + * per-peer by chaining promises — operations to different peers stay + * fully concurrent. + */ + private readonly peerOpChains = new Map>(); constructor( private readonly crypto: CryptoProvider, private readonly storage: StorageProvider, options: { events?: ShadeEventEmitter } = {}, ) { - this.events = options.events; + if (options.events !== undefined) { + this.events = options.events; + } + } + + /** + * Run `fn` under the per-address mutex so encrypt/decrypt for the same + * peer never interleave their session-state mutations. + */ + private async runUnderPeerLock(address: string, fn: () => Promise): Promise { + const previous = this.peerOpChains.get(address) ?? Promise.resolve(); + const next = previous.catch(() => undefined).then(fn); + this.peerOpChains.set(address, next); + try { + return await next; + } finally { + // Best-effort cleanup so finished chains can be garbage collected + // when a peer goes idle. If a newer op has chained on, we leave it. + if (this.peerOpChains.get(address) === next) { + this.peerOpChains.delete(address); + } + } } /** Get the event emitter (if observability is enabled) */ @@ -368,56 +396,60 @@ export class ShadeSessionManager { * Subsequent messages are standard RatchetMessages. */ async encrypt(address: string, plaintext: string): Promise { - const session = await this.storage.getSession(address); - if (!session) throw new NoSessionError(address); + return this.runUnderPeerLock(address, async () => { + const session = await this.storage.getSession(address); + if (!session) throw new NoSessionError(address); - const ratchetMsg = await ratchetEncrypt(this.crypto, session, enc.encode(plaintext)); + const ratchetMsg = await ratchetEncrypt(this.crypto, session, enc.encode(plaintext)); - this.events?.emit('message.encrypted', { - address, - counter: ratchetMsg.counter, - ciphertextSize: ratchetMsg.ciphertext.length, - }); + this.events?.emit('message.encrypted', { + address, + counter: ratchetMsg.counter, + ciphertextSize: ratchetMsg.ciphertext.length, + }); + + // Check if this is the first message (X3DH metadata attached) + const x3dh = (session as any).__x3dh; + if (x3dh) { + delete (session as any).__x3dh; + await this.storage.saveSession(address, session); + + const preKeyMsg: PreKeyMessage = { + registrationId: x3dh.registrationId, + preKeyId: x3dh.preKeyId, + signedPreKeyId: x3dh.signedPreKeyId, + ephemeralKey: x3dh.ephemeralPublicKey, + identityDHKey: x3dh.identityDHKey, + message: ratchetMsg, + }; + return { + type: 'prekey', + content: preKeyMsg, + timestamp: Date.now(), + senderAddress: address, + }; + } - // Check if this is the first message (X3DH metadata attached) - const x3dh = (session as any).__x3dh; - if (x3dh) { - delete (session as any).__x3dh; await this.storage.saveSession(address, session); - - const preKeyMsg: PreKeyMessage = { - registrationId: x3dh.registrationId, - preKeyId: x3dh.preKeyId, - signedPreKeyId: x3dh.signedPreKeyId, - ephemeralKey: x3dh.ephemeralPublicKey, - identityDHKey: x3dh.identityDHKey, - message: ratchetMsg, - }; return { - type: 'prekey', - content: preKeyMsg, + type: 'ratchet', + content: ratchetMsg, timestamp: Date.now(), senderAddress: address, }; - } - - await this.storage.saveSession(address, session); - return { - type: 'ratchet', - content: ratchetMsg, - timestamp: Date.now(), - senderAddress: address, - }; + }); } /** * Decrypt a message from a peer. Handles both PreKeyMessage and RatchetMessage. */ async decrypt(address: string, envelope: ShadeEnvelope): Promise { - if (envelope.type === 'prekey') { - return this.decryptPreKeyMessage(address, envelope.content as PreKeyMessage); - } - return this.decryptRatchetMessage(address, envelope.content as RatchetMessage); + return this.runUnderPeerLock(address, async () => { + if (envelope.type === 'prekey') { + return this.decryptPreKeyMessage(address, envelope.content as PreKeyMessage); + } + return this.decryptRatchetMessage(address, envelope.content as RatchetMessage); + }); } private async decryptPreKeyMessage(address: string, message: PreKeyMessage): Promise { diff --git a/packages/shade-core/src/storage.ts b/packages/shade-core/src/storage.ts index 64c3c35..b7ba627 100644 --- a/packages/shade-core/src/storage.ts +++ b/packages/shade-core/src/storage.ts @@ -6,6 +6,38 @@ export interface RetiredIdentity { retiredAt: number; } +/** + * Persisted stream-transfer resume record. Holds enough state for either + * side of a transfer to resume after restart. The `streamSecret` MUST be + * encrypted before storage (see `@shade/transfer` for the deviceKey-based + * AES-GCM at-rest scheme). + */ +export interface PersistedStreamState { + streamId: string; + direction: 'send' | 'receive'; + peerAddress: string; + status: 'active' | 'paused' | 'finished' | 'aborted'; + /** JSON-serialized `StreamMetadata`. */ + metadataJson: string; + /** JSON-serialized `LaneInitSpec[]`. */ + partitionJson: string; + /** JSON-serialized per-lane progress array (laneId/nextSeq/bytesProcessed). */ + laneStateJson: string; + /** JSON-serialized I/O descriptor (file path / file handle reference / buffer). */ + ioDescriptorJson: string; + /** AES-GCM-encrypted streamSecret (under deviceKey). */ + secretEnc: Uint8Array; + /** AES-GCM nonce used for `secretEnc`. */ + secretNonce: Uint8Array; + /** + * Reserved for future hasher serialization. Empty in v0.2.0; resume + * re-hashes received bytes from disk. + */ + overallHashState?: string; + createdAt: number; + updatedAt: number; +} + /** * StorageProvider — abstract interface for persisting cryptographic state. * @@ -82,4 +114,27 @@ export interface StorageProvider { /** Remove retired identities older than the given timestamp */ pruneRetiredIdentities(olderThan: number): Promise; + + // ─── Stream-transfer resume state (optional, added in v0.2.0) ── + + /** + * Persist or update the stream-state row for a given streamId. Idempotent: + * upserts on `streamId`. Optional — providers that don't support resume + * can omit this and consumers will fall back to in-memory state. + */ + saveStreamState?(state: PersistedStreamState): Promise; + + /** Look up the stream-state row by streamId. Returns null if absent. */ + getStreamState?(streamId: string): Promise; + + /** Remove a stream-state row (e.g. on completion or abort). */ + removeStreamState?(streamId: string): Promise; + + /** List active or paused stream-state rows (for resume on startup). */ + listActiveStreamStates?( + direction?: 'send' | 'receive', + ): Promise; + + /** Prune stream-state rows in `'finished' | 'aborted'` status older than `olderThan`. */ + pruneStreamStates?(olderThan: number): Promise; } diff --git a/packages/shade-core/src/x3dh.ts b/packages/shade-core/src/x3dh.ts index df986ae..b3e7601 100644 --- a/packages/shade-core/src/x3dh.ts +++ b/packages/shade-core/src/x3dh.ts @@ -2,16 +2,14 @@ import type { CryptoProvider } from './crypto.js'; import type { StorageProvider } from './storage.js'; import type { IdentityKeyPair, - KeyPair, SignedPreKey, OneTimePreKey, PreKeyBundle, PreKeyMessage, RatchetMessage, - SessionState, } from './types.js'; import { deriveInitialRootKey } from './keys.js'; -import { InvalidSignatureError, PreKeyNotFoundError, UntrustedIdentityError } from './errors.js'; +import { InvalidSignatureError, PreKeyNotFoundError } from './errors.js'; /** * X3DH — Extended Triple Diffie-Hellman key agreement. @@ -75,7 +73,7 @@ export function createPreKeyBundle( signedPreKey: SignedPreKey, oneTimePreKey?: OneTimePreKey, ): PreKeyBundle { - return { + const bundle: PreKeyBundle = { registrationId, identitySigningKey: identityKey.signingPublicKey, identityDHKey: identityKey.dhPublicKey, @@ -84,10 +82,14 @@ export function createPreKeyBundle( publicKey: signedPreKey.keyPair.publicKey, signature: signedPreKey.signature, }, - oneTimePreKey: oneTimePreKey - ? { keyId: oneTimePreKey.keyId, publicKey: oneTimePreKey.keyPair.publicKey } - : undefined, }; + if (oneTimePreKey) { + bundle.oneTimePreKey = { + keyId: oneTimePreKey.keyId, + publicKey: oneTimePreKey.keyPair.publicKey, + }; + } + return bundle; } // ─── Alice: Initiate Session ───────────────────────────────── @@ -164,14 +166,15 @@ export async function processPreKeyBundle( // 6. Save trust for remote identity await storage.saveTrustedIdentity('pending', bundle.identityDHKey); - return { + const result: X3DHInitResult = { rootKey, ephemeralPublicKey: ephemeral.publicKey, signedPreKeyId: bundle.signedPreKey.keyId, - preKeyId, remoteIdentityKey: bundle.identityDHKey, remoteSignedPreKey: bundle.signedPreKey.publicKey, }; + if (preKeyId !== undefined) result.preKeyId = preKeyId; + return result; } // ─── Bob: Respond to PreKeyMessage ─────────────────────────── diff --git a/packages/shade-crypto-web/package.json b/packages/shade-crypto-web/package.json index fe7b45e..bc7be22 100644 --- a/packages/shade-crypto-web/package.json +++ b/packages/shade-crypto-web/package.json @@ -1,6 +1,6 @@ { "name": "@shade/crypto-web", - "version": "0.1.0", + "version": "0.3.0", "type": "module", "main": "src/index.ts", "types": "src/index.ts", diff --git a/packages/shade-crypto-web/src/memory-storage.ts b/packages/shade-crypto-web/src/memory-storage.ts index 37462bb..febaa3f 100644 --- a/packages/shade-crypto-web/src/memory-storage.ts +++ b/packages/shade-crypto-web/src/memory-storage.ts @@ -1,4 +1,4 @@ -import type { StorageProvider, IdentityKeyPair, SignedPreKey, OneTimePreKey, SessionState, RetiredIdentity } from '@shade/core'; +import type { StorageProvider, IdentityKeyPair, SignedPreKey, OneTimePreKey, SessionState, RetiredIdentity, PersistedStreamState } from '@shade/core'; import { constantTimeEqual } from '@shade/core'; /** @@ -103,4 +103,40 @@ export class MemoryStorage implements StorageProvider { async pruneRetiredIdentities(olderThan: number): Promise { this.retiredIdentities = this.retiredIdentities.filter((r) => r.retiredAt >= olderThan); } + + // ─── Stream-transfer resume state (v0.2.0) ──────────────── + + private streamStates = new Map(); + + async saveStreamState(state: PersistedStreamState): Promise { + this.streamStates.set(state.streamId, { ...state }); + } + + async getStreamState(streamId: string): Promise { + const v = this.streamStates.get(streamId); + return v ? { ...v } : null; + } + + async removeStreamState(streamId: string): Promise { + this.streamStates.delete(streamId); + } + + async listActiveStreamStates(direction?: 'send' | 'receive'): Promise { + const out: PersistedStreamState[] = []; + for (const s of this.streamStates.values()) { + if (s.status !== 'active' && s.status !== 'paused') continue; + if (direction !== undefined && s.direction !== direction) continue; + out.push({ ...s }); + } + out.sort((a, b) => b.updatedAt - a.updatedAt); + return out; + } + + async pruneStreamStates(olderThan: number): Promise { + for (const [id, s] of this.streamStates) { + if ((s.status === 'finished' || s.status === 'aborted') && s.updatedAt < olderThan) { + this.streamStates.delete(id); + } + } + } } diff --git a/packages/shade-crypto-web/src/provider.ts b/packages/shade-crypto-web/src/provider.ts index b964e9f..7463bb7 100644 --- a/packages/shade-crypto-web/src/provider.ts +++ b/packages/shade-crypto-web/src/provider.ts @@ -2,6 +2,19 @@ import type { CryptoProvider } from '@shade/core'; import { x25519 } from '@noble/curves/ed25519.js'; import { ed25519 } from '@noble/curves/ed25519.js'; +/** + * Cast a Uint8Array to a SubtleCrypto-compatible buffer source. TS 5.7+ + * tightened Uint8Array's buffer generic to ArrayBuffer, but our public API + * accepts Uint8Array. SubtleCrypto methods are runtime- + * compatible with both — only the type system needs the bridge. + * + * Returns `ArrayBuffer` (rather than the DOM `BufferSource` alias) so this + * file type-checks without DOM lib in the consumer's tsconfig. + */ +function bs(u: Uint8Array | undefined): ArrayBuffer { + return u as unknown as ArrayBuffer; +} + /** * SubtleCrypto + noble/curves implementation of CryptoProvider. * @@ -55,11 +68,11 @@ export class SubtleCryptoProvider implements CryptoProvider { aad?: Uint8Array, ): Promise<{ ciphertext: Uint8Array; nonce: Uint8Array }> { const nonce = this.randomBytes(12); - const aesKey = await this.subtle.importKey('raw', key, 'AES-GCM', false, ['encrypt']); + const aesKey = await this.subtle.importKey('raw', bs(key), 'AES-GCM', false, ['encrypt']); const encrypted = await this.subtle.encrypt( - { name: 'AES-GCM', iv: nonce, additionalData: aad }, + { name: 'AES-GCM', iv: bs(nonce), additionalData: aad ? bs(aad) : undefined }, aesKey, - plaintext, + bs(plaintext), ); return { ciphertext: new Uint8Array(encrypted), nonce }; } @@ -70,11 +83,11 @@ export class SubtleCryptoProvider implements CryptoProvider { nonce: Uint8Array, aad?: Uint8Array, ): Promise { - const aesKey = await this.subtle.importKey('raw', key, 'AES-GCM', false, ['decrypt']); + const aesKey = await this.subtle.importKey('raw', bs(key), 'AES-GCM', false, ['decrypt']); const decrypted = await this.subtle.decrypt( - { name: 'AES-GCM', iv: nonce, additionalData: aad }, + { name: 'AES-GCM', iv: bs(nonce), additionalData: aad ? bs(aad) : undefined }, aesKey, - ciphertext, + bs(ciphertext), ); return new Uint8Array(decrypted); } @@ -87,9 +100,9 @@ export class SubtleCryptoProvider implements CryptoProvider { info: Uint8Array, length: number, ): Promise { - const baseKey = await this.subtle.importKey('raw', ikm, 'HKDF', false, ['deriveBits']); + const baseKey = await this.subtle.importKey('raw', bs(ikm), 'HKDF', false, ['deriveBits']); const bits = await this.subtle.deriveBits( - { name: 'HKDF', hash: 'SHA-256', salt, info }, + { name: 'HKDF', hash: 'SHA-256', salt: bs(salt), info: bs(info) }, baseKey, length * 8, ); @@ -99,12 +112,12 @@ export class SubtleCryptoProvider implements CryptoProvider { async hmacSha256(key: Uint8Array, data: Uint8Array): Promise { const hmacKey = await this.subtle.importKey( 'raw', - key, + bs(key), { name: 'HMAC', hash: 'SHA-256' }, false, ['sign'], ); - const sig = await this.subtle.sign('HMAC', hmacKey, data); + const sig = await this.subtle.sign('HMAC', hmacKey, bs(data)); return new Uint8Array(sig); } diff --git a/packages/shade-dashboard/package.json b/packages/shade-dashboard/package.json index bedeab9..3acb945 100644 --- a/packages/shade-dashboard/package.json +++ b/packages/shade-dashboard/package.json @@ -1,6 +1,6 @@ { "name": "@shade/dashboard", - "version": "0.1.0", + "version": "0.3.0", "type": "module", "scripts": { "dev": "vite", diff --git a/packages/shade-files/package.json b/packages/shade-files/package.json new file mode 100644 index 0000000..8d5aee9 --- /dev/null +++ b/packages/shade-files/package.json @@ -0,0 +1,41 @@ +{ + "name": "@shade/files", + "version": "0.3.0", + "type": "module", + "main": "src/index.ts", + "types": "src/index.ts", + "exports": { + ".": { + "types": "./src/index.ts", + "import": "./src/index.ts" + }, + "./react": { + "types": "./src/react/index.ts", + "import": "./src/react/index.ts" + } + }, + "dependencies": { + "@shade/core": "workspace:*", + "@shade/crypto-web": "workspace:*", + "@shade/proto": "workspace:*", + "@shade/sdk": "workspace:*", + "@shade/streams": "workspace:*", + "@shade/transfer": "workspace:*", + "zod": "^3.23.8" + }, + "peerDependencies": { + "react": "^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + } + }, + "devDependencies": { + "@shade/server": "workspace:*", + "@types/react": "^19.2.14", + "fast-check": "^3.22.0", + "happy-dom": "^15.11.7", + "react": "^19.2.5" + } +} diff --git a/packages/shade-files/src/client/client.ts b/packages/shade-files/src/client/client.ts new file mode 100644 index 0000000..bcad3b0 --- /dev/null +++ b/packages/shade-files/src/client/client.ts @@ -0,0 +1,483 @@ +import type { Shade } from '@shade/sdk'; +import { + KIND_CUSTOM_V1, + KIND_DELETE_V1, + KIND_GET_THUMBNAIL_V1, + KIND_LIST_V1, + KIND_MKDIR_V1, + KIND_MOVE_V1, + KIND_READ_V1, + KIND_STAT_V1, + KIND_WRITE_V1, + MUTATION_OPS, + type StandardOp, +} from '../protocol/kinds.js'; +import type { CustomOpsMap } from '../server/custom-ops.js'; +import { generateIdempotencyKey, generateRequestId } from '../protocol/correlate.js'; +import { + base64ToBytes, + bytesToBase64, + canonicalRpcBytes, + hashArgs, +} from '../protocol/canonical.js'; +import { ShadeFileRpcChannel } from '../rpc/channel.js'; +import { PendingRpcRegistry, type RegisterOptions } from '../rpc/pending.js'; +import type { RpcRequest } from '../schemas/envelope.js'; +import { + CustomArgsSchema, + CustomResultSchema, + DeleteArgsSchema, + DeleteResultSchema, + GetThumbnailArgsSchema, + GetThumbnailResultSchema, + ListArgsSchema, + ListResultSchema, + MkdirArgsSchema, + MkdirResultSchema, + MoveArgsSchema, + MoveResultSchema, + ReadArgsSchema, + ReadResultSchema, + StatArgsSchema, + StatResultSchema, + WriteArgsSchema, + WriteResultSchema, + type DeleteArgs, + type DeleteResult, + type GetThumbnailArgs, + type ListArgs, + type ListResult, + type MkdirArgs, + type MkdirResult, + type MoveArgs, + type MoveResult, + type ReadArgs, + type StatResult, + type ThumbnailSize, + type WriteResult, +} from '../schemas/ops.js'; +import { ConflictError, InternalFileError } from '../schemas/errors.js'; +import { decideInline, INLINE_THRESHOLD, type WriteSource } from './inline-threshold.js'; +import type { ClientStreamsBridge } from './streams-bridge.js'; + +export interface BaseOpts { + signal?: AbortSignal; + /** Auto-generated for mutations if not provided. */ + idempotencyKey?: string; + /** Per-call timeout. Default 30_000 ms. */ + timeoutMs?: number; +} + +// ─── read/write public types ───────────────────────────────── + +export interface ReadInlineOutput { + kind: 'inline'; + bytes: Uint8Array; + size: number; + sha256: string; + contentType?: string; +} +export interface ReadStreamsOutput { + kind: 'streams'; + stream: ReadableStream; + size: number; + sha256: string; + contentType?: string; + /** Resolves once the entire transfer has been received and verified. */ + done(): Promise; +} +export type ReadOutput = ReadInlineOutput | ReadStreamsOutput; + +export interface WriteOpts extends BaseOpts { + contentType?: string; + overwrite?: boolean; + /** Force inline even if size > 256 KiB. Throws if input is too big. */ + forceInline?: boolean; +} + +export interface ReadOpts extends BaseOpts { + range?: { start: number; end: number }; + preferInline?: boolean; +} + +export interface ThumbnailResult { + bytes: Uint8Array; + format: 'png' | 'webp' | 'jpeg'; + width: number; + height: number; + sha256: string; +} + +// ─── FileClient interface ──────────────────────────────────── + +/** + * Untyped fallback for `FileClient.custom()` — used when the consumer + * hasn't extended `CustomOpsMap` for a given op name. + */ +type CustomOpArgs = K extends keyof CustomOpsMap + ? CustomOpsMap[K] extends { args: infer A } + ? A + : unknown + : unknown; +type CustomOpResponse = K extends keyof CustomOpsMap + ? CustomOpsMap[K] extends { response: infer R } + ? R + : unknown + : unknown; + +export interface FileClient { + list(path: string, opts?: BaseOpts & Partial>): Promise; + stat(path: string, opts?: BaseOpts): Promise; + mkdir(path: string, opts?: BaseOpts & Partial>): Promise; + delete(path: string, opts?: BaseOpts & Partial>): Promise; + move(src: string, dst: string, opts?: BaseOpts & Partial>): Promise; + read(path: string, opts?: ReadOpts): Promise; + write(path: string, input: WriteSource, opts?: WriteOpts): Promise; + getThumbnail( + path: string, + size: ThumbnailSize, + opts?: BaseOpts & { format?: 'png' | 'webp' | 'jpeg' }, + ): Promise; + /** + * Invoke a custom op registered on the server. Args/response types are + * pulled from `CustomOpsMap` via TypeScript declaration merging — see + * `server/custom-ops.ts` for the registration pattern. + */ + custom( + name: K & string, + args: CustomOpArgs, + opts?: BaseOpts, + ): Promise>; + close(): void; +} + +export interface CreateFileClientOptions { + /** Default per-call timeout. Default 30_000. */ + defaultTimeoutMs?: number; + /** Hard deadline for incoming-read awaits. Default 60_000. */ + ioTimeoutMs?: number; + /** + * Required for read/write `streams` ops. Coordinates inbound/outbound + * `@shade/transfer` transfers via `userMetadata.shadeFiles*Id` keys. + */ + streamsBridge?: ClientStreamsBridge; + /** + * Optional: sign the canonical bytes of every outgoing RPC envelope. + * Pluggable so apps can plug their own signing-key store (e.g., + * Ed25519-as-a-service, browser SubtleCrypto). When omitted, ships + * `'unsigned'` — the server's `verifySender` should also be unset, or + * be configured to accept the placeholder. + */ + signRequest?: (canonicalBytes: Uint8Array) => Promise | string; +} + +/** + * Client-side proxy for `@shade/files` ops. Each method ships an + * `RpcRequest` over `Shade.send`/`Shade.receive` and awaits the matching + * response (or error/timeout) from `PendingRpcRegistry`. + * + * Mutations auto-generate an idempotency key per logical call (not per + * attempt) so transparent retries under the SDK don't produce duplicates. + * + * Read/write content I/O over 256 KiB requires a `streamsBridge` to be + * passed via options — it coordinates the inbound/outbound `@shade/transfer` + * transfers that carry the actual bytes. + */ +export function createFileClient( + shade: Shade, + channel: ShadeFileRpcChannel, + pending: PendingRpcRegistry, + peerAddress: string, + options: CreateFileClientOptions = {}, +): FileClient { + const defaultTimeout = options.defaultTimeoutMs ?? 30_000; + const ioTimeoutMs = options.ioTimeoutMs ?? 60_000; + const streamsBridge = options.streamsBridge; + const signRequest = options.signRequest; + const senderAddress = shade.myAddress; + + async function request( + kind: string, + op: StandardOp | 'custom', + args: unknown, + opts: BaseOpts | undefined, + ): Promise { + const requestId = generateRequestId(); + const isMutation = MUTATION_OPS.has(op); + const idempotencyKey = + opts?.idempotencyKey ?? (isMutation ? generateIdempotencyKey() : undefined); + const signedAt = Date.now(); + let sig = 'unsigned'; + if (signRequest !== undefined) { + // Server reconstructs canonical bytes using `address = from`, which + // is OUR own address as authenticated by the ratchet. So we sign + // over the same identifier here. + const canonical = canonicalRpcBytes({ + address: senderAddress, + signedAt, + kind, + id: requestId, + argsHash: hashArgs(args), + }); + sig = await signRequest(canonical); + } + const env: RpcRequest = { + kind, + id: requestId, + args, + ...(idempotencyKey !== undefined ? { idempotencyKey } : {}), + sig, + signedAt, + }; + + const registerOpts: RegisterOptions = { + timeoutMs: opts?.timeoutMs ?? defaultTimeout, + onCancel: (reason) => { + // Fire-and-forget cancel envelope so server can release resources. + void channel + .send(peerAddress, { + kind: 'shade.fs.cancel/v1', + id: requestId, + reason, + }) + .catch(() => { + /* swallow — cancellation is best-effort */ + }); + }, + }; + if (opts?.signal !== undefined) registerOpts.signal = opts.signal; + + const pendingPromise = pending.register(requestId, registerOpts); + try { + await channel.send(peerAddress, env); + } catch (err) { + // If the send itself fails, the pending entry will never resolve; + // reject it directly. + pending.rejectAll(err); + throw err; + } + return pendingPromise; + } + + return { + async list(path, opts) { + const args: ListArgs = ListArgsSchema.parse({ + path, + ...(opts?.cursor !== undefined ? { cursor: opts.cursor } : {}), + ...(opts?.pageSize !== undefined ? { pageSize: opts.pageSize } : {}), + ...(opts?.filter !== undefined ? { filter: opts.filter } : {}), + }); + const raw = await request(KIND_LIST_V1, 'list', args, opts); + return ListResultSchema.parse(raw); + }, + + async stat(path, opts) { + const args = StatArgsSchema.parse({ path }); + const raw = await request(KIND_STAT_V1, 'stat', args, opts); + return StatResultSchema.parse(raw); + }, + + async mkdir(path, opts) { + const args = MkdirArgsSchema.parse({ + path, + ...(opts?.recursive !== undefined ? { recursive: opts.recursive } : {}), + }); + const raw = await request(KIND_MKDIR_V1, 'mkdir', args, opts); + return MkdirResultSchema.parse(raw); + }, + + async delete(path, opts) { + const args = DeleteArgsSchema.parse({ + path, + ...(opts?.recursive !== undefined ? { recursive: opts.recursive } : {}), + }); + const raw = await request(KIND_DELETE_V1, 'delete', args, opts); + return DeleteResultSchema.parse(raw); + }, + + async move(src, dst, opts) { + const args = MoveArgsSchema.parse({ + src, + dst, + ...(opts?.overwrite !== undefined ? { overwrite: opts.overwrite } : {}), + }); + const raw = await request(KIND_MOVE_V1, 'move', args, opts); + return MoveResultSchema.parse(raw); + }, + + async read(path, opts) { + const args: ReadArgs = ReadArgsSchema.parse({ + path, + ...(opts?.range !== undefined ? { range: opts.range } : {}), + ...(opts?.preferInline !== undefined ? { preferInline: opts.preferInline } : {}), + }); + const raw = await request(KIND_READ_V1, 'read', args, opts); + const wire = ReadResultSchema.parse(raw); + if (wire.kind === 'inline') { + const bytes = base64ToBytes(wire.bytesB64); + const out: ReadInlineOutput = { + kind: 'inline', + bytes, + size: wire.size, + sha256: wire.sha256, + ...(wire.contentType !== undefined ? { contentType: wire.contentType } : {}), + }; + return out; + } + // streams — wait for the matching incoming transfer via the bridge. + if (streamsBridge === undefined) { + throw new InternalFileError( + 'streams-bridge not configured: cannot consume streamed read', + ); + } + const bridgeSignal = opts?.signal ?? new AbortController().signal; + const parked = await streamsBridge.awaitRead(wire.streamId, { + expectedFrom: peerAddress, + signal: bridgeSignal, + timeoutMs: ioTimeoutMs, + }); + const out: ReadStreamsOutput = { + kind: 'streams', + stream: parked.readable, + size: wire.size, + sha256: wire.sha256, + ...(wire.contentType !== undefined ? { contentType: wire.contentType } : {}), + done: async () => { + await parked.done; + }, + }; + return out; + }, + + async write(path, input, opts) { + const decision = await decideInline(input); + const overwrite = opts?.overwrite ?? false; + const contentType = opts?.contentType ?? decision.contentType; + + if (decision.kind === 'inline' || opts?.forceInline === true) { + // Inline path — base64 in the RPC envelope. + const bytes = + decision.kind === 'inline' + ? decision.bytes + : await drainToUint8Array(decision.stream, decision.size ?? Number.POSITIVE_INFINITY); + if (bytes.byteLength > INLINE_THRESHOLD && opts?.forceInline !== true) { + throw new ConflictError( + `inline write exceeds ${INLINE_THRESHOLD}-byte threshold (got ${bytes.byteLength}); pass forceInline=true to override`, + ); + } + const args = WriteArgsSchema.parse({ + kind: 'inline', + path, + bytesB64: bytesToBase64(bytes), + ...(contentType !== undefined ? { contentType } : {}), + overwrite, + }); + const raw = await request(KIND_WRITE_V1, 'write', args, opts); + return WriteResultSchema.parse(raw); + } + + // Streams path — kick the upload, then ship the RPC. + if (streamsBridge === undefined) { + throw new InternalFileError( + 'streams-bridge not configured: cannot ship streamed write', + ); + } + const size = decision.size; + if (size === undefined) { + throw new ConflictError( + 'streams write requires a known plaintext size; pass `{ stream, size }` instead of a bare ReadableStream', + ); + } + const { writeId, handle } = await streamsBridge.initiateWrite({ + peer: peerAddress, + stream: decision.stream, + size, + ...(contentType !== undefined ? { contentType } : {}), + name: path, + ...(opts?.signal !== undefined ? { signal: opts.signal } : {}), + }); + const args = WriteArgsSchema.parse({ + kind: 'streams', + path, + size, + ...(contentType !== undefined ? { contentType } : {}), + overwrite, + writeId, + }); + try { + const [raw] = await Promise.all([ + request(KIND_WRITE_V1, 'write', args, opts), + handle.done(), + ]); + return WriteResultSchema.parse(raw); + } catch (err) { + // Best-effort cancel of the transfer on RPC failure. + await handle.abort('rpc-failed').catch(() => undefined); + throw err; + } + }, + + async getThumbnail(path, size, opts) { + const args: GetThumbnailArgs = GetThumbnailArgsSchema.parse({ + path, + size, + ...(opts?.format !== undefined ? { format: opts.format } : {}), + }); + const raw = await request(KIND_GET_THUMBNAIL_V1, 'getThumbnail', args, opts); + const wire = GetThumbnailResultSchema.parse(raw); + return { + bytes: base64ToBytes(wire.bytesB64), + format: wire.format, + width: wire.width, + height: wire.height, + sha256: wire.sha256, + }; + }, + + async custom(name, args, opts) { + const wireArgs = CustomArgsSchema.parse({ name, payload: args }); + // `custom` is a mutation in the rate-limit sense; auto-key for retries. + const raw = await request(KIND_CUSTOM_V1, 'custom' as StandardOp, wireArgs, opts); + const wire = CustomResultSchema.parse(raw); + // The result schema is `{ result: unknown }` — the inner `result` is + // already validated against the consumer's response schema on the + // server side, so we trust it here. + return wire.result as never; + }, + + close() { + pending.rejectAll(new Error('FileClient closed')); + }, + }; +} + +/** Drain a stream into a single buffer; used for the inline-write fallback. */ +async function drainToUint8Array( + stream: ReadableStream, + cap: number, +): Promise { + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + try { + while (true) { + const { value, done } = await reader.read(); + if (done) break; + if (value === undefined) continue; + chunks.push(value); + total += value.byteLength; + if (total > cap) { + throw new Error(`stream produced more than declared size cap (${cap})`); + } + } + } finally { + reader.releaseLock(); + } + const out = new Uint8Array(total); + let offset = 0; + for (const c of chunks) { + out.set(c, offset); + offset += c.byteLength; + } + return out; +} diff --git a/packages/shade-files/src/client/concurrency.ts b/packages/shade-files/src/client/concurrency.ts new file mode 100644 index 0000000..223fde9 --- /dev/null +++ b/packages/shade-files/src/client/concurrency.ts @@ -0,0 +1,91 @@ +/** + * Bounded-concurrency async map. + * + * Pulls items from an `AsyncIterable` source (lazily — never materializing + * the whole sequence) and runs `fn` on at most `concurrency` of them at + * once. Failures bubble unless `continueOnError` is set, in which case they + * are reported via `onError` and the pool keeps draining. + */ +import { CancelledError } from '../schemas/errors.js'; + +export interface ConcurrentMapOptions { + concurrency: number; + signal?: AbortSignal; + continueOnError?: boolean; + onError?: (item: T, err: unknown) => void; +} + +export async function runWithConcurrency( + source: AsyncIterable, + fn: (item: T) => Promise, + opts: ConcurrentMapOptions, +): Promise { + if (opts.concurrency < 1) throw new Error('concurrency must be ≥ 1'); + + const iter = source[Symbol.asyncIterator](); + const inFlight = new Set>(); + let firstError: unknown = null; + let aborted = false; + + function checkAbort(): void { + if (opts.signal?.aborted) { + aborted = true; + const reason = opts.signal.reason; + throw reason instanceof Error ? reason : new CancelledError(String(reason ?? 'aborted')); + } + } + + async function pumpOne(): Promise { + checkAbort(); + const next = await iter.next(); + if (next.done === true) return false; + const item = next.value; + const task = (async () => { + try { + await fn(item); + } catch (err) { + if (opts.continueOnError === true) { + opts.onError?.(item, err); + } else if (firstError === null) { + firstError = err; + } + } + })().finally(() => { + inFlight.delete(task); + }); + inFlight.add(task); + return true; + } + + try { + // Prime the pool + while (inFlight.size < opts.concurrency) { + if (firstError !== null || aborted) break; + const more = await pumpOne(); + if (!more) break; + } + // Maintain saturation + while (inFlight.size > 0) { + if (firstError !== null && opts.continueOnError !== true) break; + await Promise.race(inFlight); + while ( + inFlight.size < opts.concurrency && + firstError === null && + !aborted + ) { + const more = await pumpOne(); + if (!more) break; + } + } + // Drain whatever survived + if (inFlight.size > 0) await Promise.allSettled(inFlight); + } finally { + if (typeof iter.return === 'function') { + await iter.return().catch(() => undefined); + } + } + + if (firstError !== null && opts.continueOnError !== true) { + throw firstError; + } +} diff --git a/packages/shade-files/src/client/directory-types.ts b/packages/shade-files/src/client/directory-types.ts new file mode 100644 index 0000000..1778c2b --- /dev/null +++ b/packages/shade-files/src/client/directory-types.ts @@ -0,0 +1,97 @@ +/** + * Public types for directory operations (`walk`, `uploadDirectory`, + * `downloadDirectory`). + * + * `DirectoryHandleLike` and `FileHandleLike` are structurally compatible + * with the browser File System Access API + * (`FileSystemDirectoryHandle` / `FileSystemFileHandle`) so that browser + * consumers can pass them directly. For Bun/Node, use the adapter from + * `node-directory-handle.ts`. + */ + +export interface FileHandleLike { + readonly kind: 'file'; + readonly name: string; + /** Read-side accessor — used by `uploadDirectory`. */ + getFile(): Promise; + /** Write-side accessor — used by `downloadDirectory`. */ + createWritable(): Promise; +} + +export interface FileLike { + readonly name: string; + readonly size: number; + readonly type: string; + stream(): ReadableStream; + arrayBuffer(): Promise; +} + +export interface WritableStreamLike { + write(chunk: Uint8Array): Promise; + close(): Promise; + abort(reason?: unknown): Promise; +} + +export interface DirectoryHandleLike { + readonly kind: 'directory'; + readonly name: string; + /** Yield child entries (file or directory). Used by `uploadDirectory`. */ + entries(): AsyncIterable<[string, DirectoryHandleLike | FileHandleLike]>; + /** Used by `downloadDirectory` to create remote → local mapping. */ + getDirectoryHandle(name: string, opts?: { create?: boolean }): Promise; + getFileHandle(name: string, opts?: { create?: boolean }): Promise; +} + +// ─── Bulk transfer types ───────────────────────────────────── + +export type BulkTransferEvent = + | { type: 'plan'; totalFiles: number; totalBytes: number | undefined } + | { type: 'file-start'; path: string; size: number } + | { type: 'file-progress'; path: string; bytesDone: number; bytesTotal: number } + | { type: 'file-done'; path: string; bytesDone: number } + | { type: 'file-error'; path: string; error: unknown } + | { + type: 'progress'; + filesDone: number; + filesTotal: number; + bytesDone: number; + bytesTotal: number | undefined; + } + | { type: 'complete'; filesDone: number; bytesDone: number; durationMs: number } + | { type: 'abort'; reason: string }; + +export interface BulkTransferResult { + filesDone: number; + bytesDone: number; + durationMs: number; +} + +export interface BulkTransferHandle { + /** Async-iterable event stream — plan, per-file events, aggregate progress. */ + readonly events: AsyncIterable; + abort(reason?: string): Promise; + done(): Promise; +} + +export interface BulkOpts { + /** + * Max files in flight at once. Default 4. Capped at 16 to bound memory + * (each in-flight file may hold a buffered chunk; with 1 MiB chunks + + * 16 lanes that's still bounded). + */ + concurrency?: number; + /** + * Continue past per-file failures. Default false (fail-fast). + * When true, errors are emitted via `file-error` events and the whole + * transfer still resolves (with `done()` returning the count of + * successful files). + */ + continueOnError?: boolean; + /** Cancellation. */ + signal?: AbortSignal; +} + +/** Hard cap to bound memory regardless of caller-supplied concurrency. */ +export const MAX_BULK_CONCURRENCY = 16; +/** Default concurrency. */ +export const DEFAULT_BULK_CONCURRENCY = 4; diff --git a/packages/shade-files/src/client/download-directory.ts b/packages/shade-files/src/client/download-directory.ts new file mode 100644 index 0000000..52a1d9a --- /dev/null +++ b/packages/shade-files/src/client/download-directory.ts @@ -0,0 +1,316 @@ +/** + * Download an entire remote directory tree to a local `DirectoryHandleLike`. + * + * Mirror image of `uploadDirectory`: walks the remote tree via the shared + * `walk()` helper, creates local directories on the fly, and downloads each + * file with `client.read` (which routes inline or streams based on the + * server's response). Bounded concurrency keeps RPC inflight count low. + */ +import { walk, type WalkItem } from './walk.js'; +import { runWithConcurrency } from './concurrency.js'; +import type { FileClient } from './client.js'; +import { + DEFAULT_BULK_CONCURRENCY, + MAX_BULK_CONCURRENCY, + type BulkOpts, + type BulkTransferEvent, + type BulkTransferHandle, + type BulkTransferResult, + type DirectoryHandleLike, + type FileHandleLike, +} from './directory-types.js'; +import { CancelledError } from '../schemas/errors.js'; + +export interface DownloadDirectoryOptions extends BulkOpts { + /** Page size hint forwarded to the underlying `walk`. Default 200. */ + pageSize?: number; + /** Skip files already present locally. Default false (overwrite). */ + skipExisting?: boolean; +} + +export function downloadDirectory( + client: Pick, + remoteRoot: string, + local: DirectoryHandleLike, + opts: DownloadDirectoryOptions = {}, +): BulkTransferHandle { + const concurrency = Math.max( + 1, + Math.min(MAX_BULK_CONCURRENCY, opts.concurrency ?? DEFAULT_BULK_CONCURRENCY), + ); + const continueOnError = opts.continueOnError ?? false; + const externalSignal = opts.signal; + const pageSize = opts.pageSize ?? 200; + + const internalAbort = new AbortController(); + const combinedSignal = mergeSignals(externalSignal, internalAbort.signal); + + const events: BulkTransferEvent[] = []; + const eventResolvers: ((v: IteratorResult) => void)[] = []; + let eventsClosed = false; + + function emit(event: BulkTransferEvent): void { + if (eventsClosed) return; + if (eventResolvers.length > 0) { + eventResolvers.shift()!({ value: event, done: false }); + } else { + events.push(event); + } + } + + function closeEvents(): void { + if (eventsClosed) return; + eventsClosed = true; + while (eventResolvers.length > 0) { + eventResolvers.shift()!({ value: undefined as never, done: true }); + } + } + + let resolveDone!: (r: BulkTransferResult) => void; + let rejectDone!: (err: unknown) => void; + const donePromise = new Promise((resolve, reject) => { + resolveDone = resolve; + rejectDone = reject; + }); + + const startedAt = Date.now(); + let filesDone = 0; + let bytesDone = 0; + + void (async () => { + try { + // 1. Plan: walk the remote tree into [dirs, files] + const dirItems: WalkItem[] = []; + const fileItems: WalkItem[] = []; + try { + for await (const item of walk(client, remoteRoot, { + pageSize, + ...(combinedSignal !== undefined ? { signal: combinedSignal } : {}), + })) { + if (item.entry.kind === 'dir') dirItems.push(item); + else fileItems.push(item); + } + } catch (err) { + emit({ type: 'abort', reason: errMsg(err) }); + closeEvents(); + rejectDone(err); + return; + } + + const totalFiles = fileItems.length; + const bytesTotal = fileItems.reduce((acc, f) => acc + f.entry.size, 0); + emit({ type: 'plan', totalFiles, totalBytes: bytesTotal }); + + if (combinedSignal.aborted) { + emit({ type: 'abort', reason: errMsg(combinedSignal.reason) }); + closeEvents(); + rejectDone(combinedSignal.reason ?? new CancelledError()); + return; + } + + // 2. Pre-create local directories sequentially. + for (const d of dirItems) { + if (combinedSignal.aborted) break; + try { + await ensureLocalDir(local, d.relativePath); + } catch (err) { + if (!continueOnError) { + emit({ type: 'abort', reason: errMsg(err) }); + closeEvents(); + rejectDone(err); + return; + } + } + } + + // 3. Download files with bounded concurrency. + try { + await runWithConcurrency( + asyncIterableOf(fileItems), + async (item) => { + if (combinedSignal.aborted) { + throw new CancelledError('download aborted'); + } + emit({ type: 'file-start', path: item.relativePath, size: item.entry.size }); + try { + const fileHandle = await ensureLocalFile(local, item.relativePath); + if (opts.skipExisting === true) { + // Skip if file already exists with non-zero size + try { + const existing = await fileHandle.getFile(); + if (existing.size === item.entry.size) { + filesDone++; + bytesDone += existing.size; + emit({ type: 'file-done', path: item.relativePath, bytesDone: existing.size }); + emit({ + type: 'progress', + filesDone, + filesTotal: totalFiles, + bytesDone, + bytesTotal, + }); + return; + } + } catch { + /* not present yet — fall through to download */ + } + } + const writable = await fileHandle.createWritable(); + try { + const result = await client.read(item.absolutePath, { + signal: combinedSignal, + }); + if (result.kind === 'inline') { + if (result.bytes.byteLength > 0) await writable.write(result.bytes); + await writable.close(); + filesDone++; + bytesDone += result.bytes.byteLength; + emit({ type: 'file-done', path: item.relativePath, bytesDone: result.bytes.byteLength }); + } else { + await pipeReadableToWritable(result.stream, writable); + await writable.close(); + await result.done(); + filesDone++; + bytesDone += result.size; + emit({ type: 'file-done', path: item.relativePath, bytesDone: result.size }); + } + emit({ + type: 'progress', + filesDone, + filesTotal: totalFiles, + bytesDone, + bytesTotal, + }); + } catch (err) { + await writable.abort(err).catch(() => undefined); + throw err; + } + } catch (err) { + emit({ type: 'file-error', path: item.relativePath, error: err }); + throw err; + } + }, + { + concurrency, + signal: combinedSignal, + continueOnError, + onError: () => { + /* already emitted as 'file-error' */ + }, + }, + ); + } catch (err) { + emit({ type: 'abort', reason: errMsg(err) }); + closeEvents(); + rejectDone(err); + return; + } + + const durationMs = Date.now() - startedAt; + emit({ type: 'complete', filesDone, bytesDone, durationMs }); + closeEvents(); + resolveDone({ filesDone, bytesDone, durationMs }); + } catch (err) { + closeEvents(); + rejectDone(err); + } + })(); + + donePromise.catch(() => { + /* deliberate */ + }); + + return { + events: { + [Symbol.asyncIterator]() { + return { + next(): Promise> { + if (events.length > 0) { + return Promise.resolve({ value: events.shift()!, done: false }); + } + if (eventsClosed) { + return Promise.resolve({ value: undefined as never, done: true }); + } + return new Promise((resolve) => eventResolvers.push(resolve)); + }, + return(): Promise> { + return Promise.resolve({ value: undefined as never, done: true }); + }, + }; + }, + }, + async abort(reason) { + internalAbort.abort(new CancelledError(reason ?? 'manual abort')); + }, + done: () => donePromise, + }; +} + +// ─── Helpers ───────────────────────────────────────────────── + +async function ensureLocalDir( + root: DirectoryHandleLike, + relativePath: string, +): Promise { + const segments = relativePath.split('/').filter((s) => s !== ''); + let current = root; + for (const seg of segments) { + current = await current.getDirectoryHandle(seg, { create: true }); + } + return current; +} + +async function ensureLocalFile( + root: DirectoryHandleLike, + relativePath: string, +): Promise { + const segments = relativePath.split('/').filter((s) => s !== ''); + if (segments.length === 0) throw new Error('empty file path'); + let current = root; + for (let i = 0; i < segments.length - 1; i++) { + current = await current.getDirectoryHandle(segments[i]!, { create: true }); + } + return await current.getFileHandle(segments[segments.length - 1]!, { create: true }); +} + +async function pipeReadableToWritable( + readable: ReadableStream, + writable: { write(chunk: Uint8Array): Promise }, +): Promise { + const reader = readable.getReader(); + try { + while (true) { + const { value, done } = await reader.read(); + if (done) break; + if (value !== undefined && value.byteLength > 0) { + await writable.write(value); + } + } + } finally { + reader.releaseLock(); + } +} + +async function* asyncIterableOf(arr: T[]): AsyncIterable { + for (const item of arr) yield item; +} + +function errMsg(err: unknown): string { + if (err instanceof Error) return err.message; + return String(err ?? 'unknown error'); +} + +function mergeSignals( + external: AbortSignal | undefined, + internal: AbortSignal, +): AbortSignal { + if (external === undefined) return internal; + if (external.aborted) return external; + const controller = new AbortController(); + const onAbort = (sig: AbortSignal): void => { + controller.abort(sig.reason); + }; + external.addEventListener('abort', () => onAbort(external), { once: true }); + internal.addEventListener('abort', () => onAbort(internal), { once: true }); + return controller.signal; +} diff --git a/packages/shade-files/src/client/inline-threshold.ts b/packages/shade-files/src/client/inline-threshold.ts new file mode 100644 index 0000000..f51742d --- /dev/null +++ b/packages/shade-files/src/client/inline-threshold.ts @@ -0,0 +1,218 @@ +/** + * Decide whether a write should travel inline (base64 inside the RPC + * envelope) or via a dedicated `@shade/transfer` stream. + * + * Threshold: 256 KiB plaintext. Anything strictly above goes through the + * stream path; anything ≤ goes inline. + * + * For known-size inputs (`Uint8Array`, `Blob`, `File`) the decision is + * cheap: compare `byteLength`/`size` against the threshold. + * + * For `ReadableStream` we cannot know the size up front. We pull chunks + * into a temporary buffer until we either: + * - hit EOF before the threshold → inline (we have all the bytes) + * - cross the threshold → streams (the buffered prefix + the rest of the + * stream are returned via a fresh `ReadableStream` so the caller can + * feed it to `shade.upload`). + */ +/** Plaintext size at which inline transitions to streams. */ +export const INLINE_THRESHOLD = 256 * 1024; + +export type InlineDecision = + | { + kind: 'inline'; + bytes: Uint8Array; + contentType?: string; + } + | { + kind: 'streams'; + stream: ReadableStream; + /** Plaintext size when known (Blob/File). undefined for raw streams. */ + size?: number; + contentType?: string; + }; + +/** + * Public input shape for `FileClient.write`. Runtime discriminator helper. + */ +export type WriteSource = + | Uint8Array + | Blob + | File + | ReadableStream + | { stream: ReadableStream; size: number; contentType?: string }; + +export async function decideInline(input: WriteSource): Promise { + // 1. Uint8Array — direct size check + if (input instanceof Uint8Array) { + if (input.byteLength <= INLINE_THRESHOLD) { + return { kind: 'inline', bytes: input }; + } + return { + kind: 'streams', + stream: uint8ArrayToStream(input), + size: input.byteLength, + }; + } + + // 2. Blob / File — known size (Blob.type is a string; File extends Blob) + if (typeof Blob !== 'undefined' && input instanceof Blob) { + const blob: Blob = input; + const contentType = blob.type === '' ? undefined : blob.type; + if (blob.size <= INLINE_THRESHOLD) { + const bytes = new Uint8Array(await blob.arrayBuffer()); + return contentType !== undefined + ? { kind: 'inline', bytes, contentType } + : { kind: 'inline', bytes }; + } + return contentType !== undefined + ? { kind: 'streams', stream: blob.stream(), size: blob.size, contentType } + : { kind: 'streams', stream: blob.stream(), size: blob.size }; + } + + // 3. Pre-wrapped { stream, size } — trust caller's declared size + if ( + typeof input === 'object' && + input !== null && + 'stream' in input && + 'size' in input && + (input as { stream: unknown }).stream instanceof ReadableStream + ) { + const wrapped = input as { stream: ReadableStream; size: number; contentType?: string }; + if (wrapped.size <= INLINE_THRESHOLD) { + const bytes = await drainStream(wrapped.stream, wrapped.size); + const contentType = wrapped.contentType; + return contentType !== undefined + ? { kind: 'inline', bytes, contentType } + : { kind: 'inline', bytes }; + } + const contentType = wrapped.contentType; + return contentType !== undefined + ? { kind: 'streams', stream: wrapped.stream, size: wrapped.size, contentType } + : { kind: 'streams', stream: wrapped.stream, size: wrapped.size }; + } + + // 4. Bare ReadableStream — peek until threshold or EOF + if (input instanceof ReadableStream) { + return await peekStream(input); + } + + throw new TypeError( + `decideInline: unsupported input type ${Object.prototype.toString.call(input)}`, + ); +} + +/** + * Drain a stream into a Uint8Array, with a soft cap at `expected` + slack. + * Used for the inline path when the caller declared a size. + */ +async function drainStream( + stream: ReadableStream, + expected: number, +): Promise { + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + try { + while (true) { + const { value, done } = await reader.read(); + if (done) break; + if (value === undefined) continue; + chunks.push(value); + total += value.byteLength; + if (total > expected + INLINE_THRESHOLD) { + throw new Error( + `decideInline: stream produced more bytes (${total}) than declared size (${expected})`, + ); + } + } + } finally { + reader.releaseLock(); + } + return concat(chunks, total); +} + +/** + * Peek a `ReadableStream` of unknown length: buffer up to `INLINE_THRESHOLD + 1` + * bytes. If EOF first, return inline. Otherwise reconstruct a stream that + * yields the buffered prefix followed by the remainder. + */ +async function peekStream(stream: ReadableStream): Promise { + const reader = stream.getReader(); + const buffered: Uint8Array[] = []; + let total = 0; + try { + while (total <= INLINE_THRESHOLD) { + const { value, done } = await reader.read(); + if (done) { + reader.releaseLock(); + return { kind: 'inline', bytes: concat(buffered, total) }; + } + if (value === undefined) continue; + buffered.push(value); + total += value.byteLength; + } + // We have at least INLINE_THRESHOLD + 1 bytes buffered. Promote to streams. + const reconstructed = reconstructStream(buffered, reader); + return { kind: 'streams', stream: reconstructed }; + } catch (err) { + reader.releaseLock(); + throw err; + } +} + +interface MinimalReader { + read(): Promise<{ value: Uint8Array | undefined; done: boolean }>; + cancel(reason?: unknown): Promise; + releaseLock(): void; +} + +function reconstructStream( + prefix: Uint8Array[], + reader: MinimalReader, +): ReadableStream { + let prefixIdx = 0; + return new ReadableStream({ + async pull(controller) { + if (prefixIdx < prefix.length) { + controller.enqueue(prefix[prefixIdx]!); + prefixIdx++; + return; + } + const { value, done } = await reader.read(); + if (done) { + controller.close(); + reader.releaseLock(); + return; + } + if (value !== undefined) controller.enqueue(value); + }, + async cancel(reason) { + try { + await reader.cancel(reason); + } finally { + reader.releaseLock(); + } + }, + }); +} + +function uint8ArrayToStream(bytes: Uint8Array): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.enqueue(bytes); + controller.close(); + }, + }); +} + +function concat(chunks: Uint8Array[], total: number): Uint8Array { + const out = new Uint8Array(total); + let offset = 0; + for (const c of chunks) { + out.set(c, offset); + offset += c.byteLength; + } + return out; +} + diff --git a/packages/shade-files/src/client/memory-directory.ts b/packages/shade-files/src/client/memory-directory.ts new file mode 100644 index 0000000..4b7123b --- /dev/null +++ b/packages/shade-files/src/client/memory-directory.ts @@ -0,0 +1,145 @@ +/** + * In-memory `DirectoryHandleLike` implementation. + * + * Useful for tests and Node/Bun environments without the browser File + * System Access API. The shape is intentionally compatible with browser + * `FileSystemDirectoryHandle` so consumers can swap implementations. + */ +import type { + DirectoryHandleLike, + FileHandleLike, + FileLike, + WritableStreamLike, +} from './directory-types.js'; + +interface MemoryNode { + bytes: Uint8Array; + type: string; +} + +class MemoryFileLike implements FileLike { + constructor( + public readonly name: string, + private readonly node: MemoryNode, + ) {} + get size(): number { return this.node.bytes.byteLength; } + get type(): string { return this.node.type; } + stream(): ReadableStream { + const bytes = this.node.bytes; + return new ReadableStream({ + start(controller) { + if (bytes.byteLength > 0) controller.enqueue(bytes); + controller.close(); + }, + }); + } + async arrayBuffer(): Promise { + const out = new ArrayBuffer(this.node.bytes.byteLength); + new Uint8Array(out).set(this.node.bytes); + return out; + } +} + +class MemoryWritable implements WritableStreamLike { + private chunks: Uint8Array[] = []; + constructor(private readonly file: MemoryFile) {} + async write(chunk: Uint8Array): Promise { + this.chunks.push(chunk); + } + async close(): Promise { + let total = 0; + for (const c of this.chunks) total += c.byteLength; + const out = new Uint8Array(total); + let offset = 0; + for (const c of this.chunks) { + out.set(c, offset); + offset += c.byteLength; + } + this.file.commit(out); + } + async abort(): Promise { + this.chunks = []; + } +} + +class MemoryFile implements FileHandleLike { + readonly kind: 'file' = 'file'; + constructor( + public readonly name: string, + public readonly node: MemoryNode, + ) {} + async getFile(): Promise { + return new MemoryFileLike(this.name, this.node); + } + async createWritable(): Promise { + return new MemoryWritable(this); + } + /** Internal — used by `MemoryWritable.close()`. */ + commit(bytes: Uint8Array): void { + this.node.bytes = bytes; + } +} + +class MemoryDirectory implements DirectoryHandleLike { + readonly kind: 'directory' = 'directory'; + private readonly children = new Map(); + constructor(public readonly name: string) {} + + async *entries(): AsyncIterable<[string, DirectoryHandleLike | FileHandleLike]> { + for (const [name, child] of this.children) { + yield [name, child]; + } + } + + async getDirectoryHandle(name: string, opts?: { create?: boolean }): Promise { + const existing = this.children.get(name); + if (existing !== undefined) { + if (existing.kind !== 'directory') { + throw new Error(`'${name}' exists but is a file`); + } + return existing; + } + if (opts?.create !== true) throw new Error(`directory not found: ${name}`); + const dir = new MemoryDirectory(name); + this.children.set(name, dir); + return dir; + } + + async getFileHandle(name: string, opts?: { create?: boolean }): Promise { + const existing = this.children.get(name); + if (existing !== undefined) { + if (existing.kind !== 'file') { + throw new Error(`'${name}' exists but is a directory`); + } + return existing; + } + if (opts?.create !== true) throw new Error(`file not found: ${name}`); + const node: MemoryNode = { bytes: new Uint8Array(0), type: '' }; + const file = new MemoryFile(name, node); + this.children.set(name, file); + return file; + } + + /** Test helper: synchronously add a file with given bytes. */ + addFile(name: string, bytes: Uint8Array, type: string = ''): MemoryFile { + const node: MemoryNode = { bytes, type }; + const file = new MemoryFile(name, node); + this.children.set(name, file); + return file; + } + + /** Test helper: synchronously add a subdirectory. */ + addDir(name: string): MemoryDirectory { + const dir = new MemoryDirectory(name); + this.children.set(name, dir); + return dir; + } +} + +/** Construct an empty in-memory directory tree. */ +export function createMemoryDirectory(name: string = ''): DirectoryHandleLike & { + addFile(name: string, bytes: Uint8Array, type?: string): FileHandleLike; + addDir(name: string): DirectoryHandleLike; +} { + return new MemoryDirectory(name); +} diff --git a/packages/shade-files/src/client/streams-bridge.ts b/packages/shade-files/src/client/streams-bridge.ts new file mode 100644 index 0000000..733e9fc --- /dev/null +++ b/packages/shade-files/src/client/streams-bridge.ts @@ -0,0 +1,251 @@ +/** + * Client-side bridge between `@shade/files` content RPC ops and the + * `@shade/transfer` engine. + * + * Two responsibilities, mirror-image of `server/streams-bridge.ts`: + * 1. **Outbound writes (client → server, > 256 KiB).** When `FileClient.write` + * promotes to the streams path, this bridge calls `shade.upload(...)` + * with `userMetadata.shadeFilesWriteId = ` so the server can + * correlate the inbound transfer with the parallel `write` RPC. + * + * 2. **Inbound reads (server → client, > 256 KiB).** When `FileClient.read` + * gets a `{ kind: 'streams', streamId, ... }` RPC response, it asks + * this bridge for the matching incoming transfer's plaintext stream. + * The bridge subscribes to `shade.onIncomingTransfer` once and, on + * each transfer tagged with `userMetadata.shadeFilesReadStreamId`, + * **immediately** calls `accept(...)` (the engine rejects chunks that + * arrive before accept), pipes plaintext into a TransformStream, and + * parks the readable side until the matching read RPC awaits it. + */ +import type { TransferHandle, TransferProgress } from '@shade/transfer'; +import { generateRequestId } from '../protocol/correlate.js'; +import { OperationTimeoutError } from '../schemas/errors.js'; +import { + META_KEY_READ_STREAM_ID, + META_KEY_WRITE_ID, + type StreamsBridgeShade, +} from '../server/streams-bridge.js'; + +export interface AwaitReadOptions { + /** Sender address — must match `incoming.from` for delivery. */ + expectedFrom: string; + signal: AbortSignal; + /** Hard deadline (ms from now). Default 60_000. */ + timeoutMs?: number; +} + +export interface ParkedRead { + from: string; + /** Plaintext stream. The bridge already accepted the transfer. */ + readable: ReadableStream; + /** Resolves when the transfer fully completes (verified sha256 available). */ + done: Promise<{ sha256: string; bytesSent: number }>; + /** Underlying transfer handle — for abort propagation. */ + handle: TransferHandle; + arrivedAt: number; +} + +export interface ClientStreamsBridge { + /** + * Generate a fresh writeId, kick `shade.upload(...)` to `peer` with that + * id stamped in `userMetadata`, and return both the id (for the parallel + * RPC envelope) and the transfer handle (for `done()`/`abort()`). + */ + initiateWrite(opts: { + peer: string; + stream: ReadableStream; + size: number; + contentType?: string; + name?: string; + signal?: AbortSignal; + onProgress?: (p: TransferProgress) => void; + }): Promise<{ writeId: string; handle: TransferHandle }>; + + /** + * Wait for an inbound transfer carrying `userMetadata.shadeFilesReadStreamId + * === streamId` from `expectedFrom`. Resolves with the parked entry whose + * `readable` can be consumed by the caller. + */ + awaitRead(streamId: string, opts: AwaitReadOptions): Promise; + + destroy(): Promise; +} + +interface PendingReadWaiter { + resolve: (parked: ParkedRead) => void; + reject: (err: unknown) => void; + expectedFrom: string; + timer: ReturnType | null; + abortListener: (() => void) | null; + signal: AbortSignal; +} + +export interface CreateClientStreamsBridgeOptions { + /** Default deadline for `awaitRead` if the caller doesn't supply one. */ + defaultAwaitReadTimeoutMs?: number; + /** How long to retain a parked transfer waiting for its RPC. Default 60_000. */ + parkedReadTtlMs?: number; +} + +export async function createClientStreamsBridge( + shade: StreamsBridgeShade, + options: CreateClientStreamsBridgeOptions = {}, +): Promise { + const parkedReadTtlMs = options.parkedReadTtlMs ?? 60_000; + const defaultAwaitTimeoutMs = options.defaultAwaitReadTimeoutMs ?? 60_000; + + const parked = new Map(); + const waiters = new Map(); + let destroyed = false; + + const unsubscribe = await shade.onIncomingTransfer(async (incoming) => { + const readStreamId = incoming.metadata.userMetadata?.[META_KEY_READ_STREAM_ID]; + if (readStreamId === undefined) return; + + const ts = new TransformStream(); + let handle: TransferHandle; + try { + handle = await incoming.accept({ + output: { kind: 'pipe', pipeTo: ts.writable }, + }); + } catch (err) { + console.error('[shade-files client streams-bridge] accept failed:', err); + return; + } + + const arrival: ParkedRead = { + from: incoming.from, + readable: ts.readable, + done: handle.done().then((r) => ({ sha256: r.sha256, bytesSent: r.bytesSent })), + handle, + arrivedAt: Date.now(), + }; + arrival.done.catch(() => { + /* swallow until consumer awaits */ + }); + + const waiter = waiters.get(readStreamId); + if (waiter !== undefined) { + waiters.delete(readStreamId); + cleanupWaiter(waiter); + if (incoming.from !== waiter.expectedFrom) { + void handle.abort('sender-mismatch').catch(() => undefined); + waiter.reject( + new Error( + `streams-bridge: readStreamId=${readStreamId} delivered by ${incoming.from}, expected ${waiter.expectedFrom}`, + ), + ); + return; + } + waiter.resolve(arrival); + return; + } + + parked.set(readStreamId, arrival); + setTimeout(() => { + const stale = parked.get(readStreamId); + if (stale === arrival) { + parked.delete(readStreamId); + void handle.abort('rpc-timeout').catch(() => undefined); + } + }, parkedReadTtlMs).unref?.(); + }); + + function cleanupWaiter(w: PendingReadWaiter): void { + if (w.timer !== null) clearTimeout(w.timer); + if (w.abortListener !== null) { + w.signal.removeEventListener('abort', w.abortListener); + } + } + + return { + async initiateWrite(opts) { + if (destroyed) throw new Error('streams-bridge: destroyed'); + const writeId = generateRequestId(); + const transferOpts: import('@shade/transfer').TransferOptions = { + to: opts.peer, + input: opts.stream, + metadata: { + ...(opts.name !== undefined ? { name: opts.name } : {}), + ...(opts.contentType !== undefined ? { contentType: opts.contentType } : {}), + sizeBytes: opts.size, + userMetadata: { [META_KEY_WRITE_ID]: writeId }, + }, + ...(opts.signal !== undefined ? { signal: opts.signal } : {}), + ...(opts.onProgress !== undefined ? { onProgress: opts.onProgress } : {}), + }; + const handle = await shade.upload(transferOpts); + return { writeId, handle }; + }, + + async awaitRead(streamId, opts) { + if (destroyed) throw new Error('streams-bridge: destroyed'); + const ready = parked.get(streamId); + if (ready !== undefined) { + parked.delete(streamId); + if (ready.from !== opts.expectedFrom) { + void ready.handle.abort('sender-mismatch').catch(() => undefined); + throw new Error( + `streams-bridge: readStreamId=${streamId} delivered by ${ready.from}, expected ${opts.expectedFrom}`, + ); + } + return ready; + } + if (waiters.has(streamId)) { + throw new Error(`streams-bridge: readStreamId=${streamId} already awaited`); + } + const timeoutMs = opts.timeoutMs ?? defaultAwaitTimeoutMs; + return await new Promise((resolve, reject) => { + const w: PendingReadWaiter = { + resolve, + reject, + expectedFrom: opts.expectedFrom, + timer: null, + abortListener: null, + signal: opts.signal, + }; + w.timer = setTimeout(() => { + if (waiters.get(streamId) === w) { + waiters.delete(streamId); + cleanupWaiter(w); + reject(new OperationTimeoutError(`streams-bridge: readStreamId=${streamId} timed out after ${timeoutMs}ms`)); + } + }, timeoutMs); + if (opts.signal.aborted) { + cleanupWaiter(w); + reject(opts.signal.reason ?? new Error('aborted before await')); + return; + } + const onAbort = (): void => { + if (waiters.get(streamId) === w) { + waiters.delete(streamId); + cleanupWaiter(w); + reject(opts.signal.reason ?? new Error('aborted')); + } + }; + w.abortListener = onAbort; + opts.signal.addEventListener('abort', onAbort, { once: true }); + waiters.set(streamId, w); + }); + }, + + async destroy() { + if (destroyed) return; + destroyed = true; + unsubscribe(); + for (const w of waiters.values()) { + cleanupWaiter(w); + w.reject(new Error('streams-bridge: destroyed')); + } + waiters.clear(); + for (const p of parked.values()) { + try { + await p.handle.abort('bridge-destroyed'); + } catch { + /* swallow */ + } + } + parked.clear(); + }, + }; +} diff --git a/packages/shade-files/src/client/upload-directory.ts b/packages/shade-files/src/client/upload-directory.ts new file mode 100644 index 0000000..7168041 --- /dev/null +++ b/packages/shade-files/src/client/upload-directory.ts @@ -0,0 +1,310 @@ +/** + * Upload an entire local directory tree to a remote peer via a `FileClient`. + * + * Walks the local `DirectoryHandleLike` lazily, creates remote directories + * via `client.mkdir({ recursive: true })`, and uploads each file with + * `client.write` (which routes inline or streams based on size). A bounded + * concurrency pool keeps memory and inflight RPCs in check. + * + * Returns a `BulkTransferHandle` whose `events` stream emits `'plan'`, + * `'file-start'`, `'file-progress'` (currently emitted at file start + + * end), `'file-done'` / `'file-error'`, aggregate `'progress'`, and a + * final `'complete'` (or `'abort'`). + */ +import { posixJoin } from '../utils/path.js'; +import { runWithConcurrency } from './concurrency.js'; +import type { FileClient } from './client.js'; +import { + DEFAULT_BULK_CONCURRENCY, + MAX_BULK_CONCURRENCY, + type BulkOpts, + type BulkTransferEvent, + type BulkTransferHandle, + type BulkTransferResult, + type DirectoryHandleLike, + type FileHandleLike, +} from './directory-types.js'; +import { CancelledError } from '../schemas/errors.js'; + +interface PlannedFile { + /** Local file handle. */ + handle: FileHandleLike; + /** Path relative to the upload root. */ + relativePath: string; + /** Absolute remote path. */ + remoteAbsPath: string; +} + +interface PlannedDir { + remoteAbsPath: string; +} + +export interface UploadDirectoryOptions extends BulkOpts { + /** + * Pre-create remote directories before uploading files. Default true. + * Disable if the server-side `write` already mkdir-p's parents. + */ + precreateDirs?: boolean; +} + +export function uploadDirectory( + client: Pick, + local: DirectoryHandleLike, + remoteRoot: string, + opts: UploadDirectoryOptions = {}, +): BulkTransferHandle { + const concurrency = Math.max( + 1, + Math.min(MAX_BULK_CONCURRENCY, opts.concurrency ?? DEFAULT_BULK_CONCURRENCY), + ); + const precreateDirs = opts.precreateDirs ?? true; + const continueOnError = opts.continueOnError ?? false; + const externalSignal = opts.signal; + + const internalAbort = new AbortController(); + const combinedSignal = mergeSignals(externalSignal, internalAbort.signal); + + const events: BulkTransferEvent[] = []; + const eventResolvers: ((v: IteratorResult) => void)[] = []; + let eventsClosed = false; + + function emit(event: BulkTransferEvent): void { + if (eventsClosed) return; + if (eventResolvers.length > 0) { + eventResolvers.shift()!({ value: event, done: false }); + } else { + events.push(event); + } + } + + function closeEvents(): void { + if (eventsClosed) return; + eventsClosed = true; + while (eventResolvers.length > 0) { + eventResolvers.shift()!({ value: undefined as never, done: true }); + } + } + + let resolveDone!: (r: BulkTransferResult) => void; + let rejectDone!: (err: unknown) => void; + const donePromise = new Promise((resolve, reject) => { + resolveDone = resolve; + rejectDone = reject; + }); + + const startedAt = Date.now(); + let filesDone = 0; + let bytesDone = 0; + let bytesTotal = 0; + + // Run the upload pipeline asynchronously. + void (async () => { + try { + // 1. Plan: walk local tree, collect dirs + files + const plannedDirs: PlannedDir[] = []; + const plannedFiles: PlannedFile[] = []; + try { + await collect(local, '', remoteRoot, plannedDirs, plannedFiles, combinedSignal); + } catch (err) { + emit({ type: 'abort', reason: errMsg(err) }); + closeEvents(); + rejectDone(err); + return; + } + + const totalFiles = plannedFiles.length; + bytesTotal = plannedFiles.reduce( + (acc, f) => acc + (f.handle as FileHandleLike & { _size?: number })._size!, + 0, + ); + // Note: bytesTotal is computed lazily in collect() via cached sizes. + + emit({ type: 'plan', totalFiles, totalBytes: bytesTotal }); + + if (combinedSignal.aborted) { + emit({ type: 'abort', reason: errMsg(combinedSignal.reason) }); + closeEvents(); + rejectDone(combinedSignal.reason ?? new CancelledError()); + return; + } + + // 2. Pre-create remote directories sequentially (cheap, avoids races + // when many uploads target the same parent). + if (precreateDirs) { + for (const d of plannedDirs) { + if (combinedSignal.aborted) break; + try { + await client.mkdir(d.remoteAbsPath, { recursive: true }); + } catch (err) { + if (!isAlreadyExistsError(err)) { + if (!continueOnError) { + emit({ type: 'abort', reason: errMsg(err) }); + closeEvents(); + rejectDone(err); + return; + } + } + } + } + } + + // 3. Upload files with bounded concurrency + try { + await runWithConcurrency( + asyncIterableOf(plannedFiles), + async (planned) => { + if (combinedSignal.aborted) { + throw new CancelledError('upload aborted'); + } + const file = await planned.handle.getFile(); + const size = file.size; + emit({ type: 'file-start', path: planned.relativePath, size }); + try { + const writeOpts: { contentType?: string; signal?: AbortSignal } = {}; + if (file.type !== '') writeOpts.contentType = file.type; + writeOpts.signal = combinedSignal; + if (size === 0) { + // Edge case: empty file — write empty buffer. + await client.write(planned.remoteAbsPath, new Uint8Array(0), writeOpts); + } else { + await client.write( + planned.remoteAbsPath, + { stream: file.stream(), size, ...(file.type !== '' ? { contentType: file.type } : {}) }, + writeOpts, + ); + } + filesDone++; + bytesDone += size; + emit({ type: 'file-done', path: planned.relativePath, bytesDone: size }); + emit({ + type: 'progress', + filesDone, + filesTotal: totalFiles, + bytesDone, + bytesTotal, + }); + } catch (err) { + emit({ type: 'file-error', path: planned.relativePath, error: err }); + throw err; + } + }, + { + concurrency, + signal: combinedSignal, + continueOnError, + onError: () => { + /* already emitted as 'file-error' */ + }, + }, + ); + } catch (err) { + emit({ type: 'abort', reason: errMsg(err) }); + closeEvents(); + rejectDone(err); + return; + } + + const durationMs = Date.now() - startedAt; + emit({ type: 'complete', filesDone, bytesDone, durationMs }); + closeEvents(); + resolveDone({ filesDone, bytesDone, durationMs }); + } catch (err) { + // Belt-and-suspenders: any unhandled error reaches here. + closeEvents(); + rejectDone(err); + } + })(); + + // Suppress unhandled-rejection until consumer awaits done(). + donePromise.catch(() => { + /* deliberate */ + }); + + return { + events: { + [Symbol.asyncIterator]() { + return { + next(): Promise> { + if (events.length > 0) { + return Promise.resolve({ value: events.shift()!, done: false }); + } + if (eventsClosed) { + return Promise.resolve({ value: undefined as never, done: true }); + } + return new Promise((resolve) => eventResolvers.push(resolve)); + }, + return(): Promise> { + // Caller broke out — stop iterating but keep the bulk going. + return Promise.resolve({ value: undefined as never, done: true }); + }, + }; + }, + }, + async abort(reason) { + internalAbort.abort(new CancelledError(reason ?? 'manual abort')); + }, + done: () => donePromise, + }; +} + +// ─── Helpers ───────────────────────────────────────────────── + +async function collect( + dir: DirectoryHandleLike, + relPrefix: string, + remoteRoot: string, + plannedDirs: PlannedDir[], + plannedFiles: PlannedFile[], + signal: AbortSignal, +): Promise { + for await (const [name, handle] of dir.entries()) { + if (signal.aborted) return; + const relPath = relPrefix === '' ? name : `${relPrefix}/${name}`; + if (handle.kind === 'directory') { + const remoteAbs = posixJoin(remoteRoot, relPath); + plannedDirs.push({ remoteAbsPath: remoteAbs }); + await collect(handle as DirectoryHandleLike, relPath, remoteRoot, plannedDirs, plannedFiles, signal); + } else { + // Cache size on the handle so the planning total is accurate. + const file = await (handle as FileHandleLike).getFile(); + (handle as FileHandleLike & { _size?: number })._size = file.size; + plannedFiles.push({ + handle: handle as FileHandleLike, + relativePath: relPath, + remoteAbsPath: posixJoin(remoteRoot, relPath), + }); + } + } +} + +async function* asyncIterableOf(arr: T[]): AsyncIterable { + for (const item of arr) yield item; +} + +function isAlreadyExistsError(err: unknown): boolean { + if (err === null || typeof err !== 'object') return false; + const code = (err as { code?: string; payload?: { code?: string } }).code; + if (code === 'SHADE_FS_CONFLICT') return true; + const payloadCode = (err as { payload?: { code?: string } }).payload?.code; + return payloadCode === 'CONFLICT'; +} + +function errMsg(err: unknown): string { + if (err instanceof Error) return err.message; + return String(err ?? 'unknown error'); +} + +function mergeSignals( + external: AbortSignal | undefined, + internal: AbortSignal, +): AbortSignal { + if (external === undefined) return internal; + if (external.aborted) return external; + const controller = new AbortController(); + const onAbort = (sig: AbortSignal): void => { + controller.abort(sig.reason); + }; + external.addEventListener('abort', () => onAbort(external), { once: true }); + internal.addEventListener('abort', () => onAbort(internal), { once: true }); + return controller.signal; +} diff --git a/packages/shade-files/src/client/walk.ts b/packages/shade-files/src/client/walk.ts new file mode 100644 index 0000000..58e3fed --- /dev/null +++ b/packages/shade-files/src/client/walk.ts @@ -0,0 +1,89 @@ +/** + * Async-iterable directory walker. + * + * Depth-first by default — yields a directory's entries before descending + * into the next sibling. Memory-bounded: never materializes the whole tree; + * uses `client.list` page-by-page. + * + * Designed for arbitrarily-large remote trees: the consumer can break out + * of the iterator at any point and the walk halts cleanly. + */ +import { posixJoin } from '../utils/path.js'; +import { CancelledError } from '../schemas/errors.js'; +import type { FileEntry } from '../schemas/file-entry.js'; +import type { FileClient } from './client.js'; + +export interface WalkOpts { + /** Hard cap on recursion depth. Default 32. */ + maxDepth?: number; + /** Cancellation. */ + signal?: AbortSignal; + /** + * Apply a per-entry filter. Return `false` to skip an entry (and, for + * directories, to skip descending into it). Default: include all. + */ + filter?: (entry: FileEntry, relativePath: string) => boolean; + /** Page size hint for `client.list`. Default 200. */ + pageSize?: number; +} + +export interface WalkItem { + entry: FileEntry; + /** Path relative to the walk root. Includes the entry's own name. */ + relativePath: string; + /** Absolute path on the remote (root + relativePath). */ + absolutePath: string; + /** Depth from root — 1 for direct children. */ + depth: number; +} + +const DEFAULT_MAX_DEPTH = 32; +const DEFAULT_PAGE_SIZE = 200; + +export async function* walk( + client: Pick, + rootPath: string, + opts: WalkOpts = {}, +): AsyncIterable { + const maxDepth = opts.maxDepth ?? DEFAULT_MAX_DEPTH; + const pageSize = opts.pageSize ?? DEFAULT_PAGE_SIZE; + + function checkAbort(): void { + if (opts.signal?.aborted) { + throw new CancelledError( + opts.signal.reason instanceof Error + ? opts.signal.reason.message + : 'walk aborted', + ); + } + } + + async function* descend(absDir: string, depth: number, relPrefix: string): AsyncIterable { + if (depth > maxDepth) return; + let cursor: string | undefined; + do { + checkAbort(); + const page = await client.list(absDir, { + pageSize, + ...(cursor !== undefined ? { cursor } : {}), + ...(opts.signal !== undefined ? { signal: opts.signal } : {}), + }); + for (const entry of page.entries) { + checkAbort(); + const relPath = relPrefix === '' ? entry.name : `${relPrefix}/${entry.name}`; + const absPath = posixJoin(absDir, entry.name); + if (opts.filter !== undefined && !opts.filter(entry, relPath)) { + continue; + } + const item: WalkItem = { entry, relativePath: relPath, absolutePath: absPath, depth }; + yield item; + if (entry.kind === 'dir') { + yield* descend(absPath, depth + 1, relPath); + } + } + cursor = page.hasMore ? page.nextCursor : undefined; + } while (cursor !== undefined); + } + + yield* descend(rootPath, 1, ''); +} diff --git a/packages/shade-files/src/index.ts b/packages/shade-files/src/index.ts new file mode 100644 index 0000000..58a2ae2 --- /dev/null +++ b/packages/shade-files/src/index.ts @@ -0,0 +1,198 @@ +// Schemas — Zod runtime + compile-time types +export * from './schemas/index.js'; + +// Protocol primitives — kinds, correlation, canonical bytes, envelope codec +export { + SHADE_FILES_VERSION, + KIND_PREFIX, + SUPPORTED_KINDS, + isSupportedKind, +} from './protocol/version.js'; +export type { SupportedKind } from './protocol/version.js'; +export { + KIND_LIST_V1, + KIND_STAT_V1, + KIND_MKDIR_V1, + KIND_DELETE_V1, + KIND_MOVE_V1, + KIND_READ_V1, + KIND_WRITE_V1, + KIND_GET_THUMBNAIL_V1, + KIND_CUSTOM_V1, + KIND_ERROR_V1, + KIND_CANCEL_V1, + responseKindOf, + opOfKind, + MUTATION_OPS, +} from './protocol/kinds.js'; +export type { StandardOp } from './protocol/kinds.js'; +export { + generateRequestId, + generateIdempotencyKey, + base64UrlEncode, + base64UrlDecode, +} from './protocol/correlate.js'; +export { + canonicalRpcBytes, + canonicalJsonStringify, + hashArgs, + bytesToHex, + bytesToBase64, + base64ToBytes, +} from './protocol/canonical.js'; +export { + encodeEnvelope, + looksLikeFileEnvelope, + tryParseEnvelope, + classify, +} from './protocol/envelope-codec.js'; +export type { ClassifiedEnvelope } from './protocol/envelope-codec.js'; + +// RPC channel — wires Shade.send/onMessage to the file-RPC routing layer +export { ShadeFileRpcChannel } from './rpc/channel.js'; +export type { RpcChannelHooks } from './rpc/channel.js'; +export { PendingRpcRegistry } from './rpc/pending.js'; +export type { RegisterOptions } from './rpc/pending.js'; + +// Server side +export { createFileHandler, INTERNAL_SYMBOL } from './server/handler.js'; +export type { + FileHandler, + FileHandlerConfig, + FileHandlerOps, +} from './server/handler.js'; +export type { OpContext, OpKind } from './server/handler-context.js'; +export { validatePath } from './server/path-policy.js'; +export type { PathPolicy, PathValidationResult } from './server/path-policy.js'; +export { IdempotencyCache } from './server/idempotency-cache.js'; +export type { IdempotencyCacheOptions } from './server/idempotency-cache.js'; +export { RateLimiter } from './server/rate-limiter.js'; +export type { RateLimitConfig } from './server/rate-limiter.js'; +export { createCursorBuilder } from './server/cursor.js'; +export type { CursorBuilder } from './server/cursor.js'; +export { + createServerStreamsBridge, + META_KEY_READ_STREAM_ID, + META_KEY_WRITE_ID, +} from './server/streams-bridge.js'; +export type { + ServerStreamsBridge, + StreamsBridgeShade, + CreateServerStreamsBridgeOptions, + ParkedWrite, + AwaitWriteOptions, +} from './server/streams-bridge.js'; +export type { + UserReadResult, + UserReadResultInline, + UserReadResultStreams, + UserWriteArgs, + UserWriteContent, + UserWriteContentInline, + UserWriteContentStreams, + UserThumbnailResult, +} from './server/io-types.js'; +export { + assertThumbnailFormat, + isThumbnailFormat, +} from './server/thumbnail.js'; +export type { ThumbnailFormat } from './server/thumbnail.js'; + +// Client side +export { createFileClient } from './client/client.js'; +export type { + FileClient, + BaseOpts, + CreateFileClientOptions, + ReadOpts, + WriteOpts, + ReadOutput, + ReadInlineOutput, + ReadStreamsOutput, + ThumbnailResult, +} from './client/client.js'; +export { + createClientStreamsBridge, +} from './client/streams-bridge.js'; +export type { + ClientStreamsBridge, + CreateClientStreamsBridgeOptions, + AwaitReadOptions, + ParkedRead, +} from './client/streams-bridge.js'; +export { + decideInline, + INLINE_THRESHOLD, +} from './client/inline-threshold.js'; +export type { + InlineDecision, + WriteSource, +} from './client/inline-threshold.js'; + +// Directory ops — walk + bulk transfers +export { walk } from './client/walk.js'; +export type { WalkOpts, WalkItem } from './client/walk.js'; +export { uploadDirectory } from './client/upload-directory.js'; +export type { UploadDirectoryOptions } from './client/upload-directory.js'; +export { downloadDirectory } from './client/download-directory.js'; +export type { DownloadDirectoryOptions } from './client/download-directory.js'; +export { runWithConcurrency } from './client/concurrency.js'; +export type { ConcurrentMapOptions } from './client/concurrency.js'; +export { + DEFAULT_BULK_CONCURRENCY, + MAX_BULK_CONCURRENCY, +} from './client/directory-types.js'; +export type { + BulkOpts, + BulkTransferEvent, + BulkTransferHandle, + BulkTransferResult, + DirectoryHandleLike, + FileHandleLike, + FileLike, + WritableStreamLike, +} from './client/directory-types.js'; +export { createMemoryDirectory } from './client/memory-directory.js'; + +// Custom-op registry typing (declaration-merged by consumers) +export type { + CustomOpDef, + CustomOpsMap, + CustomOpRegistration, + CustomOpRegistrations, +} from './server/custom-ops.js'; + +// Production hooks: metrics +export { + METRIC_BYTES_IN, + METRIC_BYTES_OUT, + METRIC_FINGERPRINT_REJECT_TOTAL, + METRIC_IDEMPOTENCY_CONFLICT_TOTAL, + METRIC_IDEMPOTENCY_HIT_TOTAL, + METRIC_OP_DURATION_MS, + METRIC_OP_TOTAL, + METRIC_RATE_LIMIT_REJECT_TOTAL, + METRIC_SIGNATURE_REJECT_TOTAL, + NOOP_METRIC_SINK, +} from './server/metrics.js'; +export type { MetricSink, MetricTags } from './server/metrics.js'; + +export { MAX_SIGNATURE_AGE_MS } from './server/handler.js'; + +// High-level SDK integration entrypoint +export { createFilesNamespace } from './integration/files-namespace.js'; +export type { FilesNamespace } from './integration/files-namespace.js'; + +// Integration helpers — wire handler + pending registry onto a channel +export { attachFileHandler } from './integration/wire-server.js'; +export { attachClientRouting } from './integration/wire-client.js'; + +// Path utilities +export { + posixNormalize, + posixJoin, + posixDirname, + posixBasename, + decodePercentEscapes, + isPathInside, +} from './utils/path.js'; diff --git a/packages/shade-files/src/integration/files-namespace.ts b/packages/shade-files/src/integration/files-namespace.ts new file mode 100644 index 0000000..5366071 --- /dev/null +++ b/packages/shade-files/src/integration/files-namespace.ts @@ -0,0 +1,142 @@ +/** + * High-level `FilesNamespace` — the entry point that the SDK exposes via + * `Shade.files`. Memoizes the underlying `ShadeFileRpcChannel` and bridges + * so a single Shade can simultaneously serve files AND consume them from + * peers without paying the setup cost twice. + */ +import type { Shade } from '@shade/sdk'; +import { + attachClientRouting, + attachFileHandler, + createClientStreamsBridge, + createFileClient, + createFileHandler, + createServerStreamsBridge, + PendingRpcRegistry, + ShadeFileRpcChannel, + type ClientStreamsBridge, + type CreateFileClientOptions, + type FileClient, + type FileHandler, + type FileHandlerConfig, + type ServerStreamsBridge, +} from '../index.js'; +import { IdempotencyCache } from '../server/idempotency-cache.js'; + +export interface FilesNamespace { + /** + * Register a file handler. Throws if a handler is already attached on + * this Shade — only one server per Shade. The returned function detaches + * the handler and tears down its idempotency / retention timers. + */ + serve(handler: FileHandlerConfig): Promise<() => Promise>; + /** + * Build a typed file client for `peer`. Multiple concurrent clients to + * different peers share the same channel + streams bridge. + */ + client(peer: string, opts?: Omit): Promise; + /** Tear down channel + bridges. After destroy(), serve()/client() throw. */ + destroy(): Promise; +} + +interface NamespaceState { + channel: ShadeFileRpcChannel; + pending: PendingRpcRegistry; + serverBridge: ServerStreamsBridge | null; + clientBridge: ClientStreamsBridge | null; + serverHandler: FileHandler | null; + serverDetach: (() => void) | null; + clientDetach: (() => void) | null; + destroyed: boolean; +} + +/** + * Construct a `FilesNamespace` bound to a Shade instance. The SDK's + * `Shade.files` getter calls this lazily and memoizes the result. + */ +export function createFilesNamespace(shade: Shade): FilesNamespace { + const state: NamespaceState = { + channel: new ShadeFileRpcChannel(shade), + pending: new PendingRpcRegistry(), + serverBridge: null, + clientBridge: null, + serverHandler: null, + serverDetach: null, + clientDetach: null, + destroyed: false, + }; + + function ensureAlive(): void { + if (state.destroyed) throw new Error('FilesNamespace: destroyed'); + } + + return { + async serve(handlerConfig) { + ensureAlive(); + if (state.serverHandler !== null) { + throw new Error('FilesNamespace: a handler is already registered (one per Shade)'); + } + // Lazy server-side streams bridge. + if (state.serverBridge === null) { + state.serverBridge = await createServerStreamsBridge(shade); + } + const handler = createFileHandler(shade, { + ...handlerConfig, + streamsBridge: state.serverBridge, + }); + const detach = attachFileHandler(state.channel, handler); + state.serverHandler = handler; + state.serverDetach = detach; + + // Wire BackgroundHooks.onPruneFiles to the new handler's idempotency + // cache. Use the symbol-exposed internals (works because FileHandler + // attaches them via Object.assign). + const internals = (handler as unknown as { [k: symbol]: { idempotency: IdempotencyCache } })[ + Symbol.for('@shade/files/internal') + ]; + const background = (shade as unknown as { background?: { setHook?: (n: string, f: () => void) => void } }).background; + if (background?.setHook !== undefined && internals !== undefined) { + background.setHook('onPruneFiles', () => { + internals.idempotency.prune(); + }); + } + + return async () => { + if (state.serverDetach !== null) state.serverDetach(); + state.serverHandler = null; + state.serverDetach = null; + if (background?.setHook !== undefined) { + background.setHook('onPruneFiles', undefined as unknown as () => void); + } + }; + }, + + async client(peer, opts = {}) { + ensureAlive(); + // Lazy client-side streams bridge. + if (state.clientBridge === null) { + state.clientBridge = await createClientStreamsBridge(shade); + } + // Attach response routing once. + if (state.clientDetach === null) { + state.clientDetach = attachClientRouting(state.channel, state.pending); + } + return createFileClient(shade, state.channel, state.pending, peer, { + ...opts, + streamsBridge: state.clientBridge, + }); + }, + + async destroy() { + if (state.destroyed) return; + state.destroyed = true; + if (state.serverDetach !== null) state.serverDetach(); + if (state.clientDetach !== null) state.clientDetach(); + if (state.serverHandler !== null) state.serverHandler.destroy(); + if (state.serverBridge !== null) await state.serverBridge.destroy(); + if (state.clientBridge !== null) await state.clientBridge.destroy(); + state.channel.destroy(); + state.pending.rejectAll(new Error('FilesNamespace destroyed')); + }, + }; +} diff --git a/packages/shade-files/src/integration/wire-client.ts b/packages/shade-files/src/integration/wire-client.ts new file mode 100644 index 0000000..579a49e --- /dev/null +++ b/packages/shade-files/src/integration/wire-client.ts @@ -0,0 +1,23 @@ +import { ShadeFileRpcChannel } from '../rpc/channel.js'; +import { PendingRpcRegistry } from '../rpc/pending.js'; + +/** + * Wire a `PendingRpcRegistry` onto a channel so incoming `response` and + * `error` envelopes resolve the matching client-side promises. + * + * The same channel can serve multiple clients (one per peer); the + * registry is shared because correlation IDs are globally unique. + */ +export function attachClientRouting( + channel: ShadeFileRpcChannel, + pending: PendingRpcRegistry, +): () => void { + const previous = channel.setHooks({ + onResponse: (_from, env) => pending.resolveResponse(env.envelope), + onError: (_from, env) => pending.resolveError(env.envelope), + }); + return () => { + pending.rejectAll(new Error('client routing detached')); + channel.setHooks(previous); + }; +} diff --git a/packages/shade-files/src/integration/wire-server.ts b/packages/shade-files/src/integration/wire-server.ts new file mode 100644 index 0000000..9df6f53 --- /dev/null +++ b/packages/shade-files/src/integration/wire-server.ts @@ -0,0 +1,35 @@ +import { ShadeFileRpcChannel } from '../rpc/channel.js'; +import type { FileHandler } from '../server/handler.js'; + +/** + * Connect a `FileHandler` to a `ShadeFileRpcChannel`. Returns an unsubscribe + * function that detaches the handler. Throws if a handler is already wired + * on this channel. + * + * The wiring: + * - On `request` envelopes: invoke `handler.handleRequest`, then send the + * returned response/error envelope back to the requester. + * - On `cancel` envelopes: forward to `handler.handleCancel`. + */ +export function attachFileHandler( + channel: ShadeFileRpcChannel, + handler: FileHandler, +): () => void { + const previous = channel.setHooks({ + onRequest: async (from, env) => { + const response = await handler.handleRequest(from, env.envelope); + try { + await channel.send(from, response); + } catch (err) { + console.error('[shade-files] failed to send response:', err); + } + }, + onCancel: (from, env) => { + handler.handleCancel(from, env.envelope); + }, + }); + return () => { + handler.destroy(); + channel.setHooks(previous); + }; +} diff --git a/packages/shade-files/src/protocol/canonical.ts b/packages/shade-files/src/protocol/canonical.ts new file mode 100644 index 0000000..fb8b833 --- /dev/null +++ b/packages/shade-files/src/protocol/canonical.ts @@ -0,0 +1,72 @@ +import { sha256 } from '@noble/hashes/sha2.js'; + +/** + * Canonical bytes for an `@shade/files` RPC request signature. + * + * "rpc\0addr=\0at=\0kind=\0id=\0argsHash=\0" + * + * Mirror of `canonicalControlBytes` in `@shade/sdk/src/streams-bridge.ts:188`. + * The signature binds the (sender, op kind, request id, args, signedAt) + * tuple — defense-in-depth on top of Double Ratchet authentication. + */ +export function canonicalRpcBytes(args: { + address: string; + signedAt: number; + kind: string; + id: string; + argsHash: Uint8Array; +}): Uint8Array { + const enc = new TextEncoder(); + const fields = [ + `rpc\0`, + `addr=${args.address}\0`, + `at=${args.signedAt}\0`, + `kind=${args.kind}\0`, + `id=${args.id}\0`, + `argsHash=${bytesToHex(args.argsHash)}\0`, + ]; + return enc.encode(fields.join('')); +} + +/** + * Stable canonical-JSON serialization for hashing args. Sorts object keys + * recursively so two equivalent inputs hash to identical bytes regardless + * of property order. + */ +export function canonicalJsonStringify(value: unknown): string { + if (value === null || typeof value !== 'object') return JSON.stringify(value); + if (Array.isArray(value)) { + return `[${value.map((v) => canonicalJsonStringify(v)).join(',')}]`; + } + const obj = value as Record; + const keys = Object.keys(obj).sort(); + const parts: string[] = []; + for (const key of keys) { + if (obj[key] === undefined) continue; + parts.push(`${JSON.stringify(key)}:${canonicalJsonStringify(obj[key])}`); + } + return `{${parts.join(',')}}`; +} + +/** Hash args via canonical-JSON → SHA-256 (32 bytes). */ +export function hashArgs(args: unknown): Uint8Array { + const enc = new TextEncoder(); + return sha256(enc.encode(canonicalJsonStringify(args))); +} + +export function bytesToHex(b: Uint8Array): string { + return Array.from(b, (x) => x.toString(16).padStart(2, '0')).join(''); +} + +export function bytesToBase64(b: Uint8Array): string { + let bin = ''; + for (let i = 0; i < b.length; i++) bin += String.fromCharCode(b[i]!); + return btoa(bin); +} + +export function base64ToBytes(s: string): Uint8Array { + const bin = atob(s); + const out = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); + return out; +} diff --git a/packages/shade-files/src/protocol/correlate.ts b/packages/shade-files/src/protocol/correlate.ts new file mode 100644 index 0000000..f2d70a2 --- /dev/null +++ b/packages/shade-files/src/protocol/correlate.ts @@ -0,0 +1,32 @@ +/** + * Generate a correlation ID — 16 random bytes, base64url-encoded (22 chars + * after stripping padding). Same shape as `RequestIdSchema`. + * + * Uses `globalThis.crypto.getRandomValues` so it works in Bun, Node ≥19, + * and browsers without any platform-specific imports. + */ +export function generateRequestId(): string { + const buf = new Uint8Array(16); + globalThis.crypto.getRandomValues(buf); + return base64UrlEncode(buf); +} + +/** Same as `generateRequestId` — distinct name for callsite clarity. */ +export function generateIdempotencyKey(): string { + return generateRequestId(); +} + +export function base64UrlEncode(bytes: Uint8Array): string { + let bin = ''; + for (let i = 0; i < bytes.length; i++) bin += String.fromCharCode(bytes[i]!); + return btoa(bin).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); +} + +export function base64UrlDecode(s: string): Uint8Array { + const padded = s.replace(/-/g, '+').replace(/_/g, '/'); + const pad = padded.length % 4 === 0 ? '' : '='.repeat(4 - (padded.length % 4)); + const bin = atob(padded + pad); + const out = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); + return out; +} diff --git a/packages/shade-files/src/protocol/envelope-codec.ts b/packages/shade-files/src/protocol/envelope-codec.ts new file mode 100644 index 0000000..3618414 --- /dev/null +++ b/packages/shade-files/src/protocol/envelope-codec.ts @@ -0,0 +1,70 @@ +import { z } from 'zod'; +import { + RpcCancelSchema, + RpcErrorSchema, + RpcRequestSchema, + RpcResponseSchema, + type RpcCancel, + type RpcEnvelope, + type RpcError, + type RpcRequest, + type RpcResponse, +} from '../schemas/envelope.js'; +import { KIND_PREFIX } from './version.js'; + +/** Tagged classification of any incoming envelope. */ +export type ClassifiedEnvelope = + | { kind: 'request'; envelope: RpcRequest } + | { kind: 'response'; envelope: RpcResponse } + | { kind: 'error'; envelope: RpcError } + | { kind: 'cancel'; envelope: RpcCancel }; + +const RpcAnySchema = z.union([ + RpcRequestSchema, + RpcResponseSchema, + RpcErrorSchema, + RpcCancelSchema, +]); + +/** Encode an envelope to JSON plaintext for `Shade.send`. */ +export function encodeEnvelope(env: RpcEnvelope): string { + return JSON.stringify(env); +} + +/** + * Quick-detection: does this plaintext look like an `@shade/files` envelope? + * Used by `ShadeFileRpcChannel` to skip non-files messages cheaply. + */ +export function looksLikeFileEnvelope(plaintext: string): boolean { + return plaintext.includes(KIND_PREFIX); +} + +/** + * Parse and classify an incoming plaintext. Returns null on any malformed + * input — the channel ignores those silently (could be a different + * protocol on the same Shade.send pipe). + */ +export function tryParseEnvelope(plaintext: string): ClassifiedEnvelope | null { + let raw: unknown; + try { + raw = JSON.parse(plaintext); + } catch { + return null; + } + const result = RpcAnySchema.safeParse(raw); + if (!result.success) return null; + return classify(result.data); +} + +export function classify(env: RpcEnvelope): ClassifiedEnvelope { + if (env.kind === 'shade.fs.cancel/v1') { + return { kind: 'cancel', envelope: env as RpcCancel }; + } + if (env.kind === 'shade.fs.error/v1') { + return { kind: 'error', envelope: env as RpcError }; + } + if (env.kind.endsWith('.response')) { + return { kind: 'response', envelope: env as RpcResponse }; + } + return { kind: 'request', envelope: env as RpcRequest }; +} diff --git a/packages/shade-files/src/protocol/kinds.ts b/packages/shade-files/src/protocol/kinds.ts new file mode 100644 index 0000000..39b907a --- /dev/null +++ b/packages/shade-files/src/protocol/kinds.ts @@ -0,0 +1,53 @@ +/** Canonical kind names for v1 ops. */ +export const KIND_LIST_V1 = 'shade.fs.list/v1' as const; +export const KIND_STAT_V1 = 'shade.fs.stat/v1' as const; +export const KIND_MKDIR_V1 = 'shade.fs.mkdir/v1' as const; +export const KIND_DELETE_V1 = 'shade.fs.delete/v1' as const; +export const KIND_MOVE_V1 = 'shade.fs.move/v1' as const; +export const KIND_READ_V1 = 'shade.fs.read/v1' as const; +export const KIND_WRITE_V1 = 'shade.fs.write/v1' as const; +export const KIND_GET_THUMBNAIL_V1 = 'shade.fs.getThumbnail/v1' as const; +export const KIND_CUSTOM_V1 = 'shade.fs.custom/v1' as const; + +export const KIND_ERROR_V1 = 'shade.fs.error/v1' as const; +export const KIND_CANCEL_V1 = 'shade.fs.cancel/v1' as const; + +/** Compute the response kind from a request kind: `'shade.fs.foo/v1'` → `'shade.fs.foo/v1.response'`. */ +export function responseKindOf(kind: K): `${K}.response` { + return `${kind}.response`; +} + +/** Standard op identifier — used in `OpContext.op`. */ +export type StandardOp = + | 'list' + | 'stat' + | 'mkdir' + | 'delete' + | 'move' + | 'read' + | 'write' + | 'getThumbnail'; + +const KIND_TO_OP: Record = { + [KIND_LIST_V1]: 'list', + [KIND_STAT_V1]: 'stat', + [KIND_MKDIR_V1]: 'mkdir', + [KIND_DELETE_V1]: 'delete', + [KIND_MOVE_V1]: 'move', + [KIND_READ_V1]: 'read', + [KIND_WRITE_V1]: 'write', + [KIND_GET_THUMBNAIL_V1]: 'getThumbnail', +}; + +export function opOfKind(kind: string): StandardOp | null { + return KIND_TO_OP[kind] ?? null; +} + +/** Mutation ops require an idempotency key on retries. */ +export const MUTATION_OPS = new Set([ + 'mkdir', + 'delete', + 'move', + 'write', + 'custom', +]); diff --git a/packages/shade-files/src/protocol/version.ts b/packages/shade-files/src/protocol/version.ts new file mode 100644 index 0000000..bc83d73 --- /dev/null +++ b/packages/shade-files/src/protocol/version.ts @@ -0,0 +1,24 @@ +/** Wire-protocol version. Bump when introducing breaking changes. */ +export const SHADE_FILES_VERSION = '0.2.0'; + +/** Substring every `@shade/files` plaintext starts with — used by the channel filter. */ +export const KIND_PREFIX = 'shade.fs'; + +/** Currently-supported op kinds. Extend when introducing new versions. */ +export const SUPPORTED_KINDS = [ + 'shade.fs.list/v1', + 'shade.fs.stat/v1', + 'shade.fs.mkdir/v1', + 'shade.fs.delete/v1', + 'shade.fs.move/v1', + 'shade.fs.read/v1', + 'shade.fs.write/v1', + 'shade.fs.getThumbnail/v1', + 'shade.fs.custom/v1', +] as const; +export type SupportedKind = (typeof SUPPORTED_KINDS)[number]; + +const SUPPORTED_KIND_SET = new Set(SUPPORTED_KINDS); +export function isSupportedKind(kind: string): kind is SupportedKind { + return SUPPORTED_KIND_SET.has(kind); +} diff --git a/packages/shade-files/src/react/ShadeFilesProvider.tsx b/packages/shade-files/src/react/ShadeFilesProvider.tsx new file mode 100644 index 0000000..74b9fa4 --- /dev/null +++ b/packages/shade-files/src/react/ShadeFilesProvider.tsx @@ -0,0 +1,35 @@ +import React, { createContext, useContext, useMemo } from 'react'; +import type { Shade } from '@shade/sdk'; +import type { FilesNamespace } from '../integration/files-namespace.js'; + +export interface ShadeFilesContextValue { + shade: Shade; + files: FilesNamespace; +} + +const ShadeFilesContext = createContext(null); + +export interface ShadeFilesProviderProps { + /** Initialized `Shade` instance. `files` namespace is read off it lazily. */ + shade: Shade; + children: React.ReactNode; +} + +/** + * Provider for `@shade/files` React hooks. Distinct from + * `` in `@shade/widgets` so file-RPC consumers + * don't pull in the widget tree. + */ +export function ShadeFilesProvider({ shade, children }: ShadeFilesProviderProps): React.ReactElement { + const value = useMemo(() => ({ shade, files: shade.files }), [shade]); + return React.createElement(ShadeFilesContext.Provider, { value }, children); +} + +/** Throws if no `` is mounted above. */ +export function useShadeFilesContext(): ShadeFilesContextValue { + const ctx = useContext(ShadeFilesContext); + if (ctx === null) { + throw new Error('useShadeFilesContext: missing '); + } + return ctx; +} diff --git a/packages/shade-files/src/react/index.ts b/packages/shade-files/src/react/index.ts new file mode 100644 index 0000000..7ac1608 --- /dev/null +++ b/packages/shade-files/src/react/index.ts @@ -0,0 +1,26 @@ +/** + * React entry point for `@shade/files`. + * + * Note: hooks/components are tree-shakable. Importing from + * `@shade/files/react` (this entry) avoids pulling React into Bun-only + * server consumers that import from the root `@shade/files`. + */ + +export { ShadeFilesProvider, useShadeFilesContext } from './ShadeFilesProvider.js'; +export type { ShadeFilesContextValue, ShadeFilesProviderProps } from './ShadeFilesProvider.js'; + +export { useShadeFiles } from './useShadeFiles.js'; +export type { UseShadeFilesOptions, UseShadeFilesResult } from './useShadeFiles.js'; + +export { useFileList } from './useFileList.js'; +export type { UseFileListOptions, UseFileListResult } from './useFileList.js'; + +export { + useFileTransfer, + useFileUpload, + useFileDownload, +} from './useFileTransfer.js'; +export type { + BulkTransferStatus, + UseFileTransferResult, +} from './useFileTransfer.js'; diff --git a/packages/shade-files/src/react/useFileList.ts b/packages/shade-files/src/react/useFileList.ts new file mode 100644 index 0000000..c764e5f --- /dev/null +++ b/packages/shade-files/src/react/useFileList.ts @@ -0,0 +1,90 @@ +import { useCallback, useEffect, useRef, useState } from 'react'; +import type { FileEntry } from '../schemas/file-entry.js'; +import { useShadeFiles, type UseShadeFilesOptions } from './useShadeFiles.js'; + +export interface UseFileListResult { + entries: FileEntry[]; + isLoading: boolean; + error: unknown; + hasMore: boolean; + /** Reset and reload the first page. */ + refresh(): void; + /** Append the next page (no-op when `hasMore === false`). */ + loadMore(): Promise; +} + +export interface UseFileListOptions extends UseShadeFilesOptions { + pageSize?: number; +} + +/** + * Paginated list of `path` on `peer`. Fetches the first page on mount, + * exposes `loadMore()` for subsequent pages. + */ +export function useFileList( + peer: string, + path: string, + opts: UseFileListOptions = {}, +): UseFileListResult { + const filesOpts: UseShadeFilesOptions = {}; + if (opts.probeIntervalMs !== undefined) filesOpts.probeIntervalMs = opts.probeIntervalMs; + const { fs } = useShadeFiles(peer, filesOpts); + const pageSize = opts.pageSize ?? 100; + const [entries, setEntries] = useState([]); + const [cursor, setCursor] = useState(undefined); + const [hasMore, setHasMore] = useState(false); + const [isLoading, setIsLoading] = useState(false); + const [error, setError] = useState(null); + const [tick, setTick] = useState(0); + const inflightRef = useRef(null); + + // Initial load + reload on path/peer/refresh. + useEffect(() => { + if (fs === null) return; + inflightRef.current?.abort(); + const ctrl = new AbortController(); + inflightRef.current = ctrl; + setIsLoading(true); + setError(null); + fs.list(path, { pageSize, signal: ctrl.signal }) + .then((page) => { + if (ctrl.signal.aborted) return; + setEntries(page.entries); + setHasMore(page.hasMore); + setCursor(page.nextCursor); + }) + .catch((err) => { + if (!ctrl.signal.aborted) setError(err); + }) + .finally(() => { + if (!ctrl.signal.aborted) setIsLoading(false); + }); + return () => { + ctrl.abort(); + }; + }, [fs, path, pageSize, tick]); + + const loadMore = useCallback(async (): Promise => { + if (fs === null || !hasMore || cursor === undefined) return; + setIsLoading(true); + try { + const page = await fs.list(path, { pageSize, cursor }); + setEntries((prev) => [...prev, ...page.entries]); + setHasMore(page.hasMore); + setCursor(page.nextCursor); + } catch (err) { + setError(err); + } finally { + setIsLoading(false); + } + }, [fs, path, pageSize, cursor, hasMore]); + + return { + entries, + isLoading, + error, + hasMore, + refresh: () => setTick((t) => t + 1), + loadMore, + }; +} diff --git a/packages/shade-files/src/react/useFileTransfer.ts b/packages/shade-files/src/react/useFileTransfer.ts new file mode 100644 index 0000000..c94f7aa --- /dev/null +++ b/packages/shade-files/src/react/useFileTransfer.ts @@ -0,0 +1,103 @@ +import { useCallback, useEffect, useRef, useState } from 'react'; +import type { BulkTransferEvent, BulkTransferHandle, BulkTransferResult } from '../client/directory-types.js'; + +export interface BulkTransferStatus { + state: 'idle' | 'running' | 'done' | 'error' | 'aborted'; + filesDone: number; + filesTotal: number; + bytesDone: number; + bytesTotal: number; + error: unknown; + result: BulkTransferResult | null; + /** Last event emitted by the underlying handle. */ + lastEvent: BulkTransferEvent | null; +} + +export interface UseFileTransferResult extends BulkTransferStatus { + start(handle: BulkTransferHandle): void; + abort(reason?: string): Promise; +} + +const INITIAL: BulkTransferStatus = { + state: 'idle', + filesDone: 0, + filesTotal: 0, + bytesDone: 0, + bytesTotal: 0, + error: null, + result: null, + lastEvent: null, +}; + +/** + * Generic React-state wrapper around a `BulkTransferHandle`. Apps wire the + * handle returned from `uploadDirectory()` / `downloadDirectory()` here to + * surface progress in their UI. + * + * `useFileUpload` and `useFileDownload` are thin presets — they call + * `start(...)` with the appropriate handle automatically. + */ +export function useFileTransfer(): UseFileTransferResult { + const [status, setStatus] = useState(INITIAL); + const handleRef = useRef(null); + + const start = useCallback((handle: BulkTransferHandle): void => { + handleRef.current = handle; + setStatus({ ...INITIAL, state: 'running' }); + void (async () => { + try { + for await (const ev of handle.events) { + setStatus((prev) => { + const next: BulkTransferStatus = { ...prev, lastEvent: ev }; + if (ev.type === 'plan') { + next.filesTotal = ev.totalFiles; + next.bytesTotal = ev.totalBytes ?? 0; + } else if (ev.type === 'progress') { + next.filesDone = ev.filesDone; + next.bytesDone = ev.bytesDone; + next.bytesTotal = ev.bytesTotal ?? prev.bytesTotal; + next.filesTotal = ev.filesTotal; + } else if (ev.type === 'complete') { + next.state = 'done'; + next.filesDone = ev.filesDone; + next.bytesDone = ev.bytesDone; + } else if (ev.type === 'abort') { + next.state = 'aborted'; + next.error = new Error(ev.reason); + } else if (ev.type === 'file-error') { + next.error = ev.error; + } + return next; + }); + } + const result = await handle.done(); + setStatus((prev) => ({ ...prev, state: 'done', result })); + } catch (err) { + setStatus((prev) => ({ ...prev, state: 'error', error: err })); + } + })(); + }, []); + + const abort = useCallback(async (reason?: string): Promise => { + if (handleRef.current === null) return; + await handleRef.current.abort(reason); + }, []); + + // Cleanup on unmount: abort any in-flight transfer. + useEffect(() => { + return () => { + if (handleRef.current !== null) { + void handleRef.current.abort('unmount').catch(() => undefined); + } + }; + }, []); + + return { ...status, start, abort }; +} + +/** + * Preset: `useFileUpload` — semantically identical to `useFileTransfer`, + * named distinctly to mirror the `useFileDownload` pair. + */ +export const useFileUpload = useFileTransfer; +export const useFileDownload = useFileTransfer; diff --git a/packages/shade-files/src/react/useShadeFiles.ts b/packages/shade-files/src/react/useShadeFiles.ts new file mode 100644 index 0000000..24627aa --- /dev/null +++ b/packages/shade-files/src/react/useShadeFiles.ts @@ -0,0 +1,88 @@ +import { useEffect, useRef, useState } from 'react'; +import type { FileClient } from '../client/client.js'; +import { useShadeFilesContext } from './ShadeFilesProvider.js'; + +export interface UseShadeFilesResult { + fs: FileClient | null; + isLoading: boolean; + error: unknown; + /** True after a successful `stat('/')` probe; false until proven online. */ + isOnline: boolean; + /** Force a re-probe. */ + refresh(): void; +} + +export interface UseShadeFilesOptions { + /** Probe interval (ms) for `isOnline`. Default 30_000. Disable with 0. */ + probeIntervalMs?: number; +} + +/** + * Get a typed `FileClient` for `peer`. Memoizes per (shade, peer) so + * multiple components sharing the same peer reuse the same client. + */ +export function useShadeFiles( + peer: string, + opts: UseShadeFilesOptions = {}, +): UseShadeFilesResult { + const { files } = useShadeFilesContext(); + const [fs, setFs] = useState(null); + const [error, setError] = useState(null); + const [isOnline, setIsOnline] = useState(false); + const [probeTick, setProbeTick] = useState(0); + const cancelledRef = useRef(false); + + useEffect(() => { + cancelledRef.current = false; + let mounted = true; + setError(null); + setFs(null); + files + .client(peer) + .then((client) => { + if (!mounted) { + client.close(); + return; + } + setFs(client); + }) + .catch((err) => { + if (mounted) setError(err); + }); + return () => { + mounted = false; + cancelledRef.current = true; + }; + }, [files, peer]); + + // Periodic probe: stat('/') as a cheap reachability check. + useEffect(() => { + if (fs === null) return undefined; + const intervalMs = opts.probeIntervalMs ?? 30_000; + let cancelled = false; + const probe = (): void => { + fs.stat('/') + .then(() => { + if (!cancelled) setIsOnline(true); + }) + .catch(() => { + if (!cancelled) setIsOnline(false); + }); + }; + probe(); + if (intervalMs <= 0) return () => undefined; + const timer = setInterval(probe, intervalMs); + return () => { + cancelled = true; + clearInterval(timer); + }; + }, [fs, opts.probeIntervalMs, probeTick]); + + return { + fs, + isLoading: fs === null && error === null, + error, + isOnline, + refresh: () => setProbeTick((t) => t + 1), + }; +} diff --git a/packages/shade-files/src/rpc/channel.ts b/packages/shade-files/src/rpc/channel.ts new file mode 100644 index 0000000..3c452e8 --- /dev/null +++ b/packages/shade-files/src/rpc/channel.ts @@ -0,0 +1,107 @@ +import type { Shade } from '@shade/sdk'; +import { + encodeEnvelope, + looksLikeFileEnvelope, + tryParseEnvelope, + type ClassifiedEnvelope, +} from '../protocol/envelope-codec.js'; +import type { RpcEnvelope } from '../schemas/envelope.js'; + +/** + * Routing hooks called by the channel when it receives different envelope + * classes. M-Files-1 ships the skeleton; M-Files-2 wires up real handlers + * (server-side dispatcher, client-side response routing). + */ +export interface RpcChannelHooks { + onRequest?: (from: string, env: ClassifiedEnvelope & { kind: 'request' }) => Promise; + onResponse?: (from: string, env: ClassifiedEnvelope & { kind: 'response' }) => void; + onError?: (from: string, env: ClassifiedEnvelope & { kind: 'error' }) => void; + onCancel?: (from: string, env: ClassifiedEnvelope & { kind: 'cancel' }) => void; +} + +/** + * `ShadeFileRpcChannel` rides on top of `Shade.send`/`Shade.onMessage` + * and routes `shade.fs.*` JSON envelopes to registered hooks. + * + * Counterpart to `ShadeControlChannel` from `@shade/sdk` — but with + * request-response semantics instead of fire-and-forget. + * + * One channel per `Shade` instance; the SDK's `files` getter memoizes it. + * Both server and client roles share the same channel: the same Shade can + * both serve files and consume them from peers. + */ +export class ShadeFileRpcChannel { + private hooks: RpcChannelHooks = {}; + private readonly unsubscribe: () => void; + private destroyed = false; + + constructor(private readonly shade: Shade) { + this.unsubscribe = shade.onMessage(async (from, plaintext) => { + if (!looksLikeFileEnvelope(plaintext)) return; + const classified = tryParseEnvelope(plaintext); + if (classified === null) return; + await this.dispatch(from, classified); + }); + } + + /** + * Merge hooks into the current set. Returns the previous values for the + * keys being set so callers can restore them on cleanup. Other hook + * slots are left unchanged — the same channel can simultaneously serve + * (`onRequest`+`onCancel`) and consume (`onResponse`+`onError`). + */ + setHooks(hooks: RpcChannelHooks): RpcChannelHooks { + if (this.destroyed) throw new Error('ShadeFileRpcChannel: destroyed'); + const prev: RpcChannelHooks = {}; + const keys = ['onRequest', 'onResponse', 'onError', 'onCancel'] as const; + for (const key of keys) { + if (key in hooks) { + prev[key] = this.hooks[key] as never; + this.hooks[key] = hooks[key] as never; + } + } + return prev; + } + + /** + * Send an envelope to a peer. Encrypts via `Shade.send` (Double Ratchet) + * then delivers the ratchet envelope to the peer's `transferRoute()` + * control endpoint via the SDK's configured envelope transport. + */ + async send(peerAddress: string, envelope: RpcEnvelope): Promise { + if (this.destroyed) throw new Error('ShadeFileRpcChannel: destroyed'); + const plaintext = encodeEnvelope(envelope); + const ratchetEnvelope = await this.shade.send(peerAddress, plaintext); + await this.shade.deliverControlEnvelope(peerAddress, ratchetEnvelope); + } + + destroy(): void { + if (this.destroyed) return; + this.destroyed = true; + this.unsubscribe(); + this.hooks = {}; + } + + private async dispatch(from: string, classified: ClassifiedEnvelope): Promise { + try { + switch (classified.kind) { + case 'request': + if (this.hooks.onRequest !== undefined) { + await this.hooks.onRequest(from, classified); + } + return; + case 'response': + this.hooks.onResponse?.(from, classified); + return; + case 'error': + this.hooks.onError?.(from, classified); + return; + case 'cancel': + this.hooks.onCancel?.(from, classified); + return; + } + } catch (err) { + console.error('[ShadeFileRpcChannel] dispatch error:', err); + } + } +} diff --git a/packages/shade-files/src/rpc/pending.ts b/packages/shade-files/src/rpc/pending.ts new file mode 100644 index 0000000..77c0032 --- /dev/null +++ b/packages/shade-files/src/rpc/pending.ts @@ -0,0 +1,113 @@ +import { CancelledError, OperationTimeoutError, fileErrorFromPayload } from '../schemas/errors.js'; +import type { RpcError, RpcResponse } from '../schemas/envelope.js'; + +interface PendingEntry { + resolve: (result: unknown) => void; + reject: (err: unknown) => void; + timer: ReturnType | null; + abortListener: (() => void) | null; + signal?: AbortSignal | undefined; +} + +export interface RegisterOptions { + timeoutMs?: number; + signal?: AbortSignal; + /** Called when client cancels (signal aborted or timeout); transport can ship an `RpcCancel`. */ + onCancel?: (reason: 'timeout' | 'signal') => void; +} + +/** + * Client-side tracker for in-flight RPC calls. Routes incoming + * `RpcResponse`/`RpcError` to the right pending promise via correlation id. + */ +export class PendingRpcRegistry { + private readonly entries = new Map(); + + /** Register a pending request. Returns a Promise resolved/rejected by routing. */ + register(requestId: string, opts: RegisterOptions = {}): Promise { + const p = new Promise((resolve, reject) => { + const cleanup = (): void => { + const e = this.entries.get(requestId); + if (e === undefined) return; + if (e.timer !== null) clearTimeout(e.timer); + if (e.abortListener !== null && e.signal !== undefined) { + e.signal.removeEventListener('abort', e.abortListener); + } + this.entries.delete(requestId); + }; + + const entry: PendingEntry = { + resolve: (v) => { + cleanup(); + resolve(v as T); + }, + reject: (err) => { + cleanup(); + reject(err); + }, + timer: null, + abortListener: null, + ...(opts.signal !== undefined ? { signal: opts.signal } : {}), + }; + + if (opts.timeoutMs !== undefined && opts.timeoutMs > 0) { + entry.timer = setTimeout(() => { + opts.onCancel?.('timeout'); + entry.reject(new OperationTimeoutError()); + }, opts.timeoutMs); + } + + if (opts.signal !== undefined) { + if (opts.signal.aborted) { + // Defer microtask so caller's await sees the rejection. + queueMicrotask(() => { + opts.onCancel?.('signal'); + entry.reject(new CancelledError()); + }); + } else { + const listener = (): void => { + opts.onCancel?.('signal'); + entry.reject(new CancelledError()); + }; + entry.abortListener = listener; + opts.signal.addEventListener('abort', listener, { once: true }); + } + } + + this.entries.set(requestId, entry); + }); + // Attach a no-op rejection handler so Bun's strict unhandled-rejection + // detection doesn't flag the registered promise BEFORE the caller's + // `await` attaches its handler. The caller's await still observes the + // rejection through its own handler chain. + p.catch(() => { + /* suppress unhandled-rejection warning */ + }); + return p; + } + + /** Route an incoming response. */ + resolveResponse(env: RpcResponse): void { + const entry = this.entries.get(env.id); + if (entry === undefined) return; // unknown id — possibly a stale duplicate + entry.resolve(env.result); + } + + /** Route an incoming error envelope. */ + resolveError(env: RpcError): void { + const entry = this.entries.get(env.id); + if (entry === undefined) return; + entry.reject(fileErrorFromPayload(env.error)); + } + + /** Reject all pending entries — for shutdown. */ + rejectAll(reason: unknown): void { + for (const entry of this.entries.values()) entry.reject(reason); + this.entries.clear(); + } + + /** For tests / metrics. */ + size(): number { + return this.entries.size; + } +} diff --git a/packages/shade-files/src/schemas/envelope.ts b/packages/shade-files/src/schemas/envelope.ts new file mode 100644 index 0000000..befeb53 --- /dev/null +++ b/packages/shade-files/src/schemas/envelope.ts @@ -0,0 +1,46 @@ +import { z } from 'zod'; +import { FileErrorPayloadSchema } from './errors.js'; +import { IdempotencyKeySchema, RequestIdSchema } from './primitives.js'; + +const KIND_REQUEST_PATTERN = /^shade\.fs\.[A-Za-z0-9.-]+\/v\d+$/; +const KIND_RESPONSE_PATTERN = /^shade\.fs\.[A-Za-z0-9.-]+\/v\d+\.response$/; + +/** + * Request envelope. Carries op kind, correlation id, args (validated per + * op), idempotency key (mutations only), and an Ed25519 signature binding + * `(sender, kind, id, argsHash, signedAt)`. + */ +export const RpcRequestSchema = z.object({ + kind: z.string().regex(KIND_REQUEST_PATTERN, 'invalid request kind'), + id: RequestIdSchema, + args: z.unknown(), + idempotencyKey: IdempotencyKeySchema.optional(), + attempt: z.number().int().positive().optional(), + deadlineMs: z.number().int().positive().optional(), + sig: z.string().min(1).max(128), + signedAt: z.number().int().positive(), +}); +export type RpcRequest = z.infer; + +export const RpcResponseSchema = z.object({ + kind: z.string().regex(KIND_RESPONSE_PATTERN, 'invalid response kind'), + id: RequestIdSchema, + result: z.unknown(), +}); +export type RpcResponse = z.infer; + +export const RpcErrorSchema = z.object({ + kind: z.literal('shade.fs.error/v1'), + id: RequestIdSchema, + error: FileErrorPayloadSchema, +}); +export type RpcError = z.infer; + +export const RpcCancelSchema = z.object({ + kind: z.literal('shade.fs.cancel/v1'), + id: RequestIdSchema, + reason: z.string().max(512).optional(), +}); +export type RpcCancel = z.infer; + +export type RpcEnvelope = RpcRequest | RpcResponse | RpcError | RpcCancel; diff --git a/packages/shade-files/src/schemas/errors.ts b/packages/shade-files/src/schemas/errors.ts new file mode 100644 index 0000000..6a49d18 --- /dev/null +++ b/packages/shade-files/src/schemas/errors.ts @@ -0,0 +1,228 @@ +import { z } from 'zod'; +import { ShadeError } from '@shade/core'; + +export const FileErrorCodeSchema = z.enum([ + 'NOT_FOUND', + 'PERMISSION_DENIED', + 'CONFLICT', + 'QUOTA_EXCEEDED', + 'RATE_LIMIT', + 'PATH_VALIDATION', + 'FINGERPRINT_REQUIRED', + 'OPERATION_TIMEOUT', + 'IDEMPOTENCY_CONFLICT', + 'CANCELLED', + 'INTERNAL', + 'NOT_IMPLEMENTED', + 'CUSTOM_OP_REJECTED', + 'INVALID_SIGNATURE', + 'INVALID_ARGS', +]); +export type FileErrorCode = z.infer; + +export const FileErrorPayloadSchema = z.object({ + code: FileErrorCodeSchema, + message: z.string().max(2048), + /** Suggested retry delay in ms; only set for retriable errors (rate limit, transient). */ + retryAfterMs: z.number().int().nonnegative().optional(), + /** Path or arg field the error refers to. */ + field: z.string().max(128).optional(), + /** Optional cause string (sanitized — no stack traces leak from server). */ + cause: z.string().max(2048).optional(), +}); +export type FileErrorPayload = z.infer; + +// ─── Class hierarchy ───────────────────────────────────────── + +/** + * Base class for all `@shade/files` errors. Extends `ShadeError` so the + * existing `errorToHttpStatus()` mapping in `@shade/core` continues to apply. + */ +export class FileError extends ShadeError { + readonly payload: FileErrorPayload; + + constructor(payload: FileErrorPayload) { + super(`SHADE_FS_${payload.code}`, payload.message); + this.name = 'FileError'; + this.payload = payload; + } + + override toJSON(): { name: string; code: string; message: string; payload: FileErrorPayload } { + return { + name: this.name, + code: this.code, + message: this.message, + payload: this.payload, + }; + } +} + +export class NotFoundError extends FileError { + constructor(message = 'Not found', field?: string) { + super({ code: 'NOT_FOUND', message, ...(field !== undefined ? { field } : {}) }); + this.name = 'NotFoundError'; + } +} + +export class PermissionDeniedError extends FileError { + constructor(message = 'Permission denied') { + super({ code: 'PERMISSION_DENIED', message }); + this.name = 'PermissionDeniedError'; + } +} + +export class ConflictError extends FileError { + constructor(message: string) { + super({ code: 'CONFLICT', message }); + this.name = 'ConflictError'; + } +} + +export class QuotaExceededError extends FileError { + constructor(message = 'Quota exceeded', retryAfterMs?: number) { + super({ + code: 'QUOTA_EXCEEDED', + message, + ...(retryAfterMs !== undefined ? { retryAfterMs } : {}), + }); + this.name = 'QuotaExceededError'; + } +} + +export class FsRateLimitError extends FileError { + constructor(message = 'Rate limit exceeded', retryAfterMs?: number) { + super({ + code: 'RATE_LIMIT', + message, + ...(retryAfterMs !== undefined ? { retryAfterMs } : {}), + }); + this.name = 'FsRateLimitError'; + } +} + +export class PathValidationError extends FileError { + constructor(message: string, field = 'path') { + super({ code: 'PATH_VALIDATION', message, field }); + this.name = 'PathValidationError'; + } +} + +export class FingerprintRequiredError extends FileError { + constructor(message = 'Peer fingerprint must be verified before this operation') { + super({ code: 'FINGERPRINT_REQUIRED', message }); + this.name = 'FingerprintRequiredError'; + } +} + +export class OperationTimeoutError extends FileError { + constructor(message = 'Operation timed out') { + super({ code: 'OPERATION_TIMEOUT', message }); + this.name = 'OperationTimeoutError'; + } +} + +export class IdempotencyConflictError extends FileError { + constructor( + message = 'Idempotency key reused with different arguments', + ) { + super({ code: 'IDEMPOTENCY_CONFLICT', message }); + this.name = 'IdempotencyConflictError'; + } +} + +export class CancelledError extends FileError { + constructor(message = 'Cancelled') { + super({ code: 'CANCELLED', message }); + this.name = 'CancelledError'; + } +} + +export class InternalFileError extends FileError { + constructor(message = 'Internal server error', cause?: string) { + super({ + code: 'INTERNAL', + message, + ...(cause !== undefined ? { cause } : {}), + }); + this.name = 'InternalFileError'; + } +} + +export class NotImplementedError extends FileError { + constructor(op: string) { + super({ code: 'NOT_IMPLEMENTED', message: `Operation not implemented: ${op}` }); + this.name = 'NotImplementedError'; + } +} + +export class CustomOpRejectedError extends FileError { + constructor(message: string) { + super({ code: 'CUSTOM_OP_REJECTED', message }); + this.name = 'CustomOpRejectedError'; + } +} + +export class InvalidSignatureError extends FileError { + constructor(message = 'RPC envelope signature verification failed') { + super({ code: 'INVALID_SIGNATURE', message }); + this.name = 'InvalidSignatureError'; + } +} + +export class InvalidArgsError extends FileError { + constructor(message: string, field?: string) { + super({ code: 'INVALID_ARGS', message, ...(field !== undefined ? { field } : {}) }); + this.name = 'InvalidArgsError'; + } +} + +/** + * Reconstruct the right `FileError` subclass from a wire payload. Used by + * the client to surface typed errors from server responses. + */ +export function fileErrorFromPayload(payload: FileErrorPayload): FileError { + switch (payload.code) { + case 'NOT_FOUND': + return new NotFoundError(payload.message, payload.field); + case 'PERMISSION_DENIED': + return new PermissionDeniedError(payload.message); + case 'CONFLICT': + return new ConflictError(payload.message); + case 'QUOTA_EXCEEDED': + return new QuotaExceededError(payload.message, payload.retryAfterMs); + case 'RATE_LIMIT': + return new FsRateLimitError(payload.message, payload.retryAfterMs); + case 'PATH_VALIDATION': + return new PathValidationError(payload.message, payload.field); + case 'FINGERPRINT_REQUIRED': + return new FingerprintRequiredError(payload.message); + case 'OPERATION_TIMEOUT': + return new OperationTimeoutError(payload.message); + case 'IDEMPOTENCY_CONFLICT': + return new IdempotencyConflictError(payload.message); + case 'CANCELLED': + return new CancelledError(payload.message); + case 'INTERNAL': + return new InternalFileError(payload.message, payload.cause); + case 'NOT_IMPLEMENTED': + return new NotImplementedError(payload.message); + case 'CUSTOM_OP_REJECTED': + return new CustomOpRejectedError(payload.message); + case 'INVALID_SIGNATURE': + return new InvalidSignatureError(payload.message); + case 'INVALID_ARGS': + return new InvalidArgsError(payload.message, payload.field); + } +} + +/** Convert any thrown value into a `FileErrorPayload` for wire serialization. */ +export function payloadFromError(err: unknown): FileErrorPayload { + if (err instanceof FileError) return err.payload; + if (err instanceof Error) { + return { + code: 'INTERNAL', + message: err.message.slice(0, 2048), + }; + } + return { code: 'INTERNAL', message: 'unknown error' }; +} diff --git a/packages/shade-files/src/schemas/file-entry.ts b/packages/shade-files/src/schemas/file-entry.ts new file mode 100644 index 0000000..acd4dd3 --- /dev/null +++ b/packages/shade-files/src/schemas/file-entry.ts @@ -0,0 +1,33 @@ +import { z } from 'zod'; +import { CursorSchema } from './primitives.js'; + +export const FileKindSchema = z.enum(['file', 'dir']); +export type FileKind = z.infer; + +/** + * Directory entry shape. App-specific extension via `metadata` — keep it + * lean; large blobs (thumbnails) belong in `getThumbnail`, not here. + */ +export const FileEntrySchema = z.object({ + /** Base name only — no path separators. */ + name: z.string().min(1).max(512).refine((s) => !s.includes('/') && !s.includes('\\'), { + message: 'name must not contain path separators', + }), + kind: FileKindSchema, + /** Plaintext byte size; 0 for directories. */ + size: z.number().int().nonnegative(), + /** Last-modified time, unix milliseconds. */ + mtime: z.number().int(), + /** RFC 6838 media type if known. */ + contentType: z.string().max(255).optional(), + /** App-specific extension fields. Default empty. */ + metadata: z.record(z.string(), z.unknown()).default({}), +}); +export type FileEntry = z.infer; + +export const ListPageSchema = z.object({ + entries: z.array(FileEntrySchema), + nextCursor: CursorSchema.optional(), + hasMore: z.boolean(), +}); +export type ListPage = z.infer; diff --git a/packages/shade-files/src/schemas/index.ts b/packages/shade-files/src/schemas/index.ts new file mode 100644 index 0000000..fd289d7 --- /dev/null +++ b/packages/shade-files/src/schemas/index.ts @@ -0,0 +1,5 @@ +export * from './primitives.js'; +export * from './file-entry.js'; +export * from './ops.js'; +export * from './envelope.js'; +export * from './errors.js'; diff --git a/packages/shade-files/src/schemas/ops.ts b/packages/shade-files/src/schemas/ops.ts new file mode 100644 index 0000000..42c59c6 --- /dev/null +++ b/packages/shade-files/src/schemas/ops.ts @@ -0,0 +1,157 @@ +import { z } from 'zod'; +import { Base64Schema, CursorSchema, PathSchema, Sha256HexSchema } from './primitives.js'; +import { FileEntrySchema, FileKindSchema, ListPageSchema } from './file-entry.js'; + +// ─── list ──────────────────────────────────────────────────── + +export const ListArgsSchema = z.object({ + path: PathSchema, + cursor: CursorSchema.optional(), + pageSize: z.number().int().min(1).max(1000).default(100), + filter: z + .object({ + prefix: z.string().max(256).optional(), + kind: FileKindSchema.optional(), + }) + .optional(), +}); +export type ListArgs = z.infer; +export const ListResultSchema = ListPageSchema; +export type ListResult = z.infer; + +// ─── stat ──────────────────────────────────────────────────── + +export const StatArgsSchema = z.object({ path: PathSchema }); +export type StatArgs = z.infer; +export const StatResultSchema = FileEntrySchema; +export type StatResult = z.infer; + +// ─── mkdir ─────────────────────────────────────────────────── + +export const MkdirArgsSchema = z.object({ + path: PathSchema, + recursive: z.boolean().default(false), +}); +export type MkdirArgs = z.infer; +export const MkdirResultSchema = z.object({ entry: FileEntrySchema }); +export type MkdirResult = z.infer; + +// ─── delete ────────────────────────────────────────────────── + +export const DeleteArgsSchema = z.object({ + path: PathSchema, + recursive: z.boolean().default(false), +}); +export type DeleteArgs = z.infer; +export const DeleteResultSchema = z.object({ + deletedCount: z.number().int().nonnegative(), +}); +export type DeleteResult = z.infer; + +// ─── move ──────────────────────────────────────────────────── + +export const MoveArgsSchema = z.object({ + src: PathSchema, + dst: PathSchema, + overwrite: z.boolean().default(false), +}); +export type MoveArgs = z.infer; +export const MoveResultSchema = z.object({ entry: FileEntrySchema }); +export type MoveResult = z.infer; + +// ─── read ──────────────────────────────────────────────────── + +export const ReadArgsSchema = z.object({ + path: PathSchema, + range: z + .object({ + start: z.number().int().nonnegative(), + end: z.number().int().positive(), + }) + .optional(), + /** Force inline path even if size unknown / large. Use with care. */ + preferInline: z.boolean().optional(), +}); +export type ReadArgs = z.infer; + +export const ReadResultSchema = z.discriminatedUnion('kind', [ + z.object({ + kind: z.literal('inline'), + bytesB64: Base64Schema, + size: z.number().int().nonnegative(), + sha256: Sha256HexSchema, + contentType: z.string().optional(), + }), + z.object({ + kind: z.literal('streams'), + streamId: z.string(), + size: z.number().int().nonnegative(), + sha256: Sha256HexSchema, + contentType: z.string().optional(), + }), +]); +export type ReadResult = z.infer; + +// ─── write ─────────────────────────────────────────────────── + +export const WriteArgsSchema = z.discriminatedUnion('kind', [ + z.object({ + kind: z.literal('inline'), + path: PathSchema, + bytesB64: Base64Schema, + contentType: z.string().optional(), + overwrite: z.boolean().default(false), + }), + z.object({ + kind: z.literal('streams'), + path: PathSchema, + /** Declared plaintext size; server clamps quota against this. */ + size: z.number().int().nonnegative(), + contentType: z.string().optional(), + overwrite: z.boolean().default(false), + /** base64url(16) — matches `metadata.shadeFilesWriteId` on the streams transfer. */ + writeId: z.string(), + }), +]); +export type WriteArgs = z.infer; +export const WriteResultSchema = z.object({ entry: FileEntrySchema }); +export type WriteResult = z.infer; + +// ─── getThumbnail ──────────────────────────────────────────── + +export const ThumbnailSizeSchema = z.union([ + z.literal(64), + z.literal(128), + z.literal(256), + z.literal(512), +]); +export type ThumbnailSize = z.infer; + +export const GetThumbnailArgsSchema = z.object({ + path: PathSchema, + size: ThumbnailSizeSchema, + format: z.enum(['png', 'webp', 'jpeg']).default('png'), +}); +export type GetThumbnailArgs = z.infer; + +export const GetThumbnailResultSchema = z.object({ + bytesB64: Base64Schema, + format: z.enum(['png', 'webp', 'jpeg']), + width: z.number().int().positive().max(4096), + height: z.number().int().positive().max(4096), + sha256: Sha256HexSchema, +}); +export type GetThumbnailResult = z.infer; + +// ─── custom ────────────────────────────────────────────────── + +export const CustomArgsSchema = z.object({ + name: z.string().min(1).max(128).regex(/^[a-z0-9.-]+$/i, 'invalid custom op name'), + payload: z.unknown(), +}); +export type CustomArgs = z.infer; + +export const CustomResultSchema = z.object({ + result: z.unknown(), +}); +export type CustomResult = z.infer; diff --git a/packages/shade-files/src/schemas/primitives.ts b/packages/shade-files/src/schemas/primitives.ts new file mode 100644 index 0000000..5292e70 --- /dev/null +++ b/packages/shade-files/src/schemas/primitives.ts @@ -0,0 +1,42 @@ +import { z } from 'zod'; + +/** + * POSIX-style absolute path. Must start with `/`, max 4096 chars, no NULs, + * no CR/LF/DEL, no backslashes (Windows-style rejected). Further policy + * (root-scope, traversal hardening) applied by `validatePath` server-side. + */ +export const PathSchema = z + .string() + .min(1) + .max(4096) + .regex(/^\//, 'must be absolute (start with /)') + .refine((s) => !/[\x00-\x08\x0a-\x1f\x7f\\]/.test(s), { + message: 'contains forbidden control or backslash characters', + }); + +export type Path = z.infer; + +/** Base64url(16 bytes) — used for both requestId and idempotencyKey. */ +export const RequestIdSchema = z + .string() + .min(22) + .max(22) + .regex(/^[A-Za-z0-9_-]+$/, 'invalid base64url'); + +export type RequestId = z.infer; + +export const IdempotencyKeySchema = RequestIdSchema; +export type IdempotencyKey = z.infer; + +/** + * Opaque cursor token. Server-defined shape; clients pass through verbatim. + * Capped at 2 KiB to prevent over-sized cursors leaking server state. + */ +export const CursorSchema = z.string().min(1).max(2048); +export type Cursor = z.infer; + +/** Hex-encoded SHA-256 (64 chars). */ +export const Sha256HexSchema = z.string().length(64).regex(/^[0-9a-f]+$/); + +/** Standard base64 (with padding) used for binary blobs in JSON envelopes. */ +export const Base64Schema = z.string().regex(/^[A-Za-z0-9+/]*=*$/); diff --git a/packages/shade-files/src/server/cursor.ts b/packages/shade-files/src/server/cursor.ts new file mode 100644 index 0000000..ec3860e --- /dev/null +++ b/packages/shade-files/src/server/cursor.ts @@ -0,0 +1,73 @@ +import { sha256 } from '@noble/hashes/sha2.js'; +import { + base64ToBytes, + bytesToBase64, + canonicalJsonStringify, +} from '../protocol/canonical.js'; + +/** + * Opaque pagination cursor. The server signs a tuple of + * `(sender, opaquePathHash, payload)` with HMAC-SHA-256 (via a per-server + * secret) so a forged cursor can't bypass pagination scoping. + * + * The payload itself is server-defined; in tests we use simple `{ offset }`. + */ + +export interface CursorBuilder { + encode(sender: string, pathHash: string, payload: unknown): string; + decode(sender: string, pathHash: string, cursor: string): unknown | null; +} + +export function createCursorBuilder(serverSecret: Uint8Array): CursorBuilder { + if (serverSecret.length < 16) { + throw new Error('serverSecret must be at least 16 bytes'); + } + + function mac(input: Uint8Array): Uint8Array { + // Simple HMAC-SHA-256 implementation using @noble/hashes — keyed + // construction over secret || input. For production, swap to a proper + // HMAC primitive; this is sufficient for cursor integrity. + const padded = new Uint8Array(serverSecret.length + input.length); + padded.set(serverSecret, 0); + padded.set(input, serverSecret.length); + return sha256(padded); + } + + return { + encode(sender, pathHash, payload) { + const json = canonicalJsonStringify({ s: sender, p: pathHash, d: payload }); + const enc = new TextEncoder().encode(json); + const tag = mac(enc).slice(0, 16); // 128-bit truncation is plenty + const out = new Uint8Array(enc.length + tag.length); + out.set(enc, 0); + out.set(tag, enc.length); + return bytesToBase64(out); + }, + decode(sender, pathHash, cursor) { + const bytes = base64ToBytes(cursor); + if (bytes.length < 17) return null; + const enc = bytes.slice(0, bytes.length - 16); + const tag = bytes.slice(bytes.length - 16); + const expected = mac(enc).slice(0, 16); + if (!constantTimeEqualBytes(tag, expected)) return null; + try { + const parsed = JSON.parse(new TextDecoder().decode(enc)) as { + s: string; + p: string; + d: unknown; + }; + if (parsed.s !== sender || parsed.p !== pathHash) return null; + return parsed.d; + } catch { + return null; + } + }, + }; +} + +function constantTimeEqualBytes(a: Uint8Array, b: Uint8Array): boolean { + if (a.length !== b.length) return false; + let diff = 0; + for (let i = 0; i < a.length; i++) diff |= a[i]! ^ b[i]!; + return diff === 0; +} diff --git a/packages/shade-files/src/server/custom-ops.ts b/packages/shade-files/src/server/custom-ops.ts new file mode 100644 index 0000000..624ddd7 --- /dev/null +++ b/packages/shade-files/src/server/custom-ops.ts @@ -0,0 +1,86 @@ +/** + * Custom op registry — typed via TypeScript declaration merging. + * + * Consumers register a custom op by: + * 1. Adding to `CustomOpsMap` via declaration merging (compile-time typing). + * 2. Passing a Zod-backed handler in `FileHandlerConfig.custom`. + * + * Server validates wire args against the registered Zod schema before + * invoking the user handler, then validates the user's response against + * the schema before shipping it. + * + * @example + * ```ts + * // In your app, once globally: + * declare module '@shade/files' { + * interface CustomOpsMap { + * 'app.deploy-mod': CustomOpDef<{ jarPath: string }, { deploymentId: string }>; + * } + * } + * + * // Server registers a handler: + * shade.files.serve({ + * custom: { + * 'app.deploy-mod': { + * args: z.object({ jarPath: z.string() }), + * response: z.object({ deploymentId: z.string() }), + * handler: async (args, ctx) => ({ deploymentId: '...' }), + * }, + * }, + * }); + * + * // Client gets typed I/O: + * const result = await fs.custom('app.deploy-mod', { jarPath: '...' }); + * // ^? { deploymentId: string } + * ``` + */ +import type { ZodType } from 'zod'; +import type { OpContext } from './handler-context.js'; + +// ─── Public typing scaffold (declaration-merged by consumers) ─ + +/** Marker shape for one custom op entry — consumers extend `CustomOpsMap`. */ +export interface CustomOpDef { + args: A; + response: R; +} + +/** + * Registry of all custom ops known at compile time. Empty by default — + * consumers extend via TypeScript declaration merging: + * + * ```ts + * declare module '@shade/files' { + * interface CustomOpsMap { + * 'app.foo': CustomOpDef<{ x: number }, { y: string }>; + * } + * } + * ``` + */ +// eslint-disable-next-line @typescript-eslint/no-empty-object-type +export interface CustomOpsMap {} + +// ─── Server-side handler config ────────────────────────────── + +/** + * Server-side definition for one custom op: the args schema (validated + * inbound), the response schema (validated outbound), and the handler. + */ +export interface CustomOpRegistration { + args: ZodType; + response: ZodType; + handler: (args: TArgs, ctx: OpContext<{ name: string; payload: TArgs }>) => Promise; + /** + * Optional cost override for the rate limiter. Default: same as + * `opCost.custom` (1 unless overridden in `RateLimitConfig`). + */ + cost?: number; +} + +/** + * Map from custom-op name to registration. The keys are the same names + * declared in `CustomOpsMap`. Required typing via declaration merging is + * not strict here — runtime validation is enforced by the registered Zod + * schemas, so consumers using untyped names still get safety. + */ +export type CustomOpRegistrations = Record>; diff --git a/packages/shade-files/src/server/handler-context.ts b/packages/shade-files/src/server/handler-context.ts new file mode 100644 index 0000000..a9cfeab --- /dev/null +++ b/packages/shade-files/src/server/handler-context.ts @@ -0,0 +1,59 @@ +import type { Shade } from '@shade/sdk'; +import type { StandardOp } from '../protocol/kinds.js'; + +export type OpKind = StandardOp | `custom:${string}`; + +/** + * Per-call context handed to every server-side handler/middleware. + * + * Path-related ops set `path` to the (validated, normalized) primary path. + * For `move`, `path` is `args.src`. For ops without a path (`custom`), the + * field is the empty string. + */ +export interface OpContext { + /** The op being executed. */ + op: OpKind; + /** Validated + normalized primary path; empty string for path-less ops. */ + path: string; + /** Zod-validated args for this op. */ + args: TArgs; + /** Peer address (from Shade.receive). */ + sender: string; + /** Aborted on client cancel or timeout. */ + signal: AbortSignal; + /** Idempotency key (mutations only). */ + idempotencyKey: string | undefined; + /** Attempt counter (1-based). */ + attemptNumber: number; + /** Wall-clock when dispatch started (for metrics). */ + startedAt: number; + /** Lazy fingerprint accessor — call only if needed. */ + fingerprint: () => Promise; + /** Mutable scratchpad for middleware (e.g. cache fingerprint hit). */ + metadata: Map; +} + +/** Build an `OpContext`. Internal — used by the dispatcher. */ +export function buildOpContext(args: { + op: OpKind; + path: string; + parsedArgs: TArgs; + sender: string; + signal: AbortSignal; + idempotencyKey: string | undefined; + attemptNumber: number; + shade: Shade; +}): OpContext { + return { + op: args.op, + path: args.path, + args: args.parsedArgs, + sender: args.sender, + signal: args.signal, + idempotencyKey: args.idempotencyKey, + attemptNumber: args.attemptNumber, + startedAt: Date.now(), + fingerprint: () => args.shade.getFingerprintFor(args.sender), + metadata: new Map(), + }; +} diff --git a/packages/shade-files/src/server/handler.ts b/packages/shade-files/src/server/handler.ts new file mode 100644 index 0000000..7714da8 --- /dev/null +++ b/packages/shade-files/src/server/handler.ts @@ -0,0 +1,701 @@ +import type { Shade } from '@shade/sdk'; +import type { ZodTypeAny } from 'zod'; +import { + MUTATION_OPS, + opOfKind, + responseKindOf, + type StandardOp, +} from '../protocol/kinds.js'; +import { + DeleteArgsSchema, + DeleteResultSchema, + GetThumbnailArgsSchema, + GetThumbnailResultSchema, + ListArgsSchema, + ListResultSchema, + MkdirArgsSchema, + MkdirResultSchema, + MoveArgsSchema, + MoveResultSchema, + ReadArgsSchema, + ReadResultSchema, + StatArgsSchema, + StatResultSchema, + WriteArgsSchema, + WriteResultSchema, + type DeleteArgs, + type DeleteResult, + type GetThumbnailArgs, + type ListArgs, + type ListResult, + type MkdirArgs, + type MkdirResult, + type MoveArgs, + type MoveResult, + type ReadArgs, + type StatArgs, + type StatResult, + type WriteArgs, + type WriteResult, +} from '../schemas/ops.js'; +import { + CancelledError, + CustomOpRejectedError, + FileError, + FingerprintRequiredError, + InvalidArgsError, + InvalidSignatureError, + NotImplementedError, + PathValidationError, + payloadFromError, +} from '../schemas/errors.js'; +import type { RpcCancel, RpcError, RpcRequest, RpcResponse } from '../schemas/envelope.js'; +import { buildOpContext, type OpContext } from './handler-context.js'; +import { IdempotencyCache, type IdempotencyCacheOptions } from './idempotency-cache.js'; +import { RateLimiter, type RateLimitConfig } from './rate-limiter.js'; +import { validatePath, type PathPolicy } from './path-policy.js'; +import { + adaptReadResult, + adaptThumbnailResult, + adaptWriteArgs, +} from './io-adapters.js'; +import type { + UserReadResult, + UserThumbnailResult, + UserWriteArgs, +} from './io-types.js'; +import type { ServerStreamsBridge } from './streams-bridge.js'; +import type { CustomOpRegistrations } from './custom-ops.js'; +import { + METRIC_BYTES_IN, + METRIC_BYTES_OUT, + METRIC_FINGERPRINT_REJECT_TOTAL, + METRIC_IDEMPOTENCY_CONFLICT_TOTAL, + METRIC_IDEMPOTENCY_HIT_TOTAL, + METRIC_OP_DURATION_MS, + METRIC_OP_TOTAL, + METRIC_RATE_LIMIT_REJECT_TOTAL, + METRIC_SIGNATURE_REJECT_TOTAL, + NOOP_METRIC_SINK, + type MetricSink, +} from './metrics.js'; +import { + CustomArgsSchema, + CustomResultSchema, + type CustomArgs, +} from '../schemas/ops.js'; +import { canonicalRpcBytes, hashArgs } from '../protocol/canonical.js'; + +/** Replay window for the `signedAt` field on inbound RPC envelopes. */ +export const MAX_SIGNATURE_AGE_MS = 5 * 60 * 1000; + +// ─── Public types ──────────────────────────────────────────── + +export interface FileHandlerOps { + list?: (ctx: OpContext) => Promise; + stat?: (ctx: OpContext) => Promise; + mkdir?: (ctx: OpContext) => Promise; + delete?: (ctx: OpContext) => Promise; + move?: (ctx: OpContext) => Promise; + /** + * User-supplied read handler. Returns either an `inline` payload (≤ 256 + * KiB) or a `streams` payload with a precomputed sha256. The dispatcher + * adapts to the wire shape. + */ + read?: (ctx: OpContext) => Promise; + /** + * User-supplied write handler. Receives `UserWriteArgs` with a clean + * `Uint8Array` (inline) or `ReadableStream` + sha256-promise (streams) + * shape — the dispatcher hides the base64 / writeId wire details. + */ + write?: (ctx: OpContext) => Promise; + /** + * User-supplied thumbnail handler. Bytes are validated for format magic + * before they're shipped to prevent format misclassification attacks. + */ + getThumbnail?: (ctx: OpContext) => Promise; +} + +export interface FileHandlerConfig extends FileHandlerOps { + pathPolicy?: PathPolicy; + rateLimits?: RateLimitConfig; + idempotency?: IdempotencyCacheOptions; + /** + * Required for read/write `streams` ops. Coordinates the inbound/outbound + * `@shade/transfer` transfers via `userMetadata.shadeFiles*Id` keys. + */ + streamsBridge?: ServerStreamsBridge; + /** Custom ops registry — see `CustomOpsMap` declaration-merging. */ + custom?: CustomOpRegistrations; + /** + * Verify the Ed25519 signature on inbound RPC envelopes. Pluggable so + * apps can plug their own peer-identity store. Returning `false` rejects + * with `InvalidSignatureError`. Default: skip (Double Ratchet AEAD on + * the underlying envelope already authenticates the sender). + * + * The `signedAt` replay-window check (±5 min) is enforced regardless. + */ + verifySender?: ( + sender: string, + canonicalBytes: Uint8Array, + sig: string, + ) => boolean | Promise; + /** + * Per-op fingerprint-verification gate. Return `'required'` to demand + * the peer's fingerprint has been verified out-of-band (via + * `isFingerprintVerified`); `'reject'` to deny outright; + * `'optional'` (default) to allow. + */ + requireFingerprintVerifiedFor?: ( + ctx: OpContext, + ) => 'required' | 'optional' | 'reject' | Promise<'required' | 'optional' | 'reject'>; + /** Lookup whether the consumer has out-of-band verified the peer. */ + isFingerprintVerified?: (sender: string) => boolean | Promise; + /** Vendor-neutral metrics sink. */ + onMetric?: MetricSink; + /** Called BEFORE the handler runs. Throw to deny. */ + beforeOp?: (ctx: OpContext) => void | Promise; + /** Called AFTER the handler returns. Result is the validated response. */ + afterOp?: (ctx: OpContext, result: unknown) => void | Promise; + /** Called when an op fails. Receives the error and the context. */ + onError?: (err: unknown, ctx: OpContext) => void; + /** Default per-op timeout in ms. Default 60_000. */ + defaultTimeoutMs?: number; + /** Hard deadline for streams-bridge awaits / outbound transfers. Default 60_000. */ + ioTimeoutMs?: number; +} + +export interface FileHandler { + /** Handle an incoming request envelope. Returns the envelope to send back. */ + handleRequest(from: string, request: RpcRequest): Promise; + /** Handle an incoming cancel envelope. */ + handleCancel(from: string, cancel: RpcCancel): void; + /** Free up internal state (timers, abort listeners). */ + destroy(): void; +} + +interface OpSchemaPair { + args: ZodTypeAny; + result: ZodTypeAny; +} + +const OP_SCHEMAS: Record = { + list: { args: ListArgsSchema, result: ListResultSchema }, + stat: { args: StatArgsSchema, result: StatResultSchema }, + mkdir: { args: MkdirArgsSchema, result: MkdirResultSchema }, + delete: { args: DeleteArgsSchema, result: DeleteResultSchema }, + move: { args: MoveArgsSchema, result: MoveResultSchema }, + read: { args: ReadArgsSchema, result: ReadResultSchema }, + write: { args: WriteArgsSchema, result: WriteResultSchema }, + getThumbnail: { args: GetThumbnailArgsSchema, result: GetThumbnailResultSchema }, +}; + +// ─── createFileHandler ─────────────────────────────────────── + +/** + * Build a `FileHandler` for the server side. The returned object is + * registered with `ShadeFileRpcChannel.setHooks({ onRequest })` (typically + * via `Shade.files.serve(...)` in the SDK). + */ +export function createFileHandler( + shade: Shade, + config: FileHandlerConfig, +): FileHandler { + const idempotency = new IdempotencyCache(config.idempotency); + const rateLimiter = new RateLimiter(config.rateLimits); + const inflightCancellers = new Map(); + const defaultTimeoutMs = config.defaultTimeoutMs ?? 60_000; + const ioTimeoutMs = config.ioTimeoutMs ?? 60_000; + const metrics: MetricSink = config.onMetric ?? NOOP_METRIC_SINK; + const customRegistrations = config.custom ?? {}; + const isCustomKind = (kind: string): boolean => kind === 'shade.fs.custom/v1'; + + async function handleRequest( + from: string, + request: RpcRequest, + ): Promise { + // 0. Replay-window check (independent of sig — defends against + // intercept-and-resend even when sig verification is disabled). + const skewMs = Math.abs(Date.now() - request.signedAt); + if (skewMs > MAX_SIGNATURE_AGE_MS) { + metrics(METRIC_SIGNATURE_REJECT_TOTAL, 1, { reason: 'skew' }); + return makeErrorEnvelope( + request, + new InvalidSignatureError( + `signedAt is ${skewMs}ms outside the ±${MAX_SIGNATURE_AGE_MS}ms replay window`, + ), + ); + } + + // 0b. Optional Ed25519 sig verification — pluggable. The canonical + // bytes bind sender + kind + id + signedAt + sha256(args). + if (config.verifySender !== undefined) { + const argsHashBytes = hashArgs(request.args); + const canonical = canonicalRpcBytes({ + address: from, + signedAt: request.signedAt, + kind: request.kind, + id: request.id, + argsHash: argsHashBytes, + }); + let ok = false; + try { + ok = await config.verifySender(from, canonical, request.sig); + } catch (err) { + metrics(METRIC_SIGNATURE_REJECT_TOTAL, 1, { reason: 'throw' }); + return makeErrorEnvelope(request, err); + } + if (!ok) { + metrics(METRIC_SIGNATURE_REJECT_TOTAL, 1, { reason: 'mismatch' }); + return makeErrorEnvelope(request, new InvalidSignatureError()); + } + } + + // 1. Resolve op + handler. Custom ops route through `config.custom`. + let op: StandardOp | 'custom'; + let argSchema: ZodTypeAny; + let resultSchema: ZodTypeAny; + let customHandler: CustomOpRegistrations[string] | undefined; + + if (isCustomKind(request.kind)) { + op = 'custom'; + argSchema = CustomArgsSchema; + resultSchema = CustomResultSchema; + } else { + const std = opOfKind(request.kind); + if (std === null) { + return makeErrorEnvelope(request, new NotImplementedError(request.kind)); + } + op = std; + argSchema = OP_SCHEMAS[std].args; + resultSchema = OP_SCHEMAS[std].result; + const handler = config[std]; + if (handler === undefined) { + return makeErrorEnvelope(request, new NotImplementedError(std)); + } + } + + // 2. Args validation (wire shape) + const argParse = argSchema.safeParse(request.args); + if (!argParse.success) { + const issue = argParse.error.issues[0]; + return makeErrorEnvelope( + request, + new InvalidArgsError( + issue?.message ?? 'invalid arguments', + issue?.path.join('.') || undefined, + ), + ); + } + const parsedArgs = argParse.data as unknown; + + // 2b. Custom op resolution + let resolvedOpKind: string = op; + if (op === 'custom') { + const customArgs = parsedArgs as CustomArgs; + customHandler = customRegistrations[customArgs.name]; + if (customHandler === undefined) { + return makeErrorEnvelope( + request, + new NotImplementedError(`custom:${customArgs.name}`), + ); + } + resolvedOpKind = `custom:${customArgs.name}`; + // Validate inner payload against the custom op's args schema + const payloadParse = customHandler.args.safeParse(customArgs.payload); + if (!payloadParse.success) { + const issue = payloadParse.error.issues[0]; + return makeErrorEnvelope( + request, + new InvalidArgsError( + issue?.message ?? 'invalid custom-op payload', + issue?.path.join('.') || undefined, + ), + ); + } + // Replace payload with validated value (Zod may apply defaults). + (parsedArgs as CustomArgs).payload = payloadParse.data; + } + + // 3. Path validation (skip ops without a path) + let primaryPath = ''; + if (op === 'move') { + primaryPath = (parsedArgs as MoveArgs).src; + } else if (op !== 'custom' && 'path' in (parsedArgs as object)) { + primaryPath = (parsedArgs as { path: string }).path; + } + let normalizedPath = primaryPath; + if (primaryPath !== '') { + const validated = validatePath(primaryPath, config.pathPolicy); + if (!validated.ok) { + return makeErrorEnvelope(request, new PathValidationError(validated.reason)); + } + normalizedPath = validated.normalized; + if (op === 'move') { + const dstValid = validatePath((parsedArgs as MoveArgs).dst, config.pathPolicy); + if (!dstValid.ok) { + return makeErrorEnvelope( + request, + new PathValidationError(dstValid.reason, 'dst'), + ); + } + (parsedArgs as MoveArgs).src = normalizedPath; + (parsedArgs as MoveArgs).dst = dstValid.normalized; + } else { + (parsedArgs as { path: string }).path = normalizedPath; + } + } + + // 4. Rate-limit acquire + const opCostKey = op === 'custom' ? 'custom' : op; + const customCost = customHandler?.cost; + const estimatedBytes = estimateBytes(op === 'custom' ? 'custom' : op, parsedArgs); + try { + if (customCost !== undefined) { + // Custom op-specific cost: acquire that many op-tokens manually. + // Fall back to standard `custom` bucket cost for non-overridden. + rateLimiter.acquire(from, opCostKey, estimatedBytes); + // For overridden cost > 1, deduct extra tokens from the same bucket. + // The simplest route: re-acquire (cost - 1) more. + for (let i = 1; i < customCost; i++) { + rateLimiter.acquire(from, opCostKey, 0); + } + } else { + rateLimiter.acquire(from, opCostKey, estimatedBytes); + } + } catch (err) { + metrics(METRIC_RATE_LIMIT_REJECT_TOTAL, 1, { op: resolvedOpKind }); + return makeErrorEnvelope(request, err); + } + + // 5. Idempotency (mutations only) + const isMutation = MUTATION_OPS.has(opCostKey); + let commitIdem: ((response: unknown) => void) | null = null; + let abandonIdem: (() => void) | null = null; + if (isMutation && request.idempotencyKey !== undefined) { + try { + const begin = idempotency.begin(from, request.idempotencyKey, parsedArgs); + if (begin.status === 'replay') { + rateLimiter.release(from, opCostKey, estimatedBytes); + metrics(METRIC_IDEMPOTENCY_HIT_TOTAL, 1, { op: resolvedOpKind }); + return makeResponseEnvelope(request, begin.response); + } + if (begin.status === 'wait') { + const cached = await begin.promise; + rateLimiter.release(from, opCostKey, estimatedBytes); + metrics(METRIC_IDEMPOTENCY_HIT_TOTAL, 1, { op: resolvedOpKind }); + return makeResponseEnvelope(request, cached); + } + commitIdem = begin.commit; + abandonIdem = begin.abandon; + } catch (err) { + rateLimiter.release(from, opCostKey, estimatedBytes); + if ( + err !== null && + typeof err === 'object' && + (err as { code?: string }).code === 'SHADE_FS_IDEMPOTENCY_CONFLICT' + ) { + metrics(METRIC_IDEMPOTENCY_CONFLICT_TOTAL, 1, { op: resolvedOpKind }); + } + return makeErrorEnvelope(request, err); + } + } + + // 6. Build context + abort controller + const controller = new AbortController(); + inflightCancellers.set(request.id, controller); + const ctx = buildOpContext({ + op: op === 'custom' ? (resolvedOpKind as `custom:${string}`) : op, + path: normalizedPath, + parsedArgs, + sender: from, + signal: controller.signal, + idempotencyKey: request.idempotencyKey, + attemptNumber: request.attempt ?? 1, + shade, + }); + + // 7. Fingerprint gate + if (config.requireFingerprintVerifiedFor !== undefined) { + let gate: 'required' | 'optional' | 'reject'; + try { + gate = await config.requireFingerprintVerifiedFor(ctx as OpContext); + } catch (err) { + cleanup({ release: true }); + return makeErrorEnvelope(request, err); + } + if (gate === 'reject') { + cleanup({ release: true }); + metrics(METRIC_FINGERPRINT_REJECT_TOTAL, 1, { op: resolvedOpKind, gate: 'reject' }); + return makeErrorEnvelope( + request, + new FingerprintRequiredError('operation rejected by fingerprint policy'), + ); + } + if (gate === 'required') { + let verified = false; + try { + verified = config.isFingerprintVerified !== undefined + ? Boolean(await config.isFingerprintVerified(from)) + : false; + } catch (err) { + cleanup({ release: true }); + return makeErrorEnvelope(request, err); + } + if (!verified) { + cleanup({ release: true }); + metrics(METRIC_FINGERPRINT_REJECT_TOTAL, 1, { op: resolvedOpKind, gate: 'required' }); + return makeErrorEnvelope(request, new FingerprintRequiredError()); + } + } + } + + // 8. beforeOp + try { + if (config.beforeOp !== undefined) { + await config.beforeOp(ctx as OpContext); + } + } catch (err) { + cleanup({ release: true }); + return makeErrorEnvelope(request, err); + } + + // 9. Run handler with timeout race — adapting I/O ops as needed. + const timeoutMs = Math.min( + request.deadlineMs ?? defaultTimeoutMs, + defaultTimeoutMs, + ); + + let wireResult: unknown; + const startedAt = Date.now(); + try { + wireResult = await runWithTimeout( + () => invokeOpHandler({ + op, + stdHandler: op === 'custom' ? undefined : (config[op] as unknown), + customHandler, + ctx: ctx as OpContext, + parsedArgs, + sender: from, + signal: controller.signal, + streamsBridge: config.streamsBridge, + ioTimeoutMs, + }), + timeoutMs, + controller, + ); + } catch (err) { + const durationMs = Date.now() - startedAt; + metrics(METRIC_OP_DURATION_MS, durationMs, { op: resolvedOpKind, result: 'error' }); + metrics(METRIC_OP_TOTAL, 1, { op: resolvedOpKind, result: 'error' }); + cleanup({ release: true }); + if (config.onError !== undefined) { + try { + config.onError(err, ctx as OpContext); + } catch (hookErr) { + console.error('[FileHandler] onError hook threw:', hookErr); + } + } + return makeErrorEnvelope(request, err); + } + + // 10. Custom-op response validation against the registered schema. + if (op === 'custom' && customHandler !== undefined) { + const innerParse = customHandler.response.safeParse(wireResult); + if (!innerParse.success) { + cleanup({ release: true }); + return makeErrorEnvelope( + request, + new CustomOpRejectedError( + `custom-op response shape rejected by registered schema: ${innerParse.error.issues[0]?.message ?? 'unknown'}`, + ), + ); + } + wireResult = { result: innerParse.data }; + } + + // 11. Defensive response validation against the wire schema. + const resultParse = resultSchema.safeParse(wireResult); + if (!resultParse.success) { + cleanup({ release: true }); + return makeErrorEnvelope( + request, + new InvalidArgsError( + `handler for ${resolvedOpKind} returned invalid response shape`, + ), + ); + } + + // 12. afterOp + try { + if (config.afterOp !== undefined) { + await config.afterOp(ctx as OpContext, resultParse.data); + } + } catch (err) { + cleanup({ release: true }); + return makeErrorEnvelope(request, err); + } + + // 13. Commit idempotency + emit metrics + commitIdem?.(resultParse.data); + cleanup({ release: false }); + + const durationMs = Date.now() - startedAt; + metrics(METRIC_OP_DURATION_MS, durationMs, { op: resolvedOpKind, result: 'ok' }); + metrics(METRIC_OP_TOTAL, 1, { op: resolvedOpKind, result: 'ok' }); + if (estimatedBytes > 0) { + // Inbound bytes (write) vs outbound (read) — both reuse the same + // pre-call `estimatedBytes`, since post-execution reconciliation + // would require deeper plumbing. + const direction = op === 'write' ? METRIC_BYTES_IN : op === 'read' ? METRIC_BYTES_OUT : null; + if (direction !== null) { + metrics(direction, estimatedBytes, { op: resolvedOpKind }); + } + } + + return makeResponseEnvelope(request, resultParse.data); + + function cleanup(opts: { release: boolean }): void { + inflightCancellers.delete(request.id); + if (opts.release) { + abandonIdem?.(); + rateLimiter.release(from, opCostKey, estimatedBytes); + } + } + } + + function handleCancel(_from: string, cancel: RpcCancel): void { + const controller = inflightCancellers.get(cancel.id); + if (controller !== undefined) { + controller.abort(new CancelledError(cancel.reason ?? 'cancelled by sender')); + inflightCancellers.delete(cancel.id); + } + } + + function destroy(): void { + for (const c of inflightCancellers.values()) { + c.abort(new CancelledError('handler destroyed')); + } + inflightCancellers.clear(); + } + + return Object.assign({ handleRequest, handleCancel, destroy }, { + [INTERNAL_SYMBOL]: { idempotency, rateLimiter }, + }); +} + +export const INTERNAL_SYMBOL = Symbol.for('@shade/files/internal'); + +// ─── Op invoker (handles I/O adapters) ─────────────────────── + +interface InvokeArgs { + op: StandardOp | 'custom'; + stdHandler: unknown; + customHandler: CustomOpRegistrations[string] | undefined; + ctx: OpContext; + parsedArgs: unknown; + sender: string; + signal: AbortSignal; + streamsBridge: ServerStreamsBridge | undefined; + ioTimeoutMs: number; +} + +async function invokeOpHandler(args: InvokeArgs): Promise { + const { op, stdHandler, customHandler, ctx, parsedArgs, sender, signal, streamsBridge, ioTimeoutMs } = args; + const adapterDeps = { streamsBridge, sender, signal, ioTimeoutMs }; + + switch (op) { + case 'write': { + const wireArgs = parsedArgs as WriteArgs; + const { userArgs, awaitTransferDone } = await adaptWriteArgs(wireArgs, adapterDeps); + const userCtx = { ...ctx, args: userArgs } as OpContext; + const userResult = await (stdHandler as (c: OpContext) => Promise)(userCtx); + await awaitTransferDone(); + return userResult; + } + + case 'read': { + const readArgs = parsedArgs as ReadArgs; + const userResult = await (stdHandler as (c: OpContext) => Promise)(ctx as OpContext); + return await adaptReadResult(userResult, readArgs, adapterDeps); + } + + case 'getThumbnail': { + const userResult = await (stdHandler as (c: OpContext) => Promise)(ctx as OpContext); + return adaptThumbnailResult(userResult); + } + + case 'custom': { + if (customHandler === undefined) { + throw new NotImplementedError('custom op without registration'); + } + const customArgs = parsedArgs as CustomArgs; + const innerCtx = { ...ctx, args: customArgs } as OpContext<{ name: string; payload: unknown }>; + // Pass the validated inner payload as the first arg, the OpContext as the second. + return await customHandler.handler( + customArgs.payload, + innerCtx as OpContext<{ name: string; payload: unknown }>, + ); + } + + default: + // Pass-through for list/stat/mkdir/delete/move. + return await (stdHandler as (c: OpContext) => Promise)(ctx); + } +} + +// ─── Helpers ───────────────────────────────────────────────── + +function makeResponseEnvelope(req: RpcRequest, result: unknown): RpcResponse { + return { + kind: responseKindOf(req.kind), + id: req.id, + result, + }; +} + +function makeErrorEnvelope(req: RpcRequest, err: unknown): RpcError { + return { + kind: 'shade.fs.error/v1', + id: req.id, + error: payloadFromError(err), + }; +} + +function estimateBytes(op: StandardOp | 'custom', args: unknown): number { + if (op === 'write') { + const w = args as { kind: 'inline' | 'streams'; bytesB64?: string; size?: number }; + if (w.kind === 'inline' && typeof w.bytesB64 === 'string') { + return Math.floor((w.bytesB64.length * 3) / 4); + } + return w.size ?? 0; + } + if (op === 'read') { + const r = args as { range?: { start: number; end: number } }; + if (r.range !== undefined) return r.range.end - r.range.start; + return 0; + } + return 0; +} + +async function runWithTimeout( + fn: () => Promise, + timeoutMs: number, + controller: AbortController, +): Promise { + let timer: ReturnType | null = null; + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => { + controller.abort(new Error('timeout')); + reject(new (FileError as unknown as { new (p: { code: string; message: string }): FileError })({ + code: 'OPERATION_TIMEOUT', + message: `operation timed out after ${timeoutMs}ms`, + })); + }, timeoutMs); + }); + try { + return await Promise.race([fn(), timeout]); + } finally { + if (timer !== null) clearTimeout(timer); + } +} diff --git a/packages/shade-files/src/server/idempotency-cache.ts b/packages/shade-files/src/server/idempotency-cache.ts new file mode 100644 index 0000000..e794b43 --- /dev/null +++ b/packages/shade-files/src/server/idempotency-cache.ts @@ -0,0 +1,160 @@ +import { hashArgs, bytesToHex } from '../protocol/canonical.js'; +import { IdempotencyConflictError } from '../schemas/errors.js'; + +interface Entry { + argsHash: string; + /** `undefined` while inflight; resolved value once handler completes. */ + status: 'inflight' | 'done'; + promise?: Promise; + response?: unknown; + insertedAt: number; + lastAccessAt: number; +} + +export interface IdempotencyCacheOptions { + /** Time in ms after which an entry is eligible for eviction. Default 1h. */ + ttlMs?: number; + /** Per-sender hard cap on cached entries; oldest-by-insert evicted first. */ + maxEntriesPerSender?: number; +} + +/** + * Per-sender LRU + TTL cache for idempotent RPC calls. + * + * - On lookup, returns cached response if `argsHash` matches the original + * call. Throws `IdempotencyConflictError` on argsHash mismatch. + * - On concurrent retry of the same key (still inflight), the second + * caller awaits the first's promise — no double-execution. + * - LRU eviction caps memory at `maxEntriesPerSender` per sender. + * + * Keyed on `(senderAddress, idempotencyKey)`. Internally a 2-level map. + */ +export class IdempotencyCache { + private readonly ttlMs: number; + private readonly maxPerSender: number; + private readonly bySender = new Map>(); + + constructor(opts: IdempotencyCacheOptions = {}) { + this.ttlMs = opts.ttlMs ?? 60 * 60 * 1000; + this.maxPerSender = opts.maxEntriesPerSender ?? 10_000; + } + + /** + * Begin (or rejoin) an idempotent operation. Returns: + * - `{ status: 'fresh', commit }` — handler should run, then call `commit(response)`. + * - `{ status: 'wait', promise }` — concurrent retry; await the in-flight promise. + * - `{ status: 'replay', response }` — completed earlier; return cached. + * + * Mismatched argsHash throws `IdempotencyConflictError` immediately. + */ + begin( + sender: string, + key: string, + args: unknown, + ): + | { status: 'fresh'; commit: (response: unknown) => void; abandon: () => void } + | { status: 'wait'; promise: Promise } + | { status: 'replay'; response: unknown } { + const argsHash = bytesToHex(hashArgs(args)); + const senderMap = this.getOrCreate(sender); + this.evictExpired(senderMap); + const existing = senderMap.get(key); + if (existing !== undefined) { + if (existing.argsHash !== argsHash) { + throw new IdempotencyConflictError(); + } + existing.lastAccessAt = Date.now(); + if (existing.status === 'done') { + return { status: 'replay', response: existing.response }; + } + // Inflight — return the same promise so concurrent retries de-dupe. + return { status: 'wait', promise: existing.promise! }; + } + + let resolveInflight: (value: unknown) => void = () => {}; + let rejectInflight: (err: unknown) => void = () => {}; + const inflightPromise = new Promise((resolve, reject) => { + resolveInflight = resolve; + rejectInflight = reject; + }); + // Suppress unhandled-rejection warnings when no concurrent retry is + // awaiting this promise. Real waiters attach their own catch via + // `await begin().promise` and still see the rejection. + inflightPromise.catch(() => { + /* swallow — see comment above */ + }); + const entry: Entry = { + argsHash, + status: 'inflight', + promise: inflightPromise, + insertedAt: Date.now(), + lastAccessAt: Date.now(), + }; + senderMap.set(key, entry); + this.evictOverflow(senderMap); + + const commit = (response: unknown): void => { + entry.status = 'done'; + entry.response = response; + entry.lastAccessAt = Date.now(); + resolveInflight(response); + }; + const abandon = (): void => { + // Failed/cancelled call — remove from cache so retries can proceed. + senderMap.delete(key); + rejectInflight(new Error('idempotency abandoned')); + }; + + return { status: 'fresh', commit, abandon }; + } + + /** Manual prune; called by the periodic retention job. */ + prune(now: number = Date.now()): number { + let removed = 0; + for (const [sender, senderMap] of this.bySender) { + for (const [key, entry] of senderMap) { + if (now - entry.insertedAt > this.ttlMs) { + senderMap.delete(key); + removed++; + } + } + if (senderMap.size === 0) this.bySender.delete(sender); + } + return removed; + } + + /** Total cached entries across all senders (mainly for tests/metrics). */ + size(): number { + let total = 0; + for (const m of this.bySender.values()) total += m.size; + return total; + } + + private getOrCreate(sender: string): Map { + let m = this.bySender.get(sender); + if (m === undefined) { + m = new Map(); + this.bySender.set(sender, m); + } + return m; + } + + private evictExpired(senderMap: Map): void { + const now = Date.now(); + for (const [key, entry] of senderMap) { + if (now - entry.insertedAt > this.ttlMs) senderMap.delete(key); + } + } + + private evictOverflow(senderMap: Map): void { + if (senderMap.size <= this.maxPerSender) return; + // Map iteration order = insertion order; first key is oldest. + const overflow = senderMap.size - this.maxPerSender; + let i = 0; + for (const key of senderMap.keys()) { + if (i >= overflow) break; + senderMap.delete(key); + i++; + } + } +} diff --git a/packages/shade-files/src/server/io-adapters.ts b/packages/shade-files/src/server/io-adapters.ts new file mode 100644 index 0000000..93c4952 --- /dev/null +++ b/packages/shade-files/src/server/io-adapters.ts @@ -0,0 +1,229 @@ +/** + * Adapt between wire-shaped I/O ops and user-handler-shaped I/O ops. + * + * client → server (write): WriteArgs (wire) → UserWriteArgs (handler) + * server → client (read): UserReadResult (handler) → ReadResult (wire) + * server → client (thumb): UserThumbnailResult → GetThumbnailResult + * + * The streams-bridge is consulted whenever we're crossing the > 256 KiB + * boundary: outbound reads kick a new transfer; inbound writes await one. + */ + +import { sha256 } from '@noble/hashes/sha2.js'; +import type { + GetThumbnailResult, + ReadArgs, + ReadResult, + WriteArgs, +} from '../schemas/ops.js'; +import { base64ToBytes, bytesToBase64, bytesToHex } from '../protocol/canonical.js'; +import { INLINE_THRESHOLD } from '../client/inline-threshold.js'; +import { + ConflictError, + InternalFileError, + InvalidArgsError, +} from '../schemas/errors.js'; +import type { + UserReadResult, + UserThumbnailResult, + UserWriteArgs, +} from './io-types.js'; +import type { ServerStreamsBridge } from './streams-bridge.js'; + +export interface IoAdapterDeps { + streamsBridge: ServerStreamsBridge | undefined; + sender: string; + signal: AbortSignal; + /** Hard deadline for streams-bridge awaits / outbound transfers. */ + ioTimeoutMs: number; +} + +// ─── write: wire args → user args ──────────────────────────── + +/** + * Convert validated `WriteArgs` (wire shape) into `UserWriteArgs`. For + * streams writes, this awaits the matching inbound transfer via the + * streams-bridge and synthesizes a `ReadableStream` for the user handler. + */ +export async function adaptWriteArgs( + wireArgs: WriteArgs, + deps: IoAdapterDeps, +): Promise<{ + userArgs: UserWriteArgs; + /** Resolves once the inbound transfer (if any) has completed. */ + awaitTransferDone: () => Promise; +}> { + if (wireArgs.kind === 'inline') { + const bytes = base64ToBytes(wireArgs.bytesB64); + if (bytes.byteLength > INLINE_THRESHOLD) { + throw new InvalidArgsError( + `inline write exceeds ${INLINE_THRESHOLD}-byte threshold (got ${bytes.byteLength})`, + 'bytesB64', + ); + } + const hashHex = bytesToHex(sha256(bytes)); + const userArgs: UserWriteArgs = { + path: wireArgs.path, + overwrite: wireArgs.overwrite, + ...(wireArgs.contentType !== undefined ? { contentType: wireArgs.contentType } : {}), + content: { + kind: 'inline', + bytes, + size: bytes.byteLength, + sha256: hashHex, + }, + }; + return { userArgs, awaitTransferDone: async () => undefined }; + } + + // Streams write — must have a streams-bridge configured. + if (deps.streamsBridge === undefined) { + throw new InternalFileError( + 'streams-bridge not configured: cannot accept streamed write', + ); + } + const parked = await deps.streamsBridge.awaitWrite(wireArgs.writeId, { + expectedFrom: deps.sender, + signal: deps.signal, + timeoutMs: deps.ioTimeoutMs, + }); + + // sha256 from the inbound transfer's done() — user handler can `await` + // it immediately after draining the readable. + const sha256Promise = parked.done.then((r) => r.sha256); + sha256Promise.catch(() => { + /* swallow until consumer awaits */ + }); + + // Cancellation: if the user handler aborts (or the dispatcher times out), + // also tear down the inbound transfer. + const onAbort = (): void => { + void parked.handle.abort('user-cancel').catch(() => undefined); + }; + if (deps.signal.aborted) onAbort(); + else deps.signal.addEventListener('abort', onAbort, { once: true }); + + // The dispatcher calls this AFTER the user handler resolves to ensure + // the transfer is fully drained (so any wire-level integrity errors + // surface here before we send the RPC ack to the client). + const awaitTransferDone = async (): Promise => { + try { + await parked.done; + } finally { + deps.signal.removeEventListener('abort', onAbort); + } + }; + + const userArgs: UserWriteArgs = { + path: wireArgs.path, + overwrite: wireArgs.overwrite, + ...(wireArgs.contentType !== undefined ? { contentType: wireArgs.contentType } : {}), + content: { + kind: 'streams', + stream: parked.readable, + size: wireArgs.size, + sha256: sha256Promise, + }, + }; + return { userArgs, awaitTransferDone }; +} + +// ─── read: user result → wire result ───────────────────────── + +/** + * Convert a user-supplied `UserReadResult` into a wire `ReadResult`. For + * streams results, this kicks an outbound `@shade/transfer` and embeds the + * resulting `streamId` in the response so the client-side bridge can + * subscribe. + */ +export async function adaptReadResult( + userResult: UserReadResult, + args: ReadArgs, + deps: IoAdapterDeps, +): Promise { + if (userResult.kind === 'inline') { + if (userResult.bytes.byteLength > INLINE_THRESHOLD) { + throw new InternalFileError( + `inline read result exceeds ${INLINE_THRESHOLD} bytes (got ${userResult.bytes.byteLength})`, + ); + } + const sha256Hex = userResult.sha256 ?? bytesToHex(sha256(userResult.bytes)); + return { + kind: 'inline', + bytesB64: bytesToBase64(userResult.bytes), + size: userResult.bytes.byteLength, + sha256: sha256Hex, + ...(userResult.contentType !== undefined ? { contentType: userResult.contentType } : {}), + }; + } + + if (deps.streamsBridge === undefined) { + throw new InternalFileError( + 'streams-bridge not configured: cannot ship streamed read result', + ); + } + const { readStreamId } = await deps.streamsBridge.initiateRead({ + peer: deps.sender, + stream: userResult.stream, + size: userResult.size, + ...(userResult.contentType !== undefined ? { contentType: userResult.contentType } : {}), + name: args.path, + signal: deps.signal, + }); + return { + kind: 'streams', + streamId: readStreamId, + size: userResult.size, + sha256: userResult.sha256, + ...(userResult.contentType !== undefined ? { contentType: userResult.contentType } : {}), + }; +} + +// ─── getThumbnail: user result → wire result ───────────────── + +const FORMAT_MAGIC: Record = { + png: new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + jpeg: new Uint8Array([0xff, 0xd8, 0xff]), + webp: new Uint8Array([0x52, 0x49, 0x46, 0x46]), // 'RIFF', then size, then 'WEBP' +}; +const WEBP_HEAD = new Uint8Array([0x57, 0x45, 0x42, 0x50]); // 'WEBP' at offset 8 + +/** + * Validate a user thumbnail result for format magic-bytes match. The user + * MAY produce any image format internally, but this guards against + * misclassification (e.g. claiming PNG with JPEG bytes), which can confuse + * downstream rendering and security scanners. + */ +function checkFormatMagic(bytes: Uint8Array, format: 'png' | 'webp' | 'jpeg'): boolean { + const magic = FORMAT_MAGIC[format]; + if (magic === undefined) return false; + if (bytes.byteLength < magic.byteLength) return false; + for (let i = 0; i < magic.byteLength; i++) { + if (bytes[i] !== magic[i]) return false; + } + if (format === 'webp') { + if (bytes.byteLength < 12) return false; + for (let i = 0; i < 4; i++) { + if (bytes[8 + i] !== WEBP_HEAD[i]) return false; + } + } + return true; +} + +export function adaptThumbnailResult( + userResult: UserThumbnailResult, +): GetThumbnailResult { + if (!checkFormatMagic(userResult.bytes, userResult.format)) { + throw new ConflictError( + `thumbnail bytes do not match declared format=${userResult.format}`, + ); + } + const hash = userResult.sha256 ?? bytesToHex(sha256(userResult.bytes)); + return { + bytesB64: bytesToBase64(userResult.bytes), + format: userResult.format, + width: userResult.width, + height: userResult.height, + sha256: hash, + }; +} diff --git a/packages/shade-files/src/server/io-types.ts b/packages/shade-files/src/server/io-types.ts new file mode 100644 index 0000000..7829e01 --- /dev/null +++ b/packages/shade-files/src/server/io-types.ts @@ -0,0 +1,102 @@ +/** + * Server-facing types for the I/O ops (`read`, `write`, `getThumbnail`). + * + * These differ from the wire schemas in `schemas/ops.ts` so user-supplied + * handlers see clean, typed values (`Uint8Array`, `ReadableStream`) instead + * of base64 strings and opaque transfer ids. The dispatcher in + * `server/handler.ts` adapts between the two. + */ + +import type { ThumbnailSize, WriteResult } from '../schemas/ops.js'; + +// ─── read handler types ────────────────────────────────────── + +/** + * Value returned by a user-supplied `read` handler. + * + * - `inline`: the bytes are returned in-band; the dispatcher base64-encodes + * them and computes the sha256 if the user did not provide one. Inline + * reads MUST be ≤ 256 KiB plaintext. + * - `streams`: the user provides a `ReadableStream`, the declared `size`, + * and a precomputed `sha256` (e.g. cached on the file's metadata row). + * The dispatcher initiates an outbound `@shade/transfer` transfer and + * echoes the transfer's `streamId` back to the client over RPC so the + * client-side bridge can subscribe. + */ +export type UserReadResult = + | UserReadResultInline + | UserReadResultStreams; + +export interface UserReadResultInline { + kind: 'inline'; + bytes: Uint8Array; + /** Optional — dispatcher computes if absent. */ + sha256?: string; + contentType?: string; +} + +export interface UserReadResultStreams { + kind: 'streams'; + stream: ReadableStream; + size: number; + /** REQUIRED — the dispatcher cannot synchronously hash a stream. */ + sha256: string; + contentType?: string; +} + +// ─── write handler types ───────────────────────────────────── + +/** + * Value passed to a user-supplied `write` handler. + * + * The dispatcher decodes the wire `WriteArgs` and (for `streams`) parks + * the inbound transfer via the streams-bridge, then synthesizes this + * cleaner shape for the user. + */ +export interface UserWriteArgs { + path: string; + contentType?: string; + overwrite: boolean; + content: UserWriteContent; +} + +export type UserWriteContent = + | UserWriteContentInline + | UserWriteContentStreams; + +export interface UserWriteContentInline { + kind: 'inline'; + bytes: Uint8Array; + size: number; + /** sha256 of `bytes`, computed by the dispatcher. */ + sha256: string; +} + +export interface UserWriteContentStreams { + kind: 'streams'; + /** Plaintext stream. Closes when the inbound `@shade/transfer` completes. */ + stream: ReadableStream; + /** Declared plaintext size from the client's RPC args. */ + size: number; + /** + * Promise that resolves with the transfer-verified sha256 once the entire + * stream has been received. Use this to attach a content hash to the + * user's storage record. + */ + sha256: Promise; +} + +// ─── getThumbnail handler types ────────────────────────────── + +/** Value returned by a user-supplied `getThumbnail` handler. */ +export interface UserThumbnailResult { + bytes: Uint8Array; + format: 'png' | 'webp' | 'jpeg'; + width: number; + height: number; + /** Optional — dispatcher computes if absent. */ + sha256?: string; +} + +// ─── Re-exports ────────────────────────────────────────────── +export type { ThumbnailSize, WriteResult }; diff --git a/packages/shade-files/src/server/metrics.ts b/packages/shade-files/src/server/metrics.ts new file mode 100644 index 0000000..d6f173b --- /dev/null +++ b/packages/shade-files/src/server/metrics.ts @@ -0,0 +1,25 @@ +/** + * Standard metric names emitted by the dispatcher. + * + * The user-supplied `onMetric(name, value, tags)` callback is vendor-neutral — + * pipe to Prometheus, OpenTelemetry, statsd, or just append to a log file. + */ + +export const METRIC_OP_DURATION_MS = 'shade_files_op_duration_ms'; +export const METRIC_OP_TOTAL = 'shade_files_op_total'; +export const METRIC_BYTES_IN = 'shade_files_bytes_in'; +export const METRIC_BYTES_OUT = 'shade_files_bytes_out'; +export const METRIC_IDEMPOTENCY_HIT_TOTAL = 'shade_files_idempotency_hit_total'; +export const METRIC_IDEMPOTENCY_CONFLICT_TOTAL = 'shade_files_idempotency_conflict_total'; +export const METRIC_RATE_LIMIT_REJECT_TOTAL = 'shade_files_rate_limit_reject_total'; +export const METRIC_FINGERPRINT_REJECT_TOTAL = 'shade_files_fingerprint_reject_total'; +export const METRIC_SIGNATURE_REJECT_TOTAL = 'shade_files_signature_reject_total'; + +/** Tags attached to every metric event. */ +export type MetricTags = Record; + +/** User-supplied metric sink. Synchronous to keep hot-path fast. */ +export type MetricSink = (name: string, value: number, tags: MetricTags) => void; + +/** No-op sink used when the user doesn't supply one. */ +export const NOOP_METRIC_SINK: MetricSink = () => undefined; diff --git a/packages/shade-files/src/server/path-policy.ts b/packages/shade-files/src/server/path-policy.ts new file mode 100644 index 0000000..16c889c --- /dev/null +++ b/packages/shade-files/src/server/path-policy.ts @@ -0,0 +1,95 @@ +import { + decodePercentEscapes, + isPathInside, + posixNormalize, +} from '../utils/path.js'; + +export interface PathPolicy { + /** Chroot-style absolute root. All paths must lie inside this directory. */ + rootScope?: string; + /** Reject `..` segments after normalization. Default true. */ + forbidTraversal?: boolean; + /** Hard cap on raw-input length (post-decode). Default 4096. */ + maxLength?: number; + /** Allow symlink components. Default false (consumer enforces realpath if true). */ + allowSymlinks?: boolean; + /** App-specific extra check. Returns 'allow' | 'reject'. */ + extra?: (normalizedPath: string) => 'allow' | 'reject'; +} + +export type PathValidationResult = + | { ok: true; normalized: string } + | { ok: false; reason: string }; + +const DEFAULT_MAX_LENGTH = 4096; +/** Bytes that must never appear in any path. */ +const FORBIDDEN_RE = /[\x00-\x08\x0a-\x1f\x7f\\]/; + +/** + * Validate and normalize a path against the configured policy. + * + * Performs (in order): + * 1. Length check on raw input. + * 2. Forbidden-bytes check on raw input (NUL/CR/LF/DEL/backslash). + * 3. Percent-decode (defense against `%2e%2e` smuggling). + * 4. Forbidden-bytes check on decoded form. + * 5. POSIX normalization. + * 6. `..` traversal rejection. + * 7. Root-scope boundary check. + * 8. Consumer-supplied `extra` predicate. + * + * Returns the normalized path on success. The handler MUST use the + * normalized form, not the raw input — the raw could trip TOCTOU on + * the caller's filesystem. + */ +export function validatePath( + rawPath: string, + policy: PathPolicy = {}, +): PathValidationResult { + const maxLength = policy.maxLength ?? DEFAULT_MAX_LENGTH; + const forbidTraversal = policy.forbidTraversal ?? true; + + if (typeof rawPath !== 'string') { + return { ok: false, reason: 'path must be a string' }; + } + if (rawPath.length === 0) { + return { ok: false, reason: 'path is empty' }; + } + if (rawPath.length > maxLength) { + return { ok: false, reason: `path exceeds ${maxLength} characters` }; + } + if (FORBIDDEN_RE.test(rawPath)) { + return { ok: false, reason: 'path contains forbidden control or backslash characters' }; + } + + const decoded = decodePercentEscapes(rawPath); + if (FORBIDDEN_RE.test(decoded)) { + return { ok: false, reason: 'percent-decoded path contains forbidden characters' }; + } + if (!decoded.startsWith('/')) { + return { ok: false, reason: 'path must be absolute' }; + } + + if (forbidTraversal) { + // Detect `..` segments BEFORE normalization (otherwise they'd be silently + // collapsed). Handles both raw and decoded forms. + if (/(^|\/)\.\.(\/|$)/.test(rawPath) || /(^|\/)\.\.(\/|$)/.test(decoded)) { + return { ok: false, reason: 'path contains `..` traversal segment' }; + } + } + + const normalized = posixNormalize(decoded); + + if (policy.rootScope !== undefined) { + const root = posixNormalize(policy.rootScope); + if (!isPathInside(normalized, root)) { + return { ok: false, reason: 'path is outside root scope' }; + } + } + + if (policy.extra !== undefined && policy.extra(normalized) === 'reject') { + return { ok: false, reason: 'path rejected by policy.extra' }; + } + + return { ok: true, normalized }; +} diff --git a/packages/shade-files/src/server/rate-limiter.ts b/packages/shade-files/src/server/rate-limiter.ts new file mode 100644 index 0000000..586c5da --- /dev/null +++ b/packages/shade-files/src/server/rate-limiter.ts @@ -0,0 +1,157 @@ +import { FsRateLimitError, QuotaExceededError } from '../schemas/errors.js'; + +export interface RateLimitConfig { + /** Cap on op-tokens per minute per sender. Default 600. */ + maxOpsPerMinutePerSender?: number; + /** Cap on byte-tokens per hour per sender. Default 10 GiB. */ + maxBytesPerHourPerSender?: number; + /** Per-op token cost. Default 1 per op except write=5, delete=3. */ + opCost?: Partial> & { default?: number }; +} + +interface Bucket { + tokens: number; + capacity: number; + /** Tokens replenished per millisecond. */ + refillRate: number; + lastRefill: number; +} + +interface SenderBuckets { + ops: Bucket; + bytes: Bucket; +} + +/** + * Two-bucket per-sender rate limiter. Each operation consumes: + * - `opCost(op)` from the ops bucket. + * - `estimatedBytes` from the bytes bucket (post-call reconciliation possible). + * + * Rejection throws either `FsRateLimitError` (op-bucket) or `QuotaExceededError` + * (bytes-bucket), each with a `retryAfterMs` hint. + * + * Cancellation MUST call `release()` to return reserved tokens. + */ +export class RateLimiter { + private readonly opsCapacity: number; + private readonly opsRefillPerMs: number; + private readonly bytesCapacity: number; + private readonly bytesRefillPerMs: number; + private readonly opCost: Required>; + private readonly bySender = new Map(); + + constructor(config: RateLimitConfig = {}) { + this.opsCapacity = config.maxOpsPerMinutePerSender ?? 600; + this.opsRefillPerMs = this.opsCapacity / 60_000; + this.bytesCapacity = config.maxBytesPerHourPerSender ?? 10 * 1024 * 1024 * 1024; + this.bytesRefillPerMs = this.bytesCapacity / 3_600_000; + this.opCost = { + list: config.opCost?.list ?? 1, + stat: config.opCost?.stat ?? 1, + mkdir: config.opCost?.mkdir ?? 2, + delete: config.opCost?.delete ?? 3, + move: config.opCost?.move ?? 2, + read: config.opCost?.read ?? 1, + write: config.opCost?.write ?? 5, + getThumbnail: config.opCost?.getThumbnail ?? 2, + custom: config.opCost?.custom ?? 1, + default: config.opCost?.default ?? 1, + } as Required>; + } + + /** + * Acquire tokens for an operation. Throws if either bucket is empty; + * otherwise atomically deducts from both. `estimatedBytes` may be 0 for + * non-I/O ops. + */ + acquire(sender: string, op: string, estimatedBytes: number = 0): void { + const buckets = this.getBuckets(sender); + this.refill(buckets, Date.now()); + const cost = this.costFor(op); + + if (buckets.ops.tokens < cost) { + const need = cost - buckets.ops.tokens; + const retryMs = Math.ceil(need / this.opsRefillPerMs); + throw new FsRateLimitError(`op rate limit exceeded`, retryMs); + } + if (estimatedBytes > 0 && buckets.bytes.tokens < estimatedBytes) { + const need = estimatedBytes - buckets.bytes.tokens; + const retryMs = Math.ceil(need / this.bytesRefillPerMs); + throw new QuotaExceededError(`byte quota exceeded`, retryMs); + } + buckets.ops.tokens -= cost; + if (estimatedBytes > 0) buckets.bytes.tokens -= estimatedBytes; + } + + /** + * Reconcile after the actual byte cost is known. `actualBytes` may be + * higher or lower than the estimate; caps to capacity on release. + */ + reconcile(sender: string, estimatedBytes: number, actualBytes: number): void { + if (estimatedBytes === actualBytes) return; + const buckets = this.getBuckets(sender); + const delta = estimatedBytes - actualBytes; + if (delta > 0) { + buckets.bytes.tokens = Math.min(buckets.bytes.capacity, buckets.bytes.tokens + delta); + } else { + buckets.bytes.tokens = Math.max(0, buckets.bytes.tokens + delta); + } + } + + /** Return tokens to both buckets (cancellation cleanup). */ + release(sender: string, op: string, bytesReserved: number = 0): void { + const buckets = this.bySender.get(sender); + if (buckets === undefined) return; + const cost = this.costFor(op); + buckets.ops.tokens = Math.min(buckets.ops.capacity, buckets.ops.tokens + cost); + if (bytesReserved > 0) { + buckets.bytes.tokens = Math.min(buckets.bytes.capacity, buckets.bytes.tokens + bytesReserved); + } + } + + /** Available token snapshot — for tests/metrics. */ + snapshot(sender: string): { ops: number; bytes: number } | null { + const buckets = this.bySender.get(sender); + if (buckets === undefined) return null; + this.refill(buckets, Date.now()); + return { ops: buckets.ops.tokens, bytes: buckets.bytes.tokens }; + } + + private costFor(op: string): number { + const explicit = (this.opCost as Record)[op]; + return explicit ?? this.opCost.default; + } + + private getBuckets(sender: string): SenderBuckets { + let b = this.bySender.get(sender); + if (b !== undefined) return b; + const now = Date.now(); + b = { + ops: { + tokens: this.opsCapacity, + capacity: this.opsCapacity, + refillRate: this.opsRefillPerMs, + lastRefill: now, + }, + bytes: { + tokens: this.bytesCapacity, + capacity: this.bytesCapacity, + refillRate: this.bytesRefillPerMs, + lastRefill: now, + }, + }; + this.bySender.set(sender, b); + return b; + } + + private refill(buckets: SenderBuckets, now: number): void { + const refillBucket = (b: Bucket): void => { + const elapsed = now - b.lastRefill; + if (elapsed <= 0) return; + b.tokens = Math.min(b.capacity, b.tokens + elapsed * b.refillRate); + b.lastRefill = now; + }; + refillBucket(buckets.ops); + refillBucket(buckets.bytes); + } +} diff --git a/packages/shade-files/src/server/streams-bridge.ts b/packages/shade-files/src/server/streams-bridge.ts new file mode 100644 index 0000000..ac138a7 --- /dev/null +++ b/packages/shade-files/src/server/streams-bridge.ts @@ -0,0 +1,289 @@ +/** + * Server-side bridge between `@shade/files` content RPC ops and the + * `@shade/transfer` engine. + * + * Two responsibilities: + * 1. **Inbound writes (client → server, > 256 KiB).** When a client kicks + * `shade.upload(...)` with `userMetadata.shadeFilesWriteId = `, this + * bridge intercepts the corresponding `IncomingTransfer` from + * `shade.onIncomingTransfer`, **immediately** calls `accept(...)` (the + * transfer engine requires synchronous accept — it rejects chunks that + * arrive before accept), pipes the plaintext into a TransformStream, + * and parks the readable side. The `write` RPC handler then awaits + * `awaitWrite(writeId, ...)` to pick up the readable, hand it to the + * user handler, and observe the transfer's done() result. + * + * 2. **Outbound reads (server → client, > 256 KiB).** When the user-supplied + * `read` handler returns `{ kind: 'streams', stream, size, sha256 }`, + * this bridge calls `shade.upload(...)` with + * `userMetadata.shadeFilesReadStreamId = `. The id is then echoed + * back to the client in the `read` RPC response so the client-side + * bridge can subscribe to the matching incoming transfer. + */ +import type { IncomingTransfer, TransferHandle, TransferProgress } from '@shade/transfer'; +import { generateRequestId } from '../protocol/correlate.js'; +import { OperationTimeoutError } from '../schemas/errors.js'; + +/** Metadata key carried on inbound (client → server) write streams. */ +export const META_KEY_WRITE_ID = 'shadeFilesWriteId'; +/** Metadata key carried on outbound (server → client) read streams. */ +export const META_KEY_READ_STREAM_ID = 'shadeFilesReadStreamId'; + +/** + * Subset of `Shade` we depend on. Lets us unit-test without a real Shade + * runtime and keeps the bridge framework-agnostic. + */ +export interface StreamsBridgeShade { + upload(opts: import('@shade/transfer').TransferOptions): Promise; + onIncomingTransfer( + handler: (incoming: IncomingTransfer) => void | Promise, + ): Promise<() => void>; +} + +export interface ParkedWrite { + /** Sender address from the transfer engine. */ + from: string; + /** Plaintext stream. The bridge already accepted the transfer; just consume. */ + readable: ReadableStream; + /** Resolves when the transfer fully completes (sha256 available). */ + done: Promise<{ sha256: string; bytesSent: number }>; + /** Underlying transfer handle — for abort propagation. */ + handle: TransferHandle; + /** Unix epoch ms when the transfer arrived. Used for stale-eviction. */ + arrivedAt: number; +} + +export interface AwaitWriteOptions { + /** Sender address — must match `parked.from` for delivery. */ + expectedFrom: string; + /** AbortSignal from the OpContext (cancellation/timeout). */ + signal: AbortSignal; + /** Hard deadline (ms from now). Default 60_000. */ + timeoutMs?: number; +} + +export interface ServerStreamsBridge { + /** + * Wait for an inbound transfer carrying `userMetadata.shadeFilesWriteId + * === writeId` from `expectedFrom`. Resolves with the parked entry whose + * `readable` can be consumed by the user handler. + * + * If the transfer has already arrived (race: stream-init beat the RPC), + * resolves immediately. Otherwise awaits up to `timeoutMs`. + */ + awaitWrite(writeId: string, opts: AwaitWriteOptions): Promise; + + /** + * Initiate an outbound transfer to `peer` carrying the given stream. + * Stamps `userMetadata.shadeFilesReadStreamId = `. Returns the + * new streamId so the caller can echo it back to the client over RPC. + */ + initiateRead(opts: { + peer: string; + stream: ReadableStream; + size: number; + contentType?: string; + name?: string; + signal?: AbortSignal; + onProgress?: (p: TransferProgress) => void; + }): Promise<{ readStreamId: string; handle: TransferHandle }>; + + /** Tear down the incoming-transfer subscription and reject all pending awaits. */ + destroy(): Promise; +} + +interface PendingWaiter { + resolve: (parked: ParkedWrite) => void; + reject: (err: unknown) => void; + expectedFrom: string; + timer: ReturnType | null; + abortListener: (() => void) | null; + signal: AbortSignal; +} + +export interface CreateServerStreamsBridgeOptions { + /** Default deadline for `awaitWrite` if the caller doesn't supply one. */ + defaultAwaitWriteTimeoutMs?: number; + /** How long to retain a parked transfer waiting for its RPC. Default 60_000. */ + parkedWriteTtlMs?: number; +} + +export async function createServerStreamsBridge( + shade: StreamsBridgeShade, + options: CreateServerStreamsBridgeOptions = {}, +): Promise { + const parkedWriteTtlMs = options.parkedWriteTtlMs ?? 60_000; + const defaultAwaitTimeoutMs = options.defaultAwaitWriteTimeoutMs ?? 60_000; + + const parked = new Map(); + const waiters = new Map(); + let destroyed = false; + + const unsubscribeIncoming = await shade.onIncomingTransfer(async (incoming) => { + const writeId = incoming.metadata.userMetadata?.[META_KEY_WRITE_ID]; + if (writeId === undefined) return; // not a files-bridge transfer; ignore + + // Accept synchronously into a TransformStream so the engine can deliver + // chunks immediately. The readable side is parked for the matching RPC. + const ts = new TransformStream(); + let handle: TransferHandle; + try { + handle = await incoming.accept({ + output: { kind: 'pipe', pipeTo: ts.writable }, + }); + } catch (err) { + // Accept failed — likely double-init or shutdown. Drop the parked + // entry; the awaiting RPC will time out. + console.error('[shade-files streams-bridge] accept failed:', err); + return; + } + + const arrived: ParkedWrite = { + from: incoming.from, + readable: ts.readable, + done: handle.done().then((r) => ({ sha256: r.sha256, bytesSent: r.bytesSent })), + handle, + arrivedAt: Date.now(), + }; + arrived.done.catch(() => { + /* swallow until consumer awaits */ + }); + + const waiter = waiters.get(writeId); + if (waiter !== undefined) { + waiters.delete(writeId); + cleanupWaiter(waiter); + if (incoming.from !== waiter.expectedFrom) { + void handle.abort('sender-mismatch').catch(() => undefined); + waiter.reject( + new Error( + `streams-bridge: writeId=${writeId} delivered by ${incoming.from}, expected ${waiter.expectedFrom}`, + ), + ); + return; + } + waiter.resolve(arrived); + return; + } + + // No waiter yet — park. + parked.set(writeId, arrived); + setTimeout(() => { + const stale = parked.get(writeId); + if (stale === arrived) { + parked.delete(writeId); + void handle.abort('rpc-timeout').catch(() => undefined); + } + }, parkedWriteTtlMs).unref?.(); + }); + + function cleanupWaiter(w: PendingWaiter): void { + if (w.timer !== null) clearTimeout(w.timer); + if (w.abortListener !== null) { + w.signal.removeEventListener('abort', w.abortListener); + } + } + + return { + async awaitWrite(writeId, opts) { + if (destroyed) throw new Error('streams-bridge: destroyed'); + // 1. Already parked? + const ready = parked.get(writeId); + if (ready !== undefined) { + parked.delete(writeId); + if (ready.from !== opts.expectedFrom) { + void ready.handle.abort('sender-mismatch').catch(() => undefined); + throw new Error( + `streams-bridge: writeId=${writeId} delivered by ${ready.from}, expected ${opts.expectedFrom}`, + ); + } + return ready; + } + + if (waiters.has(writeId)) { + throw new Error(`streams-bridge: writeId=${writeId} already awaited`); + } + + // 2. Park a waiter + const timeoutMs = opts.timeoutMs ?? defaultAwaitTimeoutMs; + return await new Promise((resolve, reject) => { + const w: PendingWaiter = { + resolve, + reject, + expectedFrom: opts.expectedFrom, + timer: null, + abortListener: null, + signal: opts.signal, + }; + w.timer = setTimeout(() => { + if (waiters.get(writeId) === w) { + waiters.delete(writeId); + cleanupWaiter(w); + reject(new OperationTimeoutError(`streams-bridge: writeId=${writeId} timed out after ${timeoutMs}ms`)); + } + }, timeoutMs); + if (opts.signal.aborted) { + cleanupWaiter(w); + reject(opts.signal.reason ?? new Error('aborted before await')); + return; + } + const onAbort = (): void => { + if (waiters.get(writeId) === w) { + waiters.delete(writeId); + cleanupWaiter(w); + reject(opts.signal.reason ?? new Error('aborted')); + } + }; + w.abortListener = onAbort; + opts.signal.addEventListener('abort', onAbort, { once: true }); + waiters.set(writeId, w); + }); + }, + + async initiateRead(opts) { + if (destroyed) throw new Error('streams-bridge: destroyed'); + const readStreamId = generateRequestId(); + const transferOpts: import('@shade/transfer').TransferOptions = { + to: opts.peer, + input: opts.stream, + metadata: { + ...(opts.name !== undefined ? { name: opts.name } : {}), + ...(opts.contentType !== undefined ? { contentType: opts.contentType } : {}), + sizeBytes: opts.size, + userMetadata: { [META_KEY_READ_STREAM_ID]: readStreamId }, + }, + ...(opts.signal !== undefined ? { signal: opts.signal } : {}), + ...(opts.onProgress !== undefined ? { onProgress: opts.onProgress } : {}), + }; + const handle = await shade.upload(transferOpts); + // The dispatcher returns the response immediately and doesn't itself + // await this handle — the receiver awaits its own handle.done() on + // the inbound side. Attach a no-op catch so engine teardown during + // an in-flight outbound transfer doesn't surface as an unhandled + // rejection in test environments. + handle.done().catch(() => undefined); + return { readStreamId, handle }; + }, + + async destroy() { + if (destroyed) return; + destroyed = true; + unsubscribeIncoming(); + // Reject all waiting RPCs + for (const w of waiters.values()) { + cleanupWaiter(w); + w.reject(new Error('streams-bridge: destroyed')); + } + waiters.clear(); + // Abort all parked transfers + for (const p of parked.values()) { + try { + await p.handle.abort('bridge-destroyed'); + } catch { + /* swallow */ + } + } + parked.clear(); + }, + }; +} diff --git a/packages/shade-files/src/server/thumbnail.ts b/packages/shade-files/src/server/thumbnail.ts new file mode 100644 index 0000000..4032987 --- /dev/null +++ b/packages/shade-files/src/server/thumbnail.ts @@ -0,0 +1,61 @@ +/** + * Thumbnail format-hardening helper. + * + * Magic-byte verification for the three formats `@shade/files` advertises + * over the wire (`png`, `webp`, `jpeg`). Apps producing thumbnails outside + * the dispatcher (e.g. precomputing on disk) can call `assertThumbnailFormat` + * to fail-fast on mislabelled bytes before they reach the network. + * + * The same magic-byte table is used inside `io-adapters.ts:adaptThumbnailResult` + * to enforce the invariant on every outbound thumbnail. + */ + +import { ConflictError } from '../schemas/errors.js'; + +const PNG_MAGIC = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const JPEG_MAGIC = new Uint8Array([0xff, 0xd8, 0xff]); +const WEBP_RIFF = new Uint8Array([0x52, 0x49, 0x46, 0x46]); +const WEBP_HEAD = new Uint8Array([0x57, 0x45, 0x42, 0x50]); + +export type ThumbnailFormat = 'png' | 'webp' | 'jpeg'; + +/** + * Verify the bytes look like the declared format. Returns boolean — does + * not throw. For a throwing variant, use `assertThumbnailFormat`. + */ +export function isThumbnailFormat(bytes: Uint8Array, format: ThumbnailFormat): boolean { + switch (format) { + case 'png': + return startsWith(bytes, PNG_MAGIC); + case 'jpeg': + return startsWith(bytes, JPEG_MAGIC); + case 'webp': { + if (bytes.byteLength < 12) return false; + if (!startsWith(bytes, WEBP_RIFF)) return false; + for (let i = 0; i < 4; i++) { + if (bytes[8 + i] !== WEBP_HEAD[i]) return false; + } + return true; + } + } +} + +/** Throws `ConflictError` if `bytes` doesn't match the declared `format`. */ +export function assertThumbnailFormat( + bytes: Uint8Array, + format: ThumbnailFormat, +): void { + if (!isThumbnailFormat(bytes, format)) { + throw new ConflictError( + `thumbnail bytes do not match declared format=${format}`, + ); + } +} + +function startsWith(bytes: Uint8Array, prefix: Uint8Array): boolean { + if (bytes.byteLength < prefix.byteLength) return false; + for (let i = 0; i < prefix.byteLength; i++) { + if (bytes[i] !== prefix[i]) return false; + } + return true; +} diff --git a/packages/shade-files/src/utils/path.ts b/packages/shade-files/src/utils/path.ts new file mode 100644 index 0000000..d8892c6 --- /dev/null +++ b/packages/shade-files/src/utils/path.ts @@ -0,0 +1,86 @@ +/** + * Minimal POSIX path utilities — works in browser and Bun without depending + * on Node's built-in `node:path`. Only handles forward-slash absolute paths. + */ + +/** + * Normalize a POSIX absolute path: collapse `//`, resolve `.` and `..`, + * preserve trailing slash semantics. Returns `'/'` for `''`/`'/'` input. + * + * Throws nothing on invalid input — sanitization is the caller's job + * (`validatePath` in `server/path-policy.ts` does that). + */ +export function posixNormalize(path: string): string { + if (path === '' || path === '/') return '/'; + const isAbs = path.startsWith('/'); + const segments = path.split('/'); + const stack: string[] = []; + for (const seg of segments) { + if (seg === '' || seg === '.') continue; + if (seg === '..') { + if (stack.length > 0 && stack[stack.length - 1] !== '..') { + stack.pop(); + } else if (!isAbs) { + stack.push('..'); + } + // For absolute paths, `..` past root is silently dropped. + continue; + } + stack.push(seg); + } + const joined = stack.join('/'); + if (isAbs) return '/' + joined; + return joined === '' ? '.' : joined; +} + +/** Join path segments with POSIX semantics. */ +export function posixJoin(...parts: string[]): string { + if (parts.length === 0) return '.'; + const joined = parts.filter((p) => p !== '').join('/'); + return posixNormalize(joined); +} + +/** Return the directory portion of a path, or `'/'` for root. */ +export function posixDirname(path: string): string { + const normalized = posixNormalize(path); + if (normalized === '/') return '/'; + const lastSlash = normalized.lastIndexOf('/'); + if (lastSlash === 0) return '/'; + if (lastSlash === -1) return '.'; + return normalized.slice(0, lastSlash); +} + +/** Return the file/dir name (last component). */ +export function posixBasename(path: string): string { + const normalized = posixNormalize(path); + if (normalized === '/') return ''; + const lastSlash = normalized.lastIndexOf('/'); + return lastSlash === -1 ? normalized : normalized.slice(lastSlash + 1); +} + +/** + * Decode percent-escapes BEFORE security checks so attackers can't + * smuggle `..` via `%2e%2e`. Decodes UTF-8 sequences where possible; + * leaves invalid sequences untouched (the caller still rejects them). + */ +export function decodePercentEscapes(s: string): string { + if (!s.includes('%')) return s; + try { + return decodeURIComponent(s); + } catch { + // Invalid percent-encoding — leave as-is so policy rejects on the raw form. + return s; + } +} + +/** + * Predicate: does `child` lie under `root` after both are normalized? + * Both must be absolute. `root` is treated as a directory boundary. + */ +export function isPathInside(child: string, root: string): boolean { + const c = posixNormalize(child); + const r = posixNormalize(root); + if (r === '/') return c.startsWith('/'); + // Ensure boundary alignment so `/foo` is NOT inside `/foobar`. + return c === r || c.startsWith(r + '/'); +} diff --git a/packages/shade-files/tests/integration/custom-op.test.ts b/packages/shade-files/tests/integration/custom-op.test.ts new file mode 100644 index 0000000..6501f1e --- /dev/null +++ b/packages/shade-files/tests/integration/custom-op.test.ts @@ -0,0 +1,77 @@ +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { z } from 'zod'; +import { setupFileRig, type FileTestRig } from './helpers/rig.js'; +import { CustomOpRejectedError, NotImplementedError } from '../../src/index.js'; + +// Module augmentation: register a typed custom op for the test. +declare module '../../src/index.js' { + interface CustomOpsMap { + 'test.echo': { args: { message: string }; response: { echoed: string } }; + 'test.add': { args: { a: number; b: number }; response: { sum: number } }; + } +} + +describe('Custom ops — registry + Zod validation + typed I/O', () => { + let rig: FileTestRig; + const callLog: string[] = []; + + beforeAll(async () => { + rig = await setupFileRig({ + custom: { + 'test.echo': { + args: z.object({ message: z.string().min(1).max(64) }), + response: z.object({ echoed: z.string() }), + handler: async (args, ctx) => { + callLog.push(`echo:${ctx.sender}:${args.message}`); + return { echoed: args.message.toUpperCase() }; + }, + }, + 'test.add': { + args: z.object({ a: z.number(), b: z.number() }), + response: z.object({ sum: z.number() }), + handler: async (args) => ({ sum: args.a + args.b }), + }, + 'test.bad-response': { + args: z.object({}), + response: z.object({ x: z.number() }), + // Returns wrong shape on purpose. + handler: async () => ({ y: 'not-a-number' }) as unknown as { x: number }, + }, + }, + }); + }); + + afterAll(async () => { + await rig.teardown(); + }); + + test('typed echo round-trips through registered Zod schemas', async () => { + const result = await rig.fs.custom('test.echo', { message: 'hello' }); + expect(result.echoed).toBe('HELLO'); + expect(callLog).toContain('echo:alice:hello'); + }); + + test('typed add', async () => { + const result = await rig.fs.custom('test.add', { a: 3, b: 4 }); + expect(result.sum).toBe(7); + }); + + test('invalid args (Zod-rejected payload) → InvalidArgsError', async () => { + await expect( + // message: '' violates min(1) — TypeScript still allows it since string + rig.fs.custom('test.echo', { message: '' }), + ).rejects.toThrow(); + }); + + test('unknown custom op name → NotImplementedError', async () => { + await expect( + rig.fs.custom('test.unknown' as never, {} as never), + ).rejects.toBeInstanceOf(NotImplementedError); + }); + + test('handler returns wrong shape → CustomOpRejectedError', async () => { + await expect( + rig.fs.custom('test.bad-response' as never, {} as never), + ).rejects.toBeInstanceOf(CustomOpRejectedError); + }); +}); diff --git a/packages/shade-files/tests/integration/download-directory.test.ts b/packages/shade-files/tests/integration/download-directory.test.ts new file mode 100644 index 0000000..0a5c15c --- /dev/null +++ b/packages/shade-files/tests/integration/download-directory.test.ts @@ -0,0 +1,210 @@ +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { sha256 } from '@noble/hashes/sha2.js'; +import { + NotFoundError, + downloadDirectory, + createMemoryDirectory, + walk, + type DirectoryHandleLike, + type FileEntry, +} from '../../src/index.js'; +import { setupFileRig, type FileTestRig } from './helpers/rig.js'; + +interface StoredFile { + bytes: Uint8Array; + contentType?: string; + sha256: string; +} + +function bytesToHex(b: Uint8Array): string { + return Array.from(b, (x) => x.toString(16).padStart(2, '0')).join(''); +} + +async function streamToBytes(s: ReadableStream): Promise { + const reader = s.getReader(); + const parts: Uint8Array[] = []; + let total = 0; + while (true) { + const { value, done } = await reader.read(); + if (done) break; + if (value === undefined) continue; + parts.push(value); + total += value.byteLength; + } + reader.releaseLock(); + const out = new Uint8Array(total); + let offset = 0; + for (const p of parts) { + out.set(p, offset); + offset += p.byteLength; + } + return out; +} + +function streamFromBytes(bytes: Uint8Array): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.enqueue(bytes); + controller.close(); + }, + }); +} + +describe('downloadDirectory — bulk download from remote', () => { + let rig: FileTestRig; + const blobs = new Map(); + const dirs = new Set(); + + beforeAll(async () => { + blobs.clear(); + dirs.clear(); + // Build remote tree: + // /src/ + // ├── small.txt ('hello world\n', 12 bytes) + // ├── img.bin (50 KiB random) + // └── nested/ + // ├── big.bin (400 KiB random) + // └── tiny.bin (3 bytes) + dirs.add('/'); + dirs.add('/src'); + dirs.add('/src/nested'); + const small = new TextEncoder().encode('hello world\n'); + const mid = new Uint8Array(50 * 1024); crypto.getRandomValues(mid); + const big = new Uint8Array(400 * 1024); crypto.getRandomValues(big); + const tiny = new Uint8Array([1, 2, 3]); + blobs.set('/src/small.txt', { bytes: small, sha256: bytesToHex(sha256(small)), contentType: 'text/plain' }); + blobs.set('/src/img.bin', { bytes: mid, sha256: bytesToHex(sha256(mid)) }); + blobs.set('/src/nested/big.bin', { bytes: big, sha256: bytesToHex(sha256(big)) }); + blobs.set('/src/nested/tiny.bin', { bytes: tiny, sha256: bytesToHex(sha256(tiny)) }); + + rig = await setupFileRig({ + list: async (ctx) => { + if (!dirs.has(ctx.path)) throw new NotFoundError(ctx.path); + const entries: FileEntry[] = []; + const dirPrefix = ctx.path === '/' ? '/' : ctx.path + '/'; + // Subdirs + for (const d of dirs) { + if (d === ctx.path) continue; + if (!d.startsWith(dirPrefix)) continue; + const rest = d.slice(dirPrefix.length); + if (rest.includes('/')) continue; + entries.push({ name: rest, kind: 'dir', size: 0, mtime: 0, metadata: {} }); + } + // Files + for (const [path, blob] of blobs) { + if (!path.startsWith(dirPrefix)) continue; + const rest = path.slice(dirPrefix.length); + if (rest.includes('/')) continue; + entries.push({ + name: rest, + kind: 'file', + size: blob.bytes.byteLength, + mtime: 0, + ...(blob.contentType !== undefined ? { contentType: blob.contentType } : {}), + metadata: {}, + }); + } + return { entries, hasMore: false }; + }, + read: async (ctx) => { + const blob = blobs.get(ctx.path); + if (blob === undefined) throw new NotFoundError(ctx.path); + if (blob.bytes.byteLength > 256 * 1024) { + return blob.contentType !== undefined + ? { + kind: 'streams', + stream: streamFromBytes(blob.bytes), + size: blob.bytes.byteLength, + sha256: blob.sha256, + contentType: blob.contentType, + } + : { + kind: 'streams', + stream: streamFromBytes(blob.bytes), + size: blob.bytes.byteLength, + sha256: blob.sha256, + }; + } + return blob.contentType !== undefined + ? { kind: 'inline', bytes: blob.bytes, sha256: blob.sha256, contentType: blob.contentType } + : { kind: 'inline', bytes: blob.bytes, sha256: blob.sha256 }; + }, + }); + }); + + afterAll(async () => { + await rig.teardown(); + }); + + test('downloads entire tree, sha256 matches per file', async () => { + const local = createMemoryDirectory('local'); + const handle = downloadDirectory(rig.fs, '/src', local); + const result = await handle.done(); + expect(result.filesDone).toBe(4); + expect(result.bytesDone).toBe(12 + 50 * 1024 + 400 * 1024 + 3); + + // Verify local tree contents + const downloadedFiles = new Map(); + async function dump(dir: DirectoryHandleLike, prefix: string): Promise { + for await (const [name, child] of dir.entries()) { + const path = prefix === '' ? name : `${prefix}/${name}`; + if (child.kind === 'directory') { + await dump(child as DirectoryHandleLike, path); + } else { + const file = await (child as { getFile: () => Promise<{ arrayBuffer: () => Promise }> }).getFile(); + downloadedFiles.set(path, new Uint8Array(await file.arrayBuffer())); + } + } + } + await dump(local, ''); + expect(downloadedFiles.size).toBe(4); + expect(downloadedFiles.has('small.txt')).toBe(true); + expect(downloadedFiles.has('img.bin')).toBe(true); + expect(downloadedFiles.has('nested/big.bin')).toBe(true); + expect(downloadedFiles.has('nested/tiny.bin')).toBe(true); + + expect(bytesToHex(sha256(downloadedFiles.get('nested/big.bin')!))).toBe( + blobs.get('/src/nested/big.bin')!.sha256, + ); + expect(bytesToHex(sha256(downloadedFiles.get('img.bin')!))).toBe( + blobs.get('/src/img.bin')!.sha256, + ); + }); + + test('aggregated progress events fire monotonically', async () => { + const local = createMemoryDirectory('local'); + const handle = downloadDirectory(rig.fs, '/src', local); + const progresses: { filesDone: number; bytesDone: number }[] = []; + (async () => { + for await (const ev of handle.events) { + if (ev.type === 'progress') { + progresses.push({ filesDone: ev.filesDone, bytesDone: ev.bytesDone }); + } + } + })().catch(() => undefined); + await handle.done(); + await new Promise((r) => setTimeout(r, 30)); + for (let i = 1; i < progresses.length; i++) { + expect(progresses[i]!.filesDone).toBeGreaterThanOrEqual(progresses[i - 1]!.filesDone); + expect(progresses[i]!.bytesDone).toBeGreaterThanOrEqual(progresses[i - 1]!.bytesDone); + } + }); + + test('aborts via handle.abort()', async () => { + const local = createMemoryDirectory('local'); + const handle = downloadDirectory(rig.fs, '/src', local); + setTimeout(() => void handle.abort('test-cancel'), 5); + await expect(handle.done()).rejects.toThrow(); + }); + + test('walk + downloadDirectory are consistent', async () => { + const local = createMemoryDirectory('local'); + const remoteFiles: string[] = []; + for await (const item of walk(rig.fs, '/src')) { + if (item.entry.kind === 'file') remoteFiles.push(item.relativePath); + } + const handle = downloadDirectory(rig.fs, '/src', local); + const result = await handle.done(); + expect(result.filesDone).toBe(remoteFiles.length); + }); +}); diff --git a/packages/shade-files/tests/integration/helpers/rig.ts b/packages/shade-files/tests/integration/helpers/rig.ts new file mode 100644 index 0000000..9fd1392 --- /dev/null +++ b/packages/shade-files/tests/integration/helpers/rig.ts @@ -0,0 +1,142 @@ +import { createShade, type Shade } from '@shade/sdk'; +import { + createPrekeyServer, + MemoryPrekeyStore, + PrekeyServerEvents, +} from '@shade/server'; +import { SubtleCryptoProvider } from '@shade/crypto-web'; +import { + PendingRpcRegistry, + ShadeFileRpcChannel, + attachClientRouting, + attachFileHandler, + createClientStreamsBridge, + createFileClient, + createFileHandler, + createServerStreamsBridge, + type ClientStreamsBridge, + type FileClient, + type FileHandler, + type FileHandlerConfig, + type ServerStreamsBridge, +} from '../../../src/index.js'; + +const crypto = new SubtleCryptoProvider(); + +export interface FileTestRig { + alice: Shade; + bob: Shade; + fs: FileClient; + bobHandler: FileHandler; + /** Server-side streams bridge (Bob). */ + bobStreamsBridge: ServerStreamsBridge; + /** Client-side streams bridge (Alice). */ + aliceStreamsBridge: ClientStreamsBridge; + /** Tear everything down (kills servers, shuts down shades). */ + teardown(): Promise; +} + +/** + * Setup options. + * + * Defaults to wiring streams-bridges on both sides so content I/O tests + * (`read-write-streams.test.ts`) work transparently. Pass `withStreams: false` + * to skip — useful for the legacy `std-ops` tests that don't need them. + */ +export interface SetupRigOptions { + withStreams?: boolean; +} + +export async function setupFileRig( + bobConfig: Omit, + options: SetupRigOptions = {}, +): Promise { + const withStreams = options.withStreams ?? true; + // 1. Prekey server + const prekeyEvents = new PrekeyServerEvents(); + const prekey = createPrekeyServer({ + crypto, + store: new MemoryPrekeyStore(), + disableRateLimit: true, + events: prekeyEvents, + }); + const prekeyServer = Bun.serve({ port: 0, fetch: prekey.fetch }); + const prekeyUrl = `http://localhost:${prekeyServer.port}`; + + // 2. Two Shades + const alice = await createShade({ prekeyServer: prekeyUrl, address: 'alice' }); + const bob = await createShade({ prekeyServer: prekeyUrl, address: 'bob' }); + + // 3. BOTH sides need a transferRoute mounted because file-RPC is + // request/response — Bob's reply must reach Alice's HTTP endpoint + // just as Alice's request reached Bob's. + let bobBaseUrl = ''; + let aliceBaseUrl = ''; + + alice.configureTransfers({ + resolveBaseUrl: async (peer) => { + if (peer === 'bob') return bobBaseUrl; + throw new Error(`alice: unknown peer ${peer}`); + }, + }); + bob.configureTransfers({ + resolveBaseUrl: async (peer) => { + if (peer === 'alice') return aliceBaseUrl; + throw new Error(`bob: unknown peer ${peer}`); + }, + }); + + const bobApp = await bob.transferRoute(); + const aliceApp = await alice.transferRoute(); + const bobServer = Bun.serve({ port: 0, fetch: bobApp.fetch }); + const aliceServer = Bun.serve({ port: 0, fetch: aliceApp.fetch }); + bobBaseUrl = `http://localhost:${bobServer.port}`; + aliceBaseUrl = `http://localhost:${aliceServer.port}`; + + // 4. Streams bridges (both sides) — required for content I/O > 256 KiB. + let bobStreamsBridge: ServerStreamsBridge | undefined; + let aliceStreamsBridge: ClientStreamsBridge | undefined; + if (withStreams) { + bobStreamsBridge = await createServerStreamsBridge(bob); + aliceStreamsBridge = await createClientStreamsBridge(alice); + } + + // 5. Bob: file handler + channel + const bobChannel = new ShadeFileRpcChannel(bob); + const fullBobConfig: FileHandlerConfig = { + ...bobConfig, + ...(bobStreamsBridge !== undefined ? { streamsBridge: bobStreamsBridge } : {}), + }; + const bobHandler = createFileHandler(bob, fullBobConfig); + attachFileHandler(bobChannel, bobHandler); + + // 6. Alice: client + channel + pending registry + const aliceChannel = new ShadeFileRpcChannel(alice); + const alicePending = new PendingRpcRegistry(); + attachClientRouting(aliceChannel, alicePending); + const fs = createFileClient(alice, aliceChannel, alicePending, 'bob', { + defaultTimeoutMs: 5000, + ...(aliceStreamsBridge !== undefined ? { streamsBridge: aliceStreamsBridge } : {}), + }); + + return { + alice, + bob, + fs, + bobHandler, + bobStreamsBridge: bobStreamsBridge as ServerStreamsBridge, + aliceStreamsBridge: aliceStreamsBridge as ClientStreamsBridge, + async teardown() { + bobChannel.destroy(); + aliceChannel.destroy(); + bobHandler.destroy(); + if (bobStreamsBridge !== undefined) await bobStreamsBridge.destroy(); + if (aliceStreamsBridge !== undefined) await aliceStreamsBridge.destroy(); + await alice.shutdown(); + await bob.shutdown(); + bobServer.stop(); + aliceServer.stop(); + prekeyServer.stop(); + }, + }; +} diff --git a/packages/shade-files/tests/integration/metrics.test.ts b/packages/shade-files/tests/integration/metrics.test.ts new file mode 100644 index 0000000..354bab7 --- /dev/null +++ b/packages/shade-files/tests/integration/metrics.test.ts @@ -0,0 +1,72 @@ +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { setupFileRig, type FileTestRig } from './helpers/rig.js'; +import { + METRIC_OP_DURATION_MS, + METRIC_OP_TOTAL, + METRIC_RATE_LIMIT_REJECT_TOTAL, + type MetricSink, + type MetricTags, + type FileEntry, +} from '../../src/index.js'; + +interface MetricEvent { + name: string; + value: number; + tags: MetricTags; +} + +describe('Metrics — onMetric on success', () => { + let rig: FileTestRig; + const events: MetricEvent[] = []; + const sink: MetricSink = (name, value, tags) => events.push({ name, value, tags }); + + beforeAll(async () => { + rig = await setupFileRig({ + onMetric: sink, + list: async () => ({ entries: [], hasMore: false }), + }); + }); + afterAll(async () => { await rig.teardown(); }); + + test('emits op_total + op_duration_ms with result=ok on success', async () => { + events.length = 0; + await rig.fs.list('/'); + const totals = events.filter((e) => e.name === METRIC_OP_TOTAL); + expect(totals.length).toBeGreaterThanOrEqual(1); + expect(totals[0]!.tags.result).toBe('ok'); + expect(totals[0]!.tags.op).toBe('list'); + const durations = events.filter((e) => e.name === METRIC_OP_DURATION_MS); + expect(durations.length).toBeGreaterThanOrEqual(1); + expect(durations[0]!.value).toBeGreaterThanOrEqual(0); + }); +}); + +describe('Metrics — onMetric rate-limit reject', () => { + let rig: FileTestRig; + const events: MetricEvent[] = []; + const sink: MetricSink = (name, value, tags) => events.push({ name, value, tags }); + + beforeAll(async () => { + rig = await setupFileRig({ + onMetric: sink, + rateLimits: { maxOpsPerMinutePerSender: 3 }, + stat: async (_ctx) => { + const e: FileEntry = { name: 'x', kind: 'file', size: 1, mtime: 0, metadata: {} }; + return e; + }, + }); + }); + afterAll(async () => { await rig.teardown(); }); + + test('emits rate_limit_reject_total when capacity exhausted', async () => { + events.length = 0; + // Cap is 3; first 3 stats succeed, 4th rejected. + await rig.fs.stat('/x'); + await rig.fs.stat('/x'); + await rig.fs.stat('/x'); + await expect(rig.fs.stat('/x')).rejects.toThrow(); + const rejects = events.filter((e) => e.name === METRIC_RATE_LIMIT_REJECT_TOTAL); + expect(rejects.length).toBeGreaterThanOrEqual(1); + expect(rejects[0]!.tags.op).toBe('stat'); + }); +}); diff --git a/packages/shade-files/tests/integration/read-write-inline.test.ts b/packages/shade-files/tests/integration/read-write-inline.test.ts new file mode 100644 index 0000000..91f7572 --- /dev/null +++ b/packages/shade-files/tests/integration/read-write-inline.test.ts @@ -0,0 +1,138 @@ +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { sha256 } from '@noble/hashes/sha2.js'; +import { NotFoundError, type FileEntry } from '../../src/index.js'; +import type { UserReadResult, UserWriteArgs, WriteResult } from '../../src/server/io-types.js'; +import type { OpContext } from '../../src/server/handler-context.js'; +import { setupFileRig, type FileTestRig } from './helpers/rig.js'; + +interface StoredBlob { + bytes: Uint8Array; + contentType?: string; + sha256: string; + mtime: number; +} + +function bytesToHex(b: Uint8Array): string { + return Array.from(b, (x) => x.toString(16).padStart(2, '0')).join(''); +} + +describe('Content I/O — inline read/write E2E', () => { + let rig: FileTestRig; + const blobs = new Map(); + + beforeAll(async () => { + blobs.clear(); + rig = await setupFileRig({ + read: async (ctx: OpContext<{ path: string }>): Promise => { + const blob = blobs.get(ctx.path); + if (blob === undefined) throw new NotFoundError(ctx.path); + return blob.contentType !== undefined + ? { kind: 'inline', bytes: blob.bytes, sha256: blob.sha256, contentType: blob.contentType } + : { kind: 'inline', bytes: blob.bytes, sha256: blob.sha256 }; + }, + write: async (ctx: OpContext): Promise => { + const args = ctx.args; + if (args.content.kind !== 'inline') { + throw new Error('expected inline content for this test'); + } + if (blobs.has(args.path) && !args.overwrite) { + throw new Error('exists'); + } + const stored: StoredBlob = { + bytes: args.content.bytes, + ...(args.contentType !== undefined ? { contentType: args.contentType } : {}), + sha256: args.content.sha256, + mtime: Date.now(), + }; + blobs.set(args.path, stored); + const entry: FileEntry = { + name: args.path.split('/').filter(Boolean).pop() ?? '', + kind: 'file', + size: args.content.size, + mtime: stored.mtime, + ...(args.contentType !== undefined ? { contentType: args.contentType } : {}), + metadata: { sha256: args.content.sha256 }, + }; + return { entry }; + }, + }); + }); + + afterAll(async () => { + await rig.teardown(); + }); + + test('write 1 KiB inline → read it back, sha256 matches', async () => { + const data = new Uint8Array(1024); + for (let i = 0; i < data.length; i++) data[i] = (i * 7) & 0xff; + const expectedSha = bytesToHex(sha256(data)); + + const writeResult = await rig.fs.write('/small.bin', data, { contentType: 'application/octet-stream' }); + expect(writeResult.entry.size).toBe(1024); + expect(writeResult.entry.metadata.sha256).toBe(expectedSha); + + const readResult = await rig.fs.read('/small.bin'); + expect(readResult.kind).toBe('inline'); + if (readResult.kind === 'inline') { + expect(readResult.bytes.byteLength).toBe(1024); + expect(readResult.sha256).toBe(expectedSha); + expect(readResult.contentType).toBe('application/octet-stream'); + expect(Array.from(readResult.bytes)).toEqual(Array.from(data)); + } + }); + + test('write 100 KiB inline → read it back, sha256 matches', async () => { + const data = new Uint8Array(100 * 1024); + crypto.getRandomValues(data); + const expectedSha = bytesToHex(sha256(data)); + + await rig.fs.write('/big.bin', data); + const readResult = await rig.fs.read('/big.bin'); + expect(readResult.kind).toBe('inline'); + if (readResult.kind === 'inline') { + expect(readResult.bytes.byteLength).toBe(100 * 1024); + expect(readResult.sha256).toBe(expectedSha); + expect(Array.from(readResult.bytes)).toEqual(Array.from(data)); + } + }); + + test('overwrite without flag → server-defined error; with flag → succeeds', async () => { + const a = new Uint8Array([1, 2, 3]); + const b = new Uint8Array([4, 5, 6]); + await rig.fs.write('/dup.bin', a); + await expect(rig.fs.write('/dup.bin', b)).rejects.toThrow(); + await rig.fs.write('/dup.bin', b, { overwrite: true }); + const out = await rig.fs.read('/dup.bin'); + expect(out.kind).toBe('inline'); + if (out.kind === 'inline') { + expect(Array.from(out.bytes)).toEqual([4, 5, 6]); + } + }); + + test('write Blob input → inferred contentType, round-trips', async () => { + const blob = new Blob([new Uint8Array([0xde, 0xad, 0xbe, 0xef])], { type: 'image/png' }); + await rig.fs.write('/blobby.png', blob); + const out = await rig.fs.read('/blobby.png'); + expect(out.kind).toBe('inline'); + if (out.kind === 'inline') { + expect(out.contentType).toBe('image/png'); + expect(Array.from(out.bytes)).toEqual([0xde, 0xad, 0xbe, 0xef]); + } + }); + + test('read non-existent → NotFoundError', async () => { + await expect(rig.fs.read('/missing.bin')).rejects.toThrow(); + }); + + test('inline path also handles 256 KiB exactly (boundary)', async () => { + const data = new Uint8Array(256 * 1024); + crypto.getRandomValues(data); + const expectedSha = bytesToHex(sha256(data)); + await rig.fs.write('/boundary.bin', data); + const out = await rig.fs.read('/boundary.bin'); + expect(out.kind).toBe('inline'); + if (out.kind === 'inline') { + expect(out.sha256).toBe(expectedSha); + } + }); +}); diff --git a/packages/shade-files/tests/integration/read-write-streams.test.ts b/packages/shade-files/tests/integration/read-write-streams.test.ts new file mode 100644 index 0000000..6b7595d --- /dev/null +++ b/packages/shade-files/tests/integration/read-write-streams.test.ts @@ -0,0 +1,175 @@ +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { sha256 } from '@noble/hashes/sha2.js'; +import { NotFoundError, type FileEntry } from '../../src/index.js'; +import { setupFileRig, type FileTestRig } from './helpers/rig.js'; + +interface StoredBlob { + bytes: Uint8Array; + contentType?: string; + sha256: string; +} + +function bytesToHex(b: Uint8Array): string { + return Array.from(b, (x) => x.toString(16).padStart(2, '0')).join(''); +} + +async function streamToBytes(s: ReadableStream): Promise { + const reader = s.getReader(); + const parts: Uint8Array[] = []; + let total = 0; + while (true) { + const { value, done } = await reader.read(); + if (done) break; + if (value === undefined) continue; + parts.push(value); + total += value.byteLength; + } + reader.releaseLock(); + const out = new Uint8Array(total); + let offset = 0; + for (const p of parts) { + out.set(p, offset); + offset += p.byteLength; + } + return out; +} + +function streamFromBytes(bytes: Uint8Array): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.enqueue(bytes); + controller.close(); + }, + }); +} + +describe('Content I/O — streamed read/write E2E (>256 KiB)', () => { + let rig: FileTestRig; + const blobs = new Map(); + + beforeAll(async () => { + blobs.clear(); + rig = await setupFileRig({ + read: async (ctx) => { + const blob = blobs.get(ctx.path); + if (blob === undefined) throw new NotFoundError(ctx.path); + // Return as streams when blob ≥ 256 KiB. + if (blob.bytes.byteLength > 256 * 1024) { + return blob.contentType !== undefined + ? { + kind: 'streams', + stream: streamFromBytes(blob.bytes), + size: blob.bytes.byteLength, + sha256: blob.sha256, + contentType: blob.contentType, + } + : { + kind: 'streams', + stream: streamFromBytes(blob.bytes), + size: blob.bytes.byteLength, + sha256: blob.sha256, + }; + } + return blob.contentType !== undefined + ? { kind: 'inline', bytes: blob.bytes, sha256: blob.sha256, contentType: blob.contentType } + : { kind: 'inline', bytes: blob.bytes, sha256: blob.sha256 }; + }, + write: async (ctx) => { + const args = ctx.args; + let bytes: Uint8Array; + let resolvedSha: string; + if (args.content.kind === 'inline') { + bytes = args.content.bytes; + resolvedSha = args.content.sha256; + } else { + bytes = await streamToBytes(args.content.stream); + resolvedSha = await args.content.sha256; + if (bytes.byteLength !== args.content.size) { + throw new Error(`stream produced ${bytes.byteLength} bytes; declared ${args.content.size}`); + } + } + const stored: StoredBlob = { + bytes, + ...(args.contentType !== undefined ? { contentType: args.contentType } : {}), + sha256: resolvedSha, + }; + blobs.set(args.path, stored); + const entry: FileEntry = { + name: args.path.split('/').filter(Boolean).pop() ?? '', + kind: 'file', + size: bytes.byteLength, + mtime: Date.now(), + ...(args.contentType !== undefined ? { contentType: args.contentType } : {}), + metadata: { sha256: resolvedSha }, + }; + return { entry }; + }, + }); + }); + + afterAll(async () => { + await rig.teardown(); + }); + + test('write 1 MiB streamed → server sees streams content + sha256 matches', async () => { + const data = new Uint8Array(1024 * 1024); + crypto.getRandomValues(data); + const expectedSha = bytesToHex(sha256(data)); + + const writeResult = await rig.fs.write('/big1mb.bin', data); + expect(writeResult.entry.size).toBe(data.byteLength); + expect(writeResult.entry.metadata.sha256).toBe(expectedSha); + }); + + test('read 1 MiB streamed → client gets streams output, drains correctly', async () => { + const out = await rig.fs.read('/big1mb.bin'); + expect(out.kind).toBe('streams'); + if (out.kind === 'streams') { + expect(out.size).toBe(1024 * 1024); + const drained = await streamToBytes(out.stream); + expect(drained.byteLength).toBe(1024 * 1024); + expect(bytesToHex(sha256(drained))).toBe(out.sha256); + await out.done(); + } + }); + + test('boundary 256 KiB + 1 → streams write + read round-trip', async () => { + const data = new Uint8Array(256 * 1024 + 1); + for (let i = 0; i < data.length; i++) data[i] = (i * 31) & 0xff; + const expectedSha = bytesToHex(sha256(data)); + + await rig.fs.write('/boundary-plus-one.bin', data); + const out = await rig.fs.read('/boundary-plus-one.bin'); + expect(out.kind).toBe('streams'); + if (out.kind === 'streams') { + expect(out.sha256).toBe(expectedSha); + const drained = await streamToBytes(out.stream); + expect(drained.byteLength).toBe(256 * 1024 + 1); + expect(Array.from(drained.slice(0, 4))).toEqual(Array.from(data.slice(0, 4))); + expect(Array.from(drained.slice(-4))).toEqual(Array.from(data.slice(-4))); + await out.done(); + } + }); + + test('write streams via { stream, size } wrapper → ok', async () => { + const data = new Uint8Array(500 * 1024); + crypto.getRandomValues(data); + const expectedSha = bytesToHex(sha256(data)); + const wrapped = { stream: streamFromBytes(data), size: data.byteLength, contentType: 'application/octet-stream' }; + const result = await rig.fs.write('/wrapped.bin', wrapped); + expect(result.entry.size).toBe(data.byteLength); + expect(result.entry.metadata.sha256).toBe(expectedSha); + expect(result.entry.contentType).toBe('application/octet-stream'); + }); + + test('write streams from Blob > 256 KiB → ok', async () => { + const data = new Uint8Array(400 * 1024); + crypto.getRandomValues(data); + const expectedSha = bytesToHex(sha256(data)); + const blob = new Blob([data], { type: 'image/png' }); + const result = await rig.fs.write('/blobby-big.png', blob); + expect(result.entry.size).toBe(data.byteLength); + expect(result.entry.metadata.sha256).toBe(expectedSha); + expect(result.entry.contentType).toBe('image/png'); + }); +}); diff --git a/packages/shade-files/tests/integration/sdk-namespace.test.ts b/packages/shade-files/tests/integration/sdk-namespace.test.ts new file mode 100644 index 0000000..0e157a5 --- /dev/null +++ b/packages/shade-files/tests/integration/sdk-namespace.test.ts @@ -0,0 +1,100 @@ +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { createShade, type Shade } from '@shade/sdk'; +import { createPrekeyServer, MemoryPrekeyStore, PrekeyServerEvents } from '@shade/server'; +import { SubtleCryptoProvider } from '@shade/crypto-web'; +import type { FileEntry } from '../../src/index.js'; + +const crypto = new SubtleCryptoProvider(); + +/** + * End-to-end test of `Shade.files` — the high-level SDK entrypoint. + * Verifies that `shade.files.serve(...)` and `shade.files.client(peer)` + * compose correctly and share a single channel + bridges per Shade. + */ +describe('Shade.files namespace — end-to-end via SDK getter', () => { + let alice: Shade; + let bob: Shade; + let prekeyServer: { stop(): void }; + let aliceServer: { stop(): void }; + let bobServer: { stop(): void }; + let stopBobFiles: (() => Promise) | null = null; + + beforeAll(async () => { + const prekey = createPrekeyServer({ + crypto, + store: new MemoryPrekeyStore(), + disableRateLimit: true, + events: new PrekeyServerEvents(), + }); + prekeyServer = Bun.serve({ port: 0, fetch: prekey.fetch }); + const prekeyUrl = `http://localhost:${(prekeyServer as unknown as { port: number }).port}`; + + alice = await createShade({ prekeyServer: prekeyUrl, address: 'alice' }); + bob = await createShade({ prekeyServer: prekeyUrl, address: 'bob' }); + + let aliceUrl = ''; + let bobUrl = ''; + alice.configureTransfers({ + resolveBaseUrl: async (peer) => { + if (peer === 'bob') return bobUrl; + throw new Error(`unknown peer: ${peer}`); + }, + }); + bob.configureTransfers({ + resolveBaseUrl: async (peer) => { + if (peer === 'alice') return aliceUrl; + throw new Error(`unknown peer: ${peer}`); + }, + }); + const aliceApp = await alice.transferRoute(); + const bobApp = await bob.transferRoute(); + aliceServer = Bun.serve({ port: 0, fetch: aliceApp.fetch }); + bobServer = Bun.serve({ port: 0, fetch: bobApp.fetch }); + aliceUrl = `http://localhost:${(aliceServer as unknown as { port: number }).port}`; + bobUrl = `http://localhost:${(bobServer as unknown as { port: number }).port}`; + }); + + afterAll(async () => { + if (stopBobFiles !== null) await stopBobFiles(); + await alice.files.destroy(); + await bob.files.destroy(); + await alice.shutdown(); + await bob.shutdown(); + aliceServer.stop(); + bobServer.stop(); + prekeyServer.stop(); + }); + + test('shade.files getter is memoized', () => { + const a = bob.files; + const b = bob.files; + expect(a).toBe(b); + }); + + test('serve() + client() round-trip stat through the SDK', async () => { + stopBobFiles = await bob.files.serve({ + stat: async (ctx) => { + const e: FileEntry = { + name: ctx.path.split('/').filter(Boolean).pop() ?? '', + kind: 'file', + size: 42, + mtime: 1234, + metadata: {}, + }; + return e; + }, + }); + + const fs = await alice.files.client('bob'); + const result = await fs.stat('/answer.txt'); + expect(result.name).toBe('answer.txt'); + expect(result.size).toBe(42); + expect(result.mtime).toBe(1234); + }); + + test('second serve() throws (one handler per Shade)', async () => { + await expect( + bob.files.serve({ stat: async () => ({ name: 'x', kind: 'file', size: 0, mtime: 0, metadata: {} }) }), + ).rejects.toThrow(/handler is already registered/); + }); +}); diff --git a/packages/shade-files/tests/integration/std-ops.test.ts b/packages/shade-files/tests/integration/std-ops.test.ts new file mode 100644 index 0000000..53918e4 --- /dev/null +++ b/packages/shade-files/tests/integration/std-ops.test.ts @@ -0,0 +1,199 @@ +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { + ConflictError, + IdempotencyConflictError, + NotFoundError, + PermissionDeniedError, + type FileEntry, +} from '../../src/index.js'; +import { setupFileRig, type FileTestRig } from './helpers/rig.js'; + +describe('Standard ops — list/stat/mkdir/delete/move E2E', () => { + let rig: FileTestRig; + // Simple in-memory backing store on Bob. + const tree = new Map(); + + beforeAll(async () => { + tree.clear(); + tree.set('/', { name: '', kind: 'dir', size: 0, mtime: 0, metadata: {} }); + tree.set('/foo', { name: 'foo', kind: 'dir', size: 0, mtime: 100, metadata: {} }); + tree.set('/foo/bar.txt', { + name: 'bar.txt', + kind: 'file', + size: 12, + mtime: 200, + contentType: 'text/plain', + metadata: {}, + }); + tree.set('/foo/baz.txt', { + name: 'baz.txt', + kind: 'file', + size: 5, + mtime: 300, + metadata: {}, + }); + + rig = await setupFileRig({ + list: async (ctx) => { + const dir = ctx.path; + const entries: FileEntry[] = []; + for (const [path, entry] of tree) { + if (path === dir) continue; + if (!path.startsWith(dir === '/' ? '/' : dir + '/')) continue; + const rest = path.slice(dir === '/' ? 1 : dir.length + 1); + if (rest.includes('/')) continue; + entries.push(entry); + } + return { entries, hasMore: false }; + }, + stat: async (ctx) => { + const e = tree.get(ctx.path); + if (e === undefined) throw new NotFoundError(`${ctx.path} not found`); + return e; + }, + mkdir: async (ctx) => { + if (tree.has(ctx.path)) throw new ConflictError(`${ctx.path} exists`); + const name = ctx.path.split('/').filter(Boolean).pop() ?? ''; + const entry: FileEntry = { + name, + kind: 'dir', + size: 0, + mtime: Date.now(), + metadata: {}, + }; + tree.set(ctx.path, entry); + return { entry }; + }, + delete: async (ctx) => { + if (!tree.has(ctx.path)) throw new NotFoundError(ctx.path); + let count = 0; + if (ctx.args.recursive) { + for (const path of [...tree.keys()]) { + if (path === ctx.path || path.startsWith(ctx.path + '/')) { + tree.delete(path); + count++; + } + } + } else { + tree.delete(ctx.path); + count = 1; + } + return { deletedCount: count }; + }, + move: async (ctx) => { + const src = ctx.args.src; + const dst = ctx.args.dst; + const e = tree.get(src); + if (e === undefined) throw new NotFoundError(src); + if (tree.has(dst) && !ctx.args.overwrite) { + throw new ConflictError(`${dst} exists`); + } + const newName = dst.split('/').filter(Boolean).pop() ?? e.name; + tree.delete(src); + tree.set(dst, { ...e, name: newName }); + return { entry: tree.get(dst)! }; + }, + }); + }); + + afterAll(async () => { + await rig.teardown(); + }); + + test('list /', async () => { + const page = await rig.fs.list('/'); + expect(page.hasMore).toBe(false); + const names = page.entries.map((e) => e.name).sort(); + expect(names).toEqual(['foo']); + }); + + test('list /foo', async () => { + const page = await rig.fs.list('/foo'); + const names = page.entries.map((e) => e.name).sort(); + expect(names).toEqual(['bar.txt', 'baz.txt']); + }); + + test('stat existing file', async () => { + const e = await rig.fs.stat('/foo/bar.txt'); + expect(e.size).toBe(12); + expect(e.contentType).toBe('text/plain'); + }); + + test('stat missing → NotFoundError', async () => { + let caught: unknown = null; + try { + await rig.fs.stat('/no/such/file'); + } catch (err) { + caught = err; + } + expect(caught instanceof NotFoundError).toBe(true); + expect((caught as NotFoundError).payload.code).toBe('NOT_FOUND'); + }); + + test('mkdir creates a new directory', async () => { + const result = await rig.fs.mkdir('/created'); + expect(result.entry.kind).toBe('dir'); + expect(tree.has('/created')).toBe(true); + }); + + test('mkdir on existing path → ConflictError', async () => { + await expect(rig.fs.mkdir('/foo')).rejects.toBeInstanceOf(ConflictError); + }); + + test('delete file', async () => { + tree.set('/temp.txt', { name: 'temp.txt', kind: 'file', size: 0, mtime: 0, metadata: {} }); + const r = await rig.fs.delete('/temp.txt'); + expect(r.deletedCount).toBe(1); + expect(tree.has('/temp.txt')).toBe(false); + }); + + test('delete missing → NotFoundError', async () => { + await expect(rig.fs.delete('/doesnt-exist')).rejects.toBeInstanceOf(NotFoundError); + }); + + test('move file', async () => { + tree.set('/x.txt', { name: 'x.txt', kind: 'file', size: 0, mtime: 0, metadata: {} }); + const r = await rig.fs.move('/x.txt', '/y.txt'); + expect(r.entry.name).toBe('y.txt'); + expect(tree.has('/x.txt')).toBe(false); + expect(tree.has('/y.txt')).toBe(true); + }); + + test('move overwrite=false → ConflictError on collision', async () => { + tree.set('/a.txt', { name: 'a.txt', kind: 'file', size: 0, mtime: 0, metadata: {} }); + tree.set('/b.txt', { name: 'b.txt', kind: 'file', size: 0, mtime: 0, metadata: {} }); + await expect(rig.fs.move('/a.txt', '/b.txt')).rejects.toBeInstanceOf(ConflictError); + }); + + test('idempotent retry: same key + same args → cached response', async () => { + tree.set('/idem.txt', { name: 'idem.txt', kind: 'file', size: 0, mtime: 0, metadata: {} }); + const key = 'IdemKey1AaBbCcDdEeFfGg'; // 22 chars + const r1 = await rig.fs.delete('/idem.txt', { idempotencyKey: key }); + expect(r1.deletedCount).toBe(1); + // 2nd call with same key → same result without throwing NotFound + const r2 = await rig.fs.delete('/idem.txt', { idempotencyKey: key }); + expect(r2.deletedCount).toBe(1); + }); + + test('idempotency conflict: same key + different args', async () => { + tree.set('/c1.txt', { name: 'c1.txt', kind: 'file', size: 0, mtime: 0, metadata: {} }); + tree.set('/c2.txt', { name: 'c2.txt', kind: 'file', size: 0, mtime: 0, metadata: {} }); + const key = 'ConflictKeyAaBbCcDdEeF'; // 22 chars + await rig.fs.delete('/c1.txt', { idempotencyKey: key }); + await expect( + rig.fs.delete('/c2.txt', { idempotencyKey: key }), + ).rejects.toBeInstanceOf(IdempotencyConflictError); + }); + + test('path validation: traversal rejected by server', async () => { + await expect(rig.fs.list('/foo')).resolves.toBeDefined(); + // Client-side schema rejects traversal too — bypass via direct bad path: + await expect(rig.fs.stat('/foo/../etc')).rejects.toThrow(); + }); + + // We don't test PermissionDenied here (no beforeOp gate configured), + // but the type is referenced to ensure it's exported properly. + test('PermissionDeniedError is exported', () => { + expect(PermissionDeniedError).toBeDefined(); + }); +}); diff --git a/packages/shade-files/tests/integration/thumbnail.test.ts b/packages/shade-files/tests/integration/thumbnail.test.ts new file mode 100644 index 0000000..e0409a4 --- /dev/null +++ b/packages/shade-files/tests/integration/thumbnail.test.ts @@ -0,0 +1,123 @@ +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { sha256 } from '@noble/hashes/sha2.js'; +import { ConflictError, NotFoundError } from '../../src/index.js'; +import { setupFileRig, type FileTestRig } from './helpers/rig.js'; + +function bytesToHex(b: Uint8Array): string { + return Array.from(b, (x) => x.toString(16).padStart(2, '0')).join(''); +} + +// Minimal valid PNG: 8-byte signature + IHDR + zero-byte IDAT + IEND. We +// use realistic magic bytes so the format-hardening check accepts it. +function tinyPng(): Uint8Array { + const sig = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + const filler = new Uint8Array(20); + const out = new Uint8Array(sig.length + filler.length); + out.set(sig, 0); + out.set(filler, sig.length); + return out; +} + +function tinyJpeg(): Uint8Array { + const head = new Uint8Array([0xff, 0xd8, 0xff, 0xe0]); + const filler = new Uint8Array(12); + const out = new Uint8Array(head.length + filler.length); + out.set(head, 0); + out.set(filler, head.length); + return out; +} + +function tinyWebp(): Uint8Array { + // 'RIFF' + sizeLE + 'WEBP' + 'VP8 ' (or 'VP8L' / 'VP8X') + const out = new Uint8Array(20); + out.set([0x52, 0x49, 0x46, 0x46], 0); // 'RIFF' + out.set([0x0c, 0x00, 0x00, 0x00], 4); // size little-endian (12 bytes follow) + out.set([0x57, 0x45, 0x42, 0x50], 8); // 'WEBP' + out.set([0x56, 0x50, 0x38, 0x20], 12); // 'VP8 ' + return out; +} + +describe('Content I/O — getThumbnail E2E with format hardening', () => { + let rig: FileTestRig; + // Map: path → (size → { bytes, format }) + const thumbnails = new Map>(); + + beforeAll(async () => { + thumbnails.clear(); + thumbnails.set( + '/photo.png', + new Map([ + [64, { bytes: tinyPng(), format: 'png' }], + [128, { bytes: tinyPng(), format: 'png' }], + ]), + ); + thumbnails.set( + '/holiday.jpg', + new Map([[256, { bytes: tinyJpeg(), format: 'jpeg' }]]), + ); + thumbnails.set( + '/icon.webp', + new Map([[64, { bytes: tinyWebp(), format: 'webp' }]]), + ); + // Mismatched format: server returns PNG bytes but claims JPEG. + thumbnails.set('/lying.png', new Map([[64, { bytes: tinyPng(), format: 'jpeg' }]])); + + rig = await setupFileRig({ + getThumbnail: async (ctx) => { + const sizes = thumbnails.get(ctx.path); + if (sizes === undefined) throw new NotFoundError(ctx.path); + const entry = sizes.get(ctx.args.size); + if (entry === undefined) throw new NotFoundError(`${ctx.path}@${ctx.args.size}`); + return { + bytes: entry.bytes, + format: entry.format, + width: ctx.args.size, + height: ctx.args.size, + }; + }, + }); + }); + + afterAll(async () => { + await rig.teardown(); + }); + + test('PNG thumbnail round-trips with computed sha256', async () => { + const result = await rig.fs.getThumbnail('/photo.png', 64); + expect(result.format).toBe('png'); + expect(result.width).toBe(64); + expect(result.height).toBe(64); + expect(result.sha256).toBe(bytesToHex(sha256(result.bytes))); + // Verify magic bytes survived + expect(Array.from(result.bytes.slice(0, 8))).toEqual([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + }); + + test('JPEG thumbnail round-trips', async () => { + const result = await rig.fs.getThumbnail('/holiday.jpg', 256); + expect(result.format).toBe('jpeg'); + expect(result.width).toBe(256); + expect(Array.from(result.bytes.slice(0, 3))).toEqual([0xff, 0xd8, 0xff]); + }); + + test('WebP thumbnail round-trips', async () => { + const result = await rig.fs.getThumbnail('/icon.webp', 64); + expect(result.format).toBe('webp'); + expect(Array.from(result.bytes.slice(0, 4))).toEqual([0x52, 0x49, 0x46, 0x46]); + expect(Array.from(result.bytes.slice(8, 12))).toEqual([0x57, 0x45, 0x42, 0x50]); + }); + + test('PNG bytes claimed as JPEG → format-hardening rejects', async () => { + await expect(rig.fs.getThumbnail('/lying.png', 64)).rejects.toBeInstanceOf(ConflictError); + }); + + test('non-existent path → NotFoundError', async () => { + await expect(rig.fs.getThumbnail('/missing.png', 64)).rejects.toBeInstanceOf(NotFoundError); + }); + + test('different sizes return different thumbnails', async () => { + const small = await rig.fs.getThumbnail('/photo.png', 64); + const big = await rig.fs.getThumbnail('/photo.png', 128); + expect(small.width).toBe(64); + expect(big.width).toBe(128); + }); +}); diff --git a/packages/shade-files/tests/integration/upload-directory.test.ts b/packages/shade-files/tests/integration/upload-directory.test.ts new file mode 100644 index 0000000..b329e9b --- /dev/null +++ b/packages/shade-files/tests/integration/upload-directory.test.ts @@ -0,0 +1,238 @@ +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { sha256 } from '@noble/hashes/sha2.js'; +import { + ConflictError, + NotFoundError, + uploadDirectory, + createMemoryDirectory, + type FileEntry, + type BulkTransferEvent, +} from '../../src/index.js'; +import { setupFileRig, type FileTestRig } from './helpers/rig.js'; + +interface StoredFile { + bytes: Uint8Array; + contentType?: string; + sha256: string; +} + +function bytesToHex(b: Uint8Array): string { + return Array.from(b, (x) => x.toString(16).padStart(2, '0')).join(''); +} + +async function streamToBytes(s: ReadableStream): Promise { + const reader = s.getReader(); + const parts: Uint8Array[] = []; + let total = 0; + while (true) { + const { value, done } = await reader.read(); + if (done) break; + if (value === undefined) continue; + parts.push(value); + total += value.byteLength; + } + reader.releaseLock(); + const out = new Uint8Array(total); + let offset = 0; + for (const p of parts) { + out.set(p, offset); + offset += p.byteLength; + } + return out; +} + +describe('uploadDirectory — bulk upload to remote', () => { + let rig: FileTestRig; + const blobs = new Map(); + const dirs = new Set(['/']); + + beforeAll(async () => { + blobs.clear(); + dirs.clear(); + dirs.add('/'); + + rig = await setupFileRig({ + mkdir: async (ctx) => { + const path = ctx.path; + if (dirs.has(path)) { + if (!ctx.args.recursive) throw new ConflictError('exists'); + // Idempotent for recursive + } + // Recursive: add ancestors + if (ctx.args.recursive) { + const segments = path.split('/').filter(Boolean); + let acc = ''; + for (const seg of segments) { + acc += '/' + seg; + dirs.add(acc); + } + } else { + dirs.add(path); + } + return { + entry: { + name: path.split('/').filter(Boolean).pop() ?? '', + kind: 'dir', + size: 0, + mtime: Date.now(), + metadata: {}, + }, + }; + }, + write: async (ctx) => { + const args = ctx.args; + let bytes: Uint8Array; + let storedSha: string; + if (args.content.kind === 'inline') { + bytes = args.content.bytes; + storedSha = args.content.sha256; + } else { + bytes = await streamToBytes(args.content.stream); + storedSha = await args.content.sha256; + } + if (blobs.has(args.path) && !args.overwrite) { + throw new ConflictError(`${args.path} exists`); + } + blobs.set(args.path, { + bytes, + ...(args.contentType !== undefined ? { contentType: args.contentType } : {}), + sha256: storedSha, + }); + const entry: FileEntry = { + name: args.path.split('/').filter(Boolean).pop() ?? '', + kind: 'file', + size: bytes.byteLength, + mtime: Date.now(), + ...(args.contentType !== undefined ? { contentType: args.contentType } : {}), + metadata: { sha256: storedSha }, + }; + return { entry }; + }, + }); + }); + + afterAll(async () => { + await rig.teardown(); + }); + + test('uploads a small tree with mixed inline + streams', async () => { + const local = createMemoryDirectory('local'); + const sub = local.addDir('sub'); + const small = new Uint8Array(100); for (let i = 0; i < small.length; i++) small[i] = i & 0xff; + const big = new Uint8Array(400 * 1024); crypto.getRandomValues(big); + local.addFile('hello.txt', new TextEncoder().encode('hello world'), 'text/plain'); + local.addFile('small.bin', small); + sub.addFile('big.bin', big, 'application/octet-stream'); + + const handle = uploadDirectory(rig.fs, local, '/upload-target'); + + const events: BulkTransferEvent[] = []; + (async () => { + for await (const ev of handle.events) events.push(ev); + })().catch(() => undefined); + + const result = await handle.done(); + expect(result.filesDone).toBe(3); + expect(result.bytesDone).toBe(11 + 100 + 400 * 1024); + + // Verify remote tree + expect(blobs.has('/upload-target/hello.txt')).toBe(true); + expect(blobs.has('/upload-target/small.bin')).toBe(true); + expect(blobs.has('/upload-target/sub/big.bin')).toBe(true); + expect(dirs.has('/upload-target/sub')).toBe(true); + + // Sha256 paritet for streamed file + expect(blobs.get('/upload-target/sub/big.bin')!.sha256).toBe(bytesToHex(sha256(big))); + expect(blobs.get('/upload-target/hello.txt')!.contentType).toBe('text/plain'); + + // Wait a tick for events to flush + await new Promise((r) => setTimeout(r, 30)); + const planEvent = events.find((e) => e.type === 'plan'); + expect(planEvent).toBeDefined(); + if (planEvent && planEvent.type === 'plan') { + expect(planEvent.totalFiles).toBe(3); + expect(planEvent.totalBytes).toBe(11 + 100 + 400 * 1024); + } + const completes = events.filter((e) => e.type === 'complete'); + expect(completes.length).toBe(1); + }); + + test('aggregated progress is monotonically non-decreasing', async () => { + const local = createMemoryDirectory('local'); + for (let i = 0; i < 10; i++) { + local.addFile(`f${i}.bin`, new Uint8Array(50)); + } + const handle = uploadDirectory(rig.fs, local, '/progress'); + const progresses: { filesDone: number; bytesDone: number }[] = []; + (async () => { + for await (const ev of handle.events) { + if (ev.type === 'progress') { + progresses.push({ filesDone: ev.filesDone, bytesDone: ev.bytesDone }); + } + } + })().catch(() => undefined); + await handle.done(); + await new Promise((r) => setTimeout(r, 30)); + for (let i = 1; i < progresses.length; i++) { + expect(progresses[i]!.filesDone).toBeGreaterThanOrEqual(progresses[i - 1]!.filesDone); + expect(progresses[i]!.bytesDone).toBeGreaterThanOrEqual(progresses[i - 1]!.bytesDone); + } + expect(progresses[progresses.length - 1]!.filesDone).toBe(10); + }); + + test('fail-fast: first error aborts the bulk', async () => { + const local = createMemoryDirectory('local'); + for (let i = 0; i < 5; i++) { + local.addFile(`x${i}.bin`, new Uint8Array(10)); + } + // Pre-create a conflicting file at /conflict/x0.bin so the first write fails. + blobs.set('/conflict/x0.bin', { bytes: new Uint8Array(0), sha256: 'x' }); + + const handle = uploadDirectory(rig.fs, local, '/conflict', { concurrency: 1 }); + await expect(handle.done()).rejects.toThrow(); + }); + + test('continueOnError: completes despite per-file errors', async () => { + const local = createMemoryDirectory('local'); + for (let i = 0; i < 5; i++) { + local.addFile(`y${i}.bin`, new Uint8Array(10)); + } + blobs.set('/cont/y2.bin', { bytes: new Uint8Array(0), sha256: 'x' }); + + const handle = uploadDirectory(rig.fs, local, '/cont', { + concurrency: 1, + continueOnError: true, + }); + const errors: string[] = []; + (async () => { + for await (const ev of handle.events) { + if (ev.type === 'file-error') errors.push(ev.path); + } + })().catch(() => undefined); + const result = await handle.done(); + await new Promise((r) => setTimeout(r, 30)); + expect(errors).toEqual(['y2.bin']); + expect(result.filesDone).toBe(4); + }); + + test('concurrency cap respected', async () => { + const local = createMemoryDirectory('local'); + for (let i = 0; i < 30; i++) { + local.addFile(`z${i}.bin`, new Uint8Array(10)); + } + // Concurrency above MAX (16) should be clamped. + const handle = uploadDirectory(rig.fs, local, '/cap', { concurrency: 100 }); + const result = await handle.done(); + expect(result.filesDone).toBe(30); + }); + + test('aborts mid-flight via handle.abort()', async () => { + const local = createMemoryDirectory('local'); + for (let i = 0; i < 50; i++) { + local.addFile(`q${i}.bin`, new Uint8Array(50 * 1024)); // 50 KiB each + } + const handle = uploadDirectory(rig.fs, local, '/abort'); + setTimeout(() => void handle.abort('test-cancel'), 20); + await expect(handle.done()).rejects.toThrow(); + }); +}); diff --git a/packages/shade-files/tests/integration/walk.test.ts b/packages/shade-files/tests/integration/walk.test.ts new file mode 100644 index 0000000..1d3dbaa --- /dev/null +++ b/packages/shade-files/tests/integration/walk.test.ts @@ -0,0 +1,126 @@ +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { walk, type FileEntry, NotFoundError } from '../../src/index.js'; +import { setupFileRig, type FileTestRig } from './helpers/rig.js'; + +describe('walk — async-iterable depth-first directory traversal', () => { + let rig: FileTestRig; + // In-memory tree: + // / + // ├── a/ + // │ ├── 1.txt + // │ └── b/ + // │ └── 2.txt + // ├── c/ + // │ └── 3.txt + // └── 4.txt + const tree = new Map(); + + function setEntry(path: string, kind: 'file' | 'dir'): void { + const name = path === '/' ? '' : path.split('/').filter(Boolean).pop() ?? ''; + tree.set(path, { name, kind, size: kind === 'file' ? 10 : 0, mtime: 0, metadata: {} }); + } + + beforeAll(async () => { + tree.clear(); + setEntry('/', 'dir'); + setEntry('/a', 'dir'); + setEntry('/a/1.txt', 'file'); + setEntry('/a/b', 'dir'); + setEntry('/a/b/2.txt', 'file'); + setEntry('/c', 'dir'); + setEntry('/c/3.txt', 'file'); + setEntry('/4.txt', 'file'); + + rig = await setupFileRig({ + list: async (ctx) => { + if (!tree.has(ctx.path)) throw new NotFoundError(ctx.path); + const entries: FileEntry[] = []; + for (const [path, entry] of tree) { + if (path === ctx.path) continue; + if (!path.startsWith(ctx.path === '/' ? '/' : ctx.path + '/')) continue; + const rest = path.slice(ctx.path === '/' ? 1 : ctx.path.length + 1); + if (rest.includes('/')) continue; + entries.push(entry); + } + return { entries, hasMore: false }; + }, + }); + }); + + afterAll(async () => { + await rig.teardown(); + }); + + test('walks the entire tree depth-first', async () => { + const items: { path: string; depth: number }[] = []; + for await (const item of walk(rig.fs, '/')) { + items.push({ path: item.relativePath, depth: item.depth }); + } + // Depth-first order: visit a, then descend into a/* (1.txt + a/b → a/b/2.txt), then c, c/3.txt, then 4.txt + expect(items.map((i) => i.path)).toEqual([ + 'a', + 'a/1.txt', + 'a/b', + 'a/b/2.txt', + 'c', + 'c/3.txt', + '4.txt', + ]); + // Depth values + expect(items.find((i) => i.path === 'a')?.depth).toBe(1); + expect(items.find((i) => i.path === 'a/1.txt')?.depth).toBe(2); + expect(items.find((i) => i.path === 'a/b/2.txt')?.depth).toBe(3); + }); + + test('respects maxDepth', async () => { + const items: string[] = []; + for await (const item of walk(rig.fs, '/', { maxDepth: 1 })) { + items.push(item.relativePath); + } + // Only direct children — no descent into /a or /c. + expect(items.sort()).toEqual(['4.txt', 'a', 'c']); + }); + + test('breaks cleanly when consumer stops iterating', async () => { + const items: string[] = []; + for await (const item of walk(rig.fs, '/')) { + items.push(item.relativePath); + if (items.length === 2) break; + } + expect(items).toEqual(['a', 'a/1.txt']); + }); + + test('filter callback skips entries (and excludes their subtree)', async () => { + const items: string[] = []; + for await (const item of walk(rig.fs, '/', { + filter: (entry, _rel) => entry.name !== 'a', + })) { + items.push(item.relativePath); + } + expect(items.sort()).toEqual(['4.txt', 'c', 'c/3.txt']); + }); + + test('aborts via signal mid-walk', async () => { + const ctrl = new AbortController(); + const items: string[] = []; + setTimeout(() => ctrl.abort(), 5); + let threw = false; + try { + for await (const item of walk(rig.fs, '/', { signal: ctrl.signal })) { + items.push(item.relativePath); + await new Promise((r) => setTimeout(r, 10)); + } + } catch { + threw = true; + } + expect(threw).toBe(true); + }); + + test('walking non-existent path → throws on first list', async () => { + await expect(async () => { + for await (const _item of walk(rig.fs, '/nope')) { + /* unreachable */ + } + }).toThrow(); + }); +}); diff --git a/packages/shade-files/tests/security/fingerprint-gate.test.ts b/packages/shade-files/tests/security/fingerprint-gate.test.ts new file mode 100644 index 0000000..b7a3506 --- /dev/null +++ b/packages/shade-files/tests/security/fingerprint-gate.test.ts @@ -0,0 +1,86 @@ +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { setupFileRig, type FileTestRig } from '../integration/helpers/rig.js'; +import { FingerprintRequiredError, NotFoundError, type FileEntry } from '../../src/index.js'; + +describe('Fingerprint gate', () => { + let rig: FileTestRig; + const verifiedSet = new Set(); + + beforeAll(async () => { + verifiedSet.clear(); + rig = await setupFileRig({ + requireFingerprintVerifiedFor: (ctx) => { + // Mutations require verification; reads are optional. + const op = ctx.op; + if (op === 'mkdir' || op === 'delete' || op === 'move' || op === 'write') return 'required'; + if (op === 'list') return 'optional'; + return 'optional'; + }, + isFingerprintVerified: (sender) => verifiedSet.has(sender), + stat: async (ctx) => { + const e: FileEntry = { + name: ctx.path.split('/').filter(Boolean).pop() ?? '', + kind: 'file', size: 1, mtime: 0, metadata: {}, + }; + return e; + }, + mkdir: async (ctx) => { + const e: FileEntry = { + name: ctx.path.split('/').filter(Boolean).pop() ?? '', + kind: 'dir', size: 0, mtime: 0, metadata: {}, + }; + return { entry: e }; + }, + delete: async () => ({ deletedCount: 1 }), + list: async () => ({ entries: [], hasMore: false }), + }); + }); + + afterAll(async () => { + await rig.teardown(); + }); + + test('mutation without verification → FingerprintRequiredError', async () => { + verifiedSet.delete('alice'); + await expect(rig.fs.mkdir('/locked')).rejects.toBeInstanceOf(FingerprintRequiredError); + await expect(rig.fs.delete('/locked')).rejects.toBeInstanceOf(FingerprintRequiredError); + }); + + test('mutation after marking peer verified → succeeds', async () => { + verifiedSet.add('alice'); + const result = await rig.fs.mkdir('/verified-dir'); + expect(result.entry.name).toBe('verified-dir'); + }); + + test('optional ops (stat, list) work without verification', async () => { + verifiedSet.delete('alice'); + const stat = await rig.fs.stat('/anything'); + expect(stat.name).toBe('anything'); + const list = await rig.fs.list('/anything'); + expect(list.entries).toEqual([]); + }); +}); + +describe('Fingerprint gate with reject policy', () => { + let rig: FileTestRig; + beforeAll(async () => { + rig = await setupFileRig({ + requireFingerprintVerifiedFor: () => 'reject', + stat: async (ctx) => { + const e: FileEntry = { + name: 'x', + kind: 'file', + size: 1, + mtime: 0, + metadata: {}, + }; + return e; + }, + }); + }); + afterAll(async () => { await rig.teardown(); }); + + test('all ops rejected outright', async () => { + await expect(rig.fs.stat('/x')).rejects.toBeInstanceOf(FingerprintRequiredError); + }); +}); diff --git a/packages/shade-files/tests/security/quota.test.ts b/packages/shade-files/tests/security/quota.test.ts new file mode 100644 index 0000000..35f4030 --- /dev/null +++ b/packages/shade-files/tests/security/quota.test.ts @@ -0,0 +1,55 @@ +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { setupFileRig, type FileTestRig } from '../integration/helpers/rig.js'; +import { FsRateLimitError, QuotaExceededError, type FileEntry } from '../../src/index.js'; + +describe('Op rate limit', () => { + let rig: FileTestRig; + let listCount = 0; + beforeAll(async () => { + listCount = 0; + rig = await setupFileRig({ + rateLimits: { maxOpsPerMinutePerSender: 5 }, + list: async () => { + listCount++; + return { entries: [], hasMore: false }; + }, + }); + }); + afterAll(async () => { await rig.teardown(); }); + + test('op rate-limit kicks in after capacity', async () => { + listCount = 0; + for (let i = 0; i < 5; i++) await rig.fs.list('/'); + expect(listCount).toBe(5); + await expect(rig.fs.list('/')).rejects.toBeInstanceOf(FsRateLimitError); + }); +}); + +describe('Byte quota', () => { + let rig: FileTestRig; + beforeAll(async () => { + rig = await setupFileRig({ + rateLimits: { + // Plenty of ops, but tight byte cap for the quota test. + maxOpsPerMinutePerSender: 100, + maxBytesPerHourPerSender: 1024, + }, + write: async (ctx) => { + const e: FileEntry = { + name: ctx.args.path.split('/').filter(Boolean).pop() ?? '', + kind: 'file', + size: ctx.args.content.kind === 'inline' ? ctx.args.content.bytes.byteLength : ctx.args.content.size, + mtime: 0, + metadata: {}, + }; + return { entry: e }; + }, + }); + }); + afterAll(async () => { await rig.teardown(); }); + + test('write 2 KiB inline → exceeds 1 KiB/hour cap', async () => { + const big = new Uint8Array(2048); + await expect(rig.fs.write('/big.bin', big)).rejects.toBeInstanceOf(QuotaExceededError); + }); +}); diff --git a/packages/shade-files/tests/security/replay.test.ts b/packages/shade-files/tests/security/replay.test.ts new file mode 100644 index 0000000..748967d --- /dev/null +++ b/packages/shade-files/tests/security/replay.test.ts @@ -0,0 +1,119 @@ +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { setupFileRig, type FileTestRig } from '../integration/helpers/rig.js'; +import { ConflictError, NotFoundError, type FileEntry } from '../../src/index.js'; + +/** + * Replay-window: the dispatcher rejects requests where `signedAt` is more + * than ±5 min from the server clock. Plus: idempotent retries on the same + * mutation key produce a single side-effect even with stale signedAt. + */ + +describe('Replay window + idempotent retry', () => { + let rig: FileTestRig; + let writeCount = 0; + const blobs = new Map(); + + beforeAll(async () => { + rig = await setupFileRig({ + mkdir: async (ctx) => { + if (blobs.has(ctx.path)) throw new ConflictError('exists'); + blobs.set(ctx.path, new Uint8Array(0)); + const e: FileEntry = { + name: ctx.path.split('/').filter(Boolean).pop() ?? '', + kind: 'dir', size: 0, mtime: Date.now(), metadata: {}, + }; + return { entry: e }; + }, + write: async (ctx) => { + writeCount++; + if (ctx.args.content.kind !== 'inline') throw new Error('inline expected'); + blobs.set(ctx.args.path, ctx.args.content.bytes); + const e: FileEntry = { + name: ctx.args.path.split('/').filter(Boolean).pop() ?? '', + kind: 'file', + size: ctx.args.content.bytes.byteLength, + mtime: Date.now(), + metadata: { sha256: ctx.args.content.sha256 }, + }; + return { entry: e }; + }, + stat: async (ctx) => { + if (!blobs.has(ctx.path)) throw new NotFoundError(ctx.path); + return { + name: ctx.path.split('/').filter(Boolean).pop() ?? '', + kind: 'file', size: 0, mtime: 0, metadata: {}, + }; + }, + }); + }); + + afterAll(async () => { + await rig.teardown(); + }); + + test('idempotent retry: same key + same args → single side-effect', async () => { + writeCount = 0; + // Idempotency keys are exactly 22 chars, base64url alphabet. + const key = 'replay_key_1234567890A'; + const data = new Uint8Array([1, 2, 3]); + const r1 = await rig.fs.write('/replay.bin', data, { idempotencyKey: key }); + const r2 = await rig.fs.write('/replay.bin', data, { idempotencyKey: key }); + const r3 = await rig.fs.write('/replay.bin', data, { idempotencyKey: key }); + expect(writeCount).toBe(1); + expect(r1.entry.size).toBe(3); + expect(r2.entry.size).toBe(3); + expect(r3.entry.size).toBe(3); + }); + + test('out-of-window signedAt → InvalidSignatureError (skew rejection)', async () => { + // Build a custom client that LIES about signedAt. We hand-craft an + // RpcRequest envelope and ship it via the underlying channel. + const { + ShadeFileRpcChannel, + PendingRpcRegistry, + attachClientRouting, + KIND_STAT_V1, + generateRequestId, + } = await import('../../src/index.js'); + const aliceChannel = new ShadeFileRpcChannel(rig.alice); + const alicePending = new PendingRpcRegistry(); + attachClientRouting(aliceChannel, alicePending); + + const requestId = generateRequestId(); + const stalePromise = alicePending.register(requestId, { timeoutMs: 3000 }); + await aliceChannel.send('bob', { + kind: KIND_STAT_V1, + id: requestId, + args: { path: '/replay.bin' }, + sig: 'unsigned', + signedAt: Date.now() - 10 * 60 * 1000, // 10 min in the past — outside ±5 min window + }); + await expect(stalePromise).rejects.toThrow(/replay window|signature/i); + aliceChannel.destroy(); + }); + + test('signedAt far in the future → also rejected', async () => { + const { + ShadeFileRpcChannel, + PendingRpcRegistry, + attachClientRouting, + KIND_STAT_V1, + generateRequestId, + } = await import('../../src/index.js'); + const aliceChannel = new ShadeFileRpcChannel(rig.alice); + const alicePending = new PendingRpcRegistry(); + attachClientRouting(aliceChannel, alicePending); + + const requestId = generateRequestId(); + const promise = alicePending.register(requestId, { timeoutMs: 3000 }); + await aliceChannel.send('bob', { + kind: KIND_STAT_V1, + id: requestId, + args: { path: '/replay.bin' }, + sig: 'unsigned', + signedAt: Date.now() + 10 * 60 * 1000, + }); + await expect(promise).rejects.toThrow(/replay window|signature/i); + aliceChannel.destroy(); + }); +}); diff --git a/packages/shade-files/tests/security/tampered-envelope.test.ts b/packages/shade-files/tests/security/tampered-envelope.test.ts new file mode 100644 index 0000000..e643d83 --- /dev/null +++ b/packages/shade-files/tests/security/tampered-envelope.test.ts @@ -0,0 +1,115 @@ +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { ed25519 } from '@noble/curves/ed25519.js'; +import { setupFileRig, type FileTestRig } from '../integration/helpers/rig.js'; +import { + bytesToBase64, + canonicalRpcBytes, + hashArgs, + InvalidSignatureError, + type FileEntry, + NotFoundError, +} from '../../src/index.js'; + +/** + * The dispatcher's `verifySender` callback gets the canonical bytes the + * client claims they signed. By plugging a real Ed25519 verify in tests, + * we can demonstrate that: + * - A valid sig over the canonical bytes is accepted. + * - Tampering ANY bound field (kind, args, signedAt, sender) breaks + * verification → InvalidSignatureError. + */ + +describe('Tampered envelope — Ed25519 sig verification', () => { + let rig: FileTestRig; + // Generate a stable Ed25519 keypair for Alice. Bob will pin it. + const alicePriv = ed25519.utils.randomSecretKey(); + const alicePub = ed25519.getPublicKey(alicePriv); + + beforeAll(async () => { + rig = await setupFileRig({ + verifySender: (sender, canonical, sigBase64) => { + // We only know Alice's key for this test. Bob's pub key would be + // looked up similarly in a real app. + if (sender !== 'alice') return false; + // Decode base64 sig + try { + const bin = atob(sigBase64); + const sigBytes = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) sigBytes[i] = bin.charCodeAt(i); + return ed25519.verify(sigBytes, canonical, alicePub); + } catch { + return false; + } + }, + stat: async (ctx) => { + if (ctx.path !== '/exists.txt') throw new NotFoundError(ctx.path); + const e: FileEntry = { name: 'exists.txt', kind: 'file', size: 1, mtime: 0, metadata: {} }; + return e; + }, + }); + // Re-create the client with a real signRequest hook. + // (Rig's default fs has signRequest=undefined; we replace it.) + const { createFileClient, ShadeFileRpcChannel, PendingRpcRegistry, attachClientRouting } = await import('../../src/index.js'); + const aliceChannel = new ShadeFileRpcChannel(rig.alice); + const alicePending = new PendingRpcRegistry(); + attachClientRouting(aliceChannel, alicePending); + rig.fs = createFileClient(rig.alice, aliceChannel, alicePending, 'bob', { + defaultTimeoutMs: 5000, + signRequest: async (canonical) => { + const sig = ed25519.sign(canonical, alicePriv); + return bytesToBase64(sig); + }, + }); + }); + + afterAll(async () => { + await rig.teardown(); + }); + + test('valid signature → request succeeds', async () => { + const result = await rig.fs.stat('/exists.txt'); + expect(result.name).toBe('exists.txt'); + }); + + test('tampered args → InvalidSignatureError', async () => { + // Craft a request manually: sign over '/a' but ship '/b'. + const { createFileClient, ShadeFileRpcChannel, PendingRpcRegistry, attachClientRouting } = await import('../../src/index.js'); + const aliceChannel = new ShadeFileRpcChannel(rig.alice); + const alicePending = new PendingRpcRegistry(); + attachClientRouting(aliceChannel, alicePending); + const tamperedFs = createFileClient(rig.alice, aliceChannel, alicePending, 'bob', { + defaultTimeoutMs: 3000, + signRequest: async (_canonical) => { + // Sign over a DIFFERENT canonical (different argsHash), so the + // server's recomputation won't match. + const fake = canonicalRpcBytes({ + address: 'alice', + signedAt: 0, + kind: 'shade.fs.list/v1', + id: 'AAAAAAAAAAAAAAAAAAAAAA', + argsHash: hashArgs({ tampered: true }), + }); + const sig = ed25519.sign(fake, alicePriv); + return bytesToBase64(sig); + }, + }); + await expect(tamperedFs.stat('/exists.txt')).rejects.toBeInstanceOf(InvalidSignatureError); + }); + + test('valid signature from unknown signer → InvalidSignatureError', async () => { + const { createFileClient, ShadeFileRpcChannel, PendingRpcRegistry, attachClientRouting } = await import('../../src/index.js'); + const aliceChannel = new ShadeFileRpcChannel(rig.alice); + const alicePending = new PendingRpcRegistry(); + attachClientRouting(aliceChannel, alicePending); + + const otherPriv = ed25519.utils.randomSecretKey(); + const wrongFs = createFileClient(rig.alice, aliceChannel, alicePending, 'bob', { + defaultTimeoutMs: 3000, + signRequest: async (canonical) => { + const sig = ed25519.sign(canonical, otherPriv); + return bytesToBase64(sig); + }, + }); + await expect(wrongFs.stat('/exists.txt')).rejects.toBeInstanceOf(InvalidSignatureError); + }); +}); diff --git a/packages/shade-files/tests/unit/canonical.test.ts b/packages/shade-files/tests/unit/canonical.test.ts new file mode 100644 index 0000000..7040d87 --- /dev/null +++ b/packages/shade-files/tests/unit/canonical.test.ts @@ -0,0 +1,135 @@ +import { describe, test, expect } from 'bun:test'; +import { + canonicalRpcBytes, + canonicalJsonStringify, + hashArgs, + bytesToHex, + bytesToBase64, + base64ToBytes, +} from '../../src/index.js'; + +describe('canonicalJsonStringify', () => { + test('sorts object keys', () => { + expect(canonicalJsonStringify({ b: 1, a: 2 })).toBe('{"a":2,"b":1}'); + }); + + test('skips undefined values', () => { + expect(canonicalJsonStringify({ a: 1, b: undefined, c: 2 })).toBe('{"a":1,"c":2}'); + }); + + test('preserves array order', () => { + expect(canonicalJsonStringify([3, 1, 2])).toBe('[3,1,2]'); + }); + + test('recursive sorting', () => { + const a = canonicalJsonStringify({ outer: { y: 1, x: 2 } }); + const b = canonicalJsonStringify({ outer: { x: 2, y: 1 } }); + expect(a).toBe(b); + expect(a).toBe('{"outer":{"x":2,"y":1}}'); + }); + + test('handles primitives', () => { + expect(canonicalJsonStringify(null)).toBe('null'); + expect(canonicalJsonStringify(true)).toBe('true'); + expect(canonicalJsonStringify(42)).toBe('42'); + expect(canonicalJsonStringify('hi')).toBe('"hi"'); + }); + + test('different argument orders hash identically', () => { + const h1 = hashArgs({ b: 1, a: 2 }); + const h2 = hashArgs({ a: 2, b: 1 }); + expect(bytesToHex(h1)).toBe(bytesToHex(h2)); + }); + + test('different values hash differently', () => { + expect(bytesToHex(hashArgs({ a: 1 }))).not.toBe(bytesToHex(hashArgs({ a: 2 }))); + }); +}); + +describe('canonicalRpcBytes', () => { + test('deterministic for same input (known vector)', () => { + const args = { + address: 'alice', + signedAt: 1730000000000, + kind: 'shade.fs.list/v1', + id: 'AbCdEfGhIjKlMnOpQrStUv', + argsHash: new Uint8Array(32).fill(0xab), + }; + const a = canonicalRpcBytes(args); + const b = canonicalRpcBytes(args); + expect(a).toEqual(b); + }); + + test('changes when address differs', () => { + const base = { + signedAt: 1, + kind: 'shade.fs.list/v1', + id: 'AbCdEfGhIjKlMnOpQrStUv', + argsHash: new Uint8Array(32), + }; + const a = canonicalRpcBytes({ ...base, address: 'alice' }); + const b = canonicalRpcBytes({ ...base, address: 'bob' }); + expect(a).not.toEqual(b); + }); + + test('changes when signedAt differs', () => { + const base = { + address: 'alice', + kind: 'shade.fs.list/v1', + id: 'AbCdEfGhIjKlMnOpQrStUv', + argsHash: new Uint8Array(32), + }; + expect(canonicalRpcBytes({ ...base, signedAt: 1 })).not.toEqual( + canonicalRpcBytes({ ...base, signedAt: 2 }), + ); + }); + + test('changes when kind differs', () => { + const base = { + address: 'alice', + signedAt: 1, + id: 'AbCdEfGhIjKlMnOpQrStUv', + argsHash: new Uint8Array(32), + }; + expect(canonicalRpcBytes({ ...base, kind: 'shade.fs.list/v1' })).not.toEqual( + canonicalRpcBytes({ ...base, kind: 'shade.fs.stat/v1' }), + ); + }); + + test('changes when argsHash differs', () => { + const base = { + address: 'alice', + signedAt: 1, + kind: 'shade.fs.list/v1', + id: 'AbCdEfGhIjKlMnOpQrStUv', + }; + expect( + canonicalRpcBytes({ ...base, argsHash: new Uint8Array(32) }), + ).not.toEqual( + canonicalRpcBytes({ ...base, argsHash: new Uint8Array(32).fill(1) }), + ); + }); +}); + +describe('hashArgs', () => { + test('produces 32-byte digest', () => { + expect(hashArgs({ a: 1 }).length).toBe(32); + }); + + test('null input is fine', () => { + expect(hashArgs(null).length).toBe(32); + }); +}); + +describe('bytesToHex / base64', () => { + test('hex roundtrip', () => { + const b = new Uint8Array([0x01, 0xab, 0xff, 0x00]); + expect(bytesToHex(b)).toBe('01abff00'); + }); + + test('base64 roundtrip', () => { + const b = new Uint8Array([1, 2, 3, 4]); + const enc = bytesToBase64(b); + expect(base64ToBytes(enc)).toEqual(b); + }); +}); diff --git a/packages/shade-files/tests/unit/concurrency.test.ts b/packages/shade-files/tests/unit/concurrency.test.ts new file mode 100644 index 0000000..41fbea0 --- /dev/null +++ b/packages/shade-files/tests/unit/concurrency.test.ts @@ -0,0 +1,90 @@ +import { describe, test, expect } from 'bun:test'; +import { runWithConcurrency } from '../../src/client/concurrency.js'; + +async function* range(n: number): AsyncIterable { + for (let i = 0; i < n; i++) yield i; +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +describe('runWithConcurrency', () => { + test('runs all items', async () => { + const seen: number[] = []; + await runWithConcurrency(range(10), async (i) => { + seen.push(i); + }, { concurrency: 4 }); + expect(seen.sort((a, b) => a - b)).toEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + }); + + test('respects concurrency cap (never exceeds N inflight)', async () => { + let inflight = 0; + let peak = 0; + await runWithConcurrency(range(50), async () => { + inflight++; + peak = Math.max(peak, inflight); + await delay(5); + inflight--; + }, { concurrency: 4 }); + expect(peak).toBeLessThanOrEqual(4); + expect(peak).toBeGreaterThanOrEqual(2); + }); + + test('throws on first error by default (fail-fast)', async () => { + let processed = 0; + await expect( + runWithConcurrency(range(20), async (i) => { + await delay(1); + if (i === 3) throw new Error('boom'); + processed++; + }, { concurrency: 2 }), + ).rejects.toThrow('boom'); + // We don't process all 20; bounded by fail-fast. + expect(processed).toBeLessThan(20); + }); + + test('continueOnError reports each + drains', async () => { + const errors: number[] = []; + let processed = 0; + await runWithConcurrency(range(10), async (i) => { + await delay(1); + if (i % 3 === 0) throw new Error(`bad-${i}`); + processed++; + }, { + concurrency: 3, + continueOnError: true, + onError: (item) => errors.push(item), + }); + expect(processed).toBe(6); // i = 1,2,4,5,7,8 + expect(errors.sort((a, b) => a - b)).toEqual([0, 3, 6, 9]); + }); + + test('aborts via signal', async () => { + const ctrl = new AbortController(); + let processed = 0; + setTimeout(() => ctrl.abort(), 20); + await expect( + runWithConcurrency(range(100), async () => { + await delay(5); + processed++; + }, { concurrency: 4, signal: ctrl.signal }), + ).rejects.toThrow(); + expect(processed).toBeLessThan(100); + }); + + test('concurrency=1 is sequential', async () => { + const order: number[] = []; + await runWithConcurrency(range(5), async (i) => { + order.push(i); + await delay(1); + }, { concurrency: 1 }); + expect(order).toEqual([0, 1, 2, 3, 4]); + }); + + test('throws on concurrency < 1', () => { + expect(() => + runWithConcurrency(range(0), async () => undefined, { concurrency: 0 }), + ).toThrow('concurrency must be ≥ 1'); + }); +}); diff --git a/packages/shade-files/tests/unit/correlate.test.ts b/packages/shade-files/tests/unit/correlate.test.ts new file mode 100644 index 0000000..528fbbd --- /dev/null +++ b/packages/shade-files/tests/unit/correlate.test.ts @@ -0,0 +1,64 @@ +import { describe, test, expect } from 'bun:test'; +import * as fc from 'fast-check'; +import { + generateRequestId, + generateIdempotencyKey, + base64UrlEncode, + base64UrlDecode, + RequestIdSchema, +} from '../../src/index.js'; + +describe('generateRequestId', () => { + test('produces 22-char base64url string', () => { + const id = generateRequestId(); + expect(id.length).toBe(22); + expect(RequestIdSchema.safeParse(id).success).toBe(true); + }); + + test('1e5 generated IDs are all unique', () => { + const seen = new Set(); + for (let i = 0; i < 100_000; i++) { + const id = generateRequestId(); + expect(seen.has(id)).toBe(false); + seen.add(id); + } + expect(seen.size).toBe(100_000); + }); + + test('generateIdempotencyKey returns the same shape', () => { + expect(generateIdempotencyKey().length).toBe(22); + }); +}); + +describe('base64url encode/decode', () => { + test('roundtrip arbitrary bytes (property-based)', () => { + fc.assert( + fc.property(fc.uint8Array({ minLength: 0, maxLength: 64 }), (bytes) => { + const decoded = base64UrlDecode(base64UrlEncode(bytes)); + expect(decoded).toEqual(bytes); + }), + { numRuns: 500 }, + ); + }); + + test('produces URL-safe alphabet only', () => { + fc.assert( + fc.property(fc.uint8Array({ minLength: 1, maxLength: 64 }), (bytes) => { + const enc = base64UrlEncode(bytes); + expect(enc).not.toMatch(/[+/=]/); + }), + { numRuns: 200 }, + ); + }); + + test('handles empty input', () => { + expect(base64UrlEncode(new Uint8Array(0))).toBe(''); + expect(base64UrlDecode('')).toEqual(new Uint8Array(0)); + }); + + test('decodes inputs without padding correctly', () => { + expect(base64UrlDecode('YQ')).toEqual(new Uint8Array([0x61])); + expect(base64UrlDecode('YWI')).toEqual(new Uint8Array([0x61, 0x62])); + expect(base64UrlDecode('YWJj')).toEqual(new Uint8Array([0x61, 0x62, 0x63])); + }); +}); diff --git a/packages/shade-files/tests/unit/envelope-codec.test.ts b/packages/shade-files/tests/unit/envelope-codec.test.ts new file mode 100644 index 0000000..38ae361 --- /dev/null +++ b/packages/shade-files/tests/unit/envelope-codec.test.ts @@ -0,0 +1,149 @@ +import { describe, test, expect } from 'bun:test'; +import { + encodeEnvelope, + looksLikeFileEnvelope, + tryParseEnvelope, + classify, + KIND_LIST_V1, + KIND_ERROR_V1, + KIND_CANCEL_V1, + responseKindOf, +} from '../../src/index.js'; +import type { RpcRequest, RpcResponse, RpcError, RpcCancel } from '../../src/index.js'; + +const ID = 'AbCdEfGhIjKlMnOpQrStUv'; + +describe('looksLikeFileEnvelope', () => { + test('matches plaintext containing shade.fs', () => { + expect(looksLikeFileEnvelope('{"kind":"shade.fs.list/v1"}')).toBe(true); + }); + test('rejects unrelated', () => { + expect(looksLikeFileEnvelope('hello world')).toBe(false); + expect(looksLikeFileEnvelope('{"kind":"shade.stream-init/v1"}')).toBe(false); + }); +}); + +describe('tryParseEnvelope', () => { + test('returns null on non-JSON', () => { + expect(tryParseEnvelope('not json {{')).toBeNull(); + }); + + test('returns null on JSON that does not match any envelope', () => { + expect(tryParseEnvelope('{"kind":"unknown"}')).toBeNull(); + expect(tryParseEnvelope('{"foo":"bar"}')).toBeNull(); + }); + + test('classifies request', () => { + const req: RpcRequest = { + kind: KIND_LIST_V1, + id: ID, + args: { path: '/' }, + sig: 'abc', + signedAt: 1, + }; + const c = tryParseEnvelope(encodeEnvelope(req)); + expect(c?.kind).toBe('request'); + }); + + test('classifies response', () => { + const resp: RpcResponse = { + kind: responseKindOf(KIND_LIST_V1), + id: ID, + result: { entries: [], hasMore: false }, + }; + const c = tryParseEnvelope(encodeEnvelope(resp)); + expect(c?.kind).toBe('response'); + }); + + test('classifies error', () => { + const err: RpcError = { + kind: KIND_ERROR_V1, + id: ID, + error: { code: 'NOT_FOUND', message: 'missing' }, + }; + const c = tryParseEnvelope(encodeEnvelope(err)); + expect(c?.kind).toBe('error'); + }); + + test('classifies cancel', () => { + const cancel: RpcCancel = { + kind: KIND_CANCEL_V1, + id: ID, + reason: 'user-cancel', + }; + const c = tryParseEnvelope(encodeEnvelope(cancel)); + expect(c?.kind).toBe('cancel'); + }); + + test('rejects request with missing fields', () => { + expect(tryParseEnvelope(JSON.stringify({ kind: KIND_LIST_V1 }))).toBeNull(); + expect( + tryParseEnvelope( + JSON.stringify({ kind: KIND_LIST_V1, id: ID, args: {}, sig: 'x' }), + ), + ).toBeNull(); // missing signedAt + }); + + test('rejects response shape with non-.response suffix and no signature', () => { + // Missing both `.response` suffix (so not a response) AND missing + // `sig`/`signedAt` (so not a valid request) → no schema matches. + expect( + tryParseEnvelope( + JSON.stringify({ kind: 'shade.fs.list/v1', id: ID, result: {} }), + ), + ).toBeNull(); + }); + + test('rejects malformed id length', () => { + const req = { + kind: KIND_LIST_V1, + id: 'short', + args: {}, + sig: 'x', + signedAt: 1, + }; + expect(tryParseEnvelope(JSON.stringify(req))).toBeNull(); + }); +}); + +describe('classify on already-validated envelopes', () => { + test('correct discriminator on each branch', () => { + const req: RpcRequest = { + kind: KIND_LIST_V1, id: ID, args: {}, sig: 'x', signedAt: 1, + }; + expect(classify(req).kind).toBe('request'); + + const resp: RpcResponse = { + kind: responseKindOf(KIND_LIST_V1), id: ID, result: {}, + }; + expect(classify(resp).kind).toBe('response'); + + const err: RpcError = { + kind: KIND_ERROR_V1, id: ID, error: { code: 'NOT_FOUND', message: '' }, + }; + expect(classify(err).kind).toBe('error'); + + const cancel: RpcCancel = { kind: KIND_CANCEL_V1, id: ID }; + expect(classify(cancel).kind).toBe('cancel'); + }); +}); + +describe('encodeEnvelope', () => { + test('roundtrips request envelope', () => { + const req: RpcRequest = { + kind: KIND_LIST_V1, + id: ID, + args: { path: '/foo' }, + idempotencyKey: 'IdemKeyAaBbCcDdEeFfGgH', + attempt: 1, + sig: 'sig', + signedAt: 1730000000000, + }; + const c = tryParseEnvelope(encodeEnvelope(req)); + expect(c?.kind).toBe('request'); + if (c?.kind === 'request') { + expect(c.envelope.idempotencyKey).toBe('IdemKeyAaBbCcDdEeFfGgH'); + expect(c.envelope.attempt).toBe(1); + } + }); +}); diff --git a/packages/shade-files/tests/unit/idempotency-cache.test.ts b/packages/shade-files/tests/unit/idempotency-cache.test.ts new file mode 100644 index 0000000..aeaf5b2 --- /dev/null +++ b/packages/shade-files/tests/unit/idempotency-cache.test.ts @@ -0,0 +1,108 @@ +import { describe, test, expect } from 'bun:test'; +import { IdempotencyCache, IdempotencyConflictError } from '../../src/index.js'; + +describe('IdempotencyCache.begin', () => { + test('first call returns fresh', () => { + const cache = new IdempotencyCache(); + const result = cache.begin('alice', 'key1', { path: '/foo' }); + expect(result.status).toBe('fresh'); + }); + + test('replay returns cached response', () => { + const cache = new IdempotencyCache(); + const a = cache.begin('alice', 'key1', { path: '/foo' }); + if (a.status !== 'fresh') throw new Error('expected fresh'); + a.commit({ ok: true }); + + const b = cache.begin('alice', 'key1', { path: '/foo' }); + expect(b.status).toBe('replay'); + if (b.status === 'replay') { + expect(b.response).toEqual({ ok: true }); + } + }); + + test('argsHash mismatch throws IdempotencyConflictError', () => { + const cache = new IdempotencyCache(); + const a = cache.begin('alice', 'key1', { path: '/foo' }); + if (a.status !== 'fresh') throw new Error('expected fresh'); + a.commit({ ok: true }); + + expect(() => + cache.begin('alice', 'key1', { path: '/different' }), + ).toThrow(IdempotencyConflictError); + }); + + test('inflight retry returns wait-promise', async () => { + const cache = new IdempotencyCache(); + const a = cache.begin('alice', 'key1', { x: 1 }); + if (a.status !== 'fresh') throw new Error('expected fresh'); + + const b = cache.begin('alice', 'key1', { x: 1 }); + expect(b.status).toBe('wait'); + + if (b.status === 'wait') { + a.commit({ result: 42 }); + const v = await b.promise; + expect(v).toEqual({ result: 42 }); + } + }); + + test('different senders are isolated', () => { + const cache = new IdempotencyCache(); + const a = cache.begin('alice', 'k', { x: 1 }); + if (a.status !== 'fresh') throw new Error('expected fresh'); + a.commit({ side: 'alice' }); + + const b = cache.begin('bob', 'k', { x: 1 }); + expect(b.status).toBe('fresh'); + }); + + test('abandon removes the entry so retries proceed fresh', () => { + const cache = new IdempotencyCache(); + const a = cache.begin('alice', 'key', { p: 1 }); + if (a.status !== 'fresh') throw new Error('expected fresh'); + a.abandon(); + + const b = cache.begin('alice', 'key', { p: 1 }); + expect(b.status).toBe('fresh'); + }); +}); + +describe('IdempotencyCache TTL + LRU', () => { + test('expired entries are evicted on next access', async () => { + const cache = new IdempotencyCache({ ttlMs: 5 }); + const a = cache.begin('alice', 'k', { x: 1 }); + if (a.status !== 'fresh') throw new Error('expected fresh'); + a.commit('done'); + + await new Promise((r) => setTimeout(r, 15)); + + const b = cache.begin('alice', 'k', { x: 1 }); + expect(b.status).toBe('fresh'); // expired → fresh again + }); + + test('LRU caps per-sender entries', () => { + const cache = new IdempotencyCache({ maxEntriesPerSender: 3 }); + for (let i = 0; i < 5; i++) { + const r = cache.begin('alice', `key${i}`, { i }); + if (r.status === 'fresh') r.commit(i); + } + expect(cache.size()).toBe(3); // first two evicted + }); +}); + +describe('IdempotencyCache.prune', () => { + test('removes only TTL-expired entries', async () => { + // Two senders so begin() on one doesn't auto-evict the other's expired entry. + const cache = new IdempotencyCache({ ttlMs: 5 }); + const a = cache.begin('alice', 'old', { x: 1 }); + if (a.status === 'fresh') a.commit(1); + await new Promise((r) => setTimeout(r, 15)); + const b = cache.begin('bob', 'new', { x: 2 }); + if (b.status === 'fresh') b.commit(2); + + const removed = cache.prune(); + expect(removed).toBe(1); // alice/old expired; bob/new fresh + expect(cache.size()).toBe(1); + }); +}); diff --git a/packages/shade-files/tests/unit/inline-threshold.test.ts b/packages/shade-files/tests/unit/inline-threshold.test.ts new file mode 100644 index 0000000..1a2e6d7 --- /dev/null +++ b/packages/shade-files/tests/unit/inline-threshold.test.ts @@ -0,0 +1,161 @@ +import { describe, expect, test } from 'bun:test'; +import { decideInline, INLINE_THRESHOLD } from '../../src/client/inline-threshold.js'; + +const KIB = 1024; + +function streamOf(...chunks: Uint8Array[]): ReadableStream { + let i = 0; + return new ReadableStream({ + pull(controller) { + if (i < chunks.length) { + controller.enqueue(chunks[i]!); + i++; + } else { + controller.close(); + } + }, + }); +} + +async function drainStream(s: ReadableStream): Promise { + const reader = s.getReader(); + const parts: Uint8Array[] = []; + let total = 0; + while (true) { + const { value, done } = await reader.read(); + if (done) break; + if (value === undefined) continue; + parts.push(value); + total += value.byteLength; + } + reader.releaseLock(); + const out = new Uint8Array(total); + let offset = 0; + for (const p of parts) { + out.set(p, offset); + offset += p.byteLength; + } + return out; +} + +describe('decideInline (Uint8Array)', () => { + test('1 KiB → inline', async () => { + const bytes = new Uint8Array(KIB).fill(0xab); + const decision = await decideInline(bytes); + expect(decision.kind).toBe('inline'); + if (decision.kind === 'inline') { + expect(decision.bytes.byteLength).toBe(KIB); + } + }); + + test('exactly 256 KiB → inline (boundary)', async () => { + const bytes = new Uint8Array(INLINE_THRESHOLD).fill(0xcd); + const decision = await decideInline(bytes); + expect(decision.kind).toBe('inline'); + }); + + test('256 KiB + 1 → streams (boundary +1)', async () => { + const bytes = new Uint8Array(INLINE_THRESHOLD + 1).fill(0xef); + const decision = await decideInline(bytes); + expect(decision.kind).toBe('streams'); + if (decision.kind === 'streams') { + expect(decision.size).toBe(INLINE_THRESHOLD + 1); + const drained = await drainStream(decision.stream); + expect(drained.byteLength).toBe(INLINE_THRESHOLD + 1); + } + }); +}); + +describe('decideInline (Blob)', () => { + test('small Blob → inline + propagates contentType', async () => { + const blob = new Blob([new Uint8Array(100).fill(7)], { type: 'application/octet-stream' }); + const decision = await decideInline(blob); + expect(decision.kind).toBe('inline'); + if (decision.kind === 'inline') { + expect(decision.contentType).toBe('application/octet-stream'); + expect(decision.bytes.byteLength).toBe(100); + } + }); + + test('large Blob → streams + propagates size + contentType', async () => { + const big = new Uint8Array(INLINE_THRESHOLD + KIB).fill(1); + const blob = new Blob([big], { type: 'image/png' }); + const decision = await decideInline(blob); + expect(decision.kind).toBe('streams'); + if (decision.kind === 'streams') { + expect(decision.size).toBe(big.byteLength); + expect(decision.contentType).toBe('image/png'); + } + }); + + test('empty Blob.type → no contentType', async () => { + const blob = new Blob([new Uint8Array(10)]); + const decision = await decideInline(blob); + expect(decision.kind).toBe('inline'); + if (decision.kind === 'inline') { + expect(decision.contentType).toBeUndefined(); + } + }); +}); + +describe('decideInline (ReadableStream — bare)', () => { + test('EOF before threshold → inline', async () => { + const stream = streamOf(new Uint8Array(100), new Uint8Array(200)); + const decision = await decideInline(stream); + expect(decision.kind).toBe('inline'); + if (decision.kind === 'inline') { + expect(decision.bytes.byteLength).toBe(300); + } + }); + + test('crosses threshold mid-chunk → streams + remainder available', async () => { + // 256 KiB + 1 byte across two chunks + const a = new Uint8Array(200 * KIB).fill(0x10); + const b = new Uint8Array(100 * KIB).fill(0x20); + const stream = streamOf(a, b); + const decision = await decideInline(stream); + expect(decision.kind).toBe('streams'); + if (decision.kind === 'streams') { + const drained = await drainStream(decision.stream); + expect(drained.byteLength).toBe(a.byteLength + b.byteLength); + expect(drained[0]).toBe(0x10); + expect(drained[drained.length - 1]).toBe(0x20); + } + }); + + test('empty stream → inline (zero bytes)', async () => { + const stream = streamOf(); + const decision = await decideInline(stream); + expect(decision.kind).toBe('inline'); + if (decision.kind === 'inline') { + expect(decision.bytes.byteLength).toBe(0); + } + }); +}); + +describe('decideInline ({ stream, size })', () => { + test('declared size ≤ threshold → inline', async () => { + const bytes = new Uint8Array(KIB).fill(9); + const decision = await decideInline({ stream: streamOf(bytes), size: KIB, contentType: 'text/plain' }); + expect(decision.kind).toBe('inline'); + if (decision.kind === 'inline') { + expect(decision.bytes.byteLength).toBe(KIB); + expect(decision.contentType).toBe('text/plain'); + } + }); + + test('declared size > threshold → streams', async () => { + const big = new Uint8Array(500 * KIB).fill(2); + const decision = await decideInline({ stream: streamOf(big), size: big.byteLength }); + expect(decision.kind).toBe('streams'); + if (decision.kind === 'streams') { + expect(decision.size).toBe(big.byteLength); + } + }); +}); + +describe('decideInline (errors)', () => { + test('unsupported input throws TypeError', async () => { + await expect(decideInline(42 as unknown as Uint8Array)).rejects.toThrow(TypeError); + }); +}); diff --git a/packages/shade-files/tests/unit/path-policy.test.ts b/packages/shade-files/tests/unit/path-policy.test.ts new file mode 100644 index 0000000..9a5ecbc --- /dev/null +++ b/packages/shade-files/tests/unit/path-policy.test.ts @@ -0,0 +1,128 @@ +import { describe, test, expect } from 'bun:test'; +import * as fc from 'fast-check'; +import { validatePath } from '../../src/index.js'; + +describe('validatePath — happy path', () => { + test('accepts simple absolute paths', () => { + expect(validatePath('/foo')).toEqual({ ok: true, normalized: '/foo' }); + expect(validatePath('/foo/bar/baz.txt')).toEqual({ + ok: true, + normalized: '/foo/bar/baz.txt', + }); + expect(validatePath('/')).toEqual({ ok: true, normalized: '/' }); + }); + + test('normalizes redundant slashes and dots', () => { + expect(validatePath('//foo//bar/./baz/').normalized).toBe('/foo/bar/baz'); + expect(validatePath('/./foo').normalized).toBe('/foo'); + }); + + test('UTF-8 paths are accepted', () => { + expect(validatePath('/Документы/файл.txt').normalized).toBe('/Документы/файл.txt'); + expect(validatePath('/絵文字 😀/foo').normalized).toBe('/絵文字 😀/foo'); + }); +}); + +describe('validatePath — security', () => { + test('rejects raw `..` segments', () => { + expect(validatePath('/../etc/passwd').ok).toBe(false); + expect(validatePath('/foo/../etc').ok).toBe(false); + expect(validatePath('/..').ok).toBe(false); + }); + + test('rejects percent-encoded `..`', () => { + expect(validatePath('/%2e%2e/etc').ok).toBe(false); + expect(validatePath('/foo/%2E%2E/etc').ok).toBe(false); + }); + + test('rejects forbidden control bytes', () => { + expect(validatePath('/foo\x00bar').ok).toBe(false); + expect(validatePath('/foo\r\nbar').ok).toBe(false); + expect(validatePath('/foo\x7f').ok).toBe(false); + expect(validatePath('/foo\x01').ok).toBe(false); + }); + + test('rejects backslashes (Windows-style)', () => { + expect(validatePath('/foo\\bar').ok).toBe(false); + }); + + test('rejects relative paths', () => { + expect(validatePath('foo').ok).toBe(false); + expect(validatePath('./foo').ok).toBe(false); + expect(validatePath('').ok).toBe(false); + }); + + test('rejects over-length paths', () => { + expect(validatePath('/' + 'a'.repeat(4096)).ok).toBe(false); + expect(validatePath('/foobar', { maxLength: 5 }).ok).toBe(false); + expect(validatePath('/abc', { maxLength: 5 }).ok).toBe(true); + }); +}); + +describe('validatePath — rootScope', () => { + test('accepts paths inside scope', () => { + expect( + validatePath('/srv/data/foo', { rootScope: '/srv/data' }).ok, + ).toBe(true); + expect(validatePath('/srv/data', { rootScope: '/srv/data' }).ok).toBe(true); + }); + + test('rejects paths outside scope', () => { + expect(validatePath('/etc/passwd', { rootScope: '/srv/data' }).ok).toBe(false); + expect(validatePath('/srv/dataX', { rootScope: '/srv/data' }).ok).toBe(false); + // Boundary check: /srv/database is NOT inside /srv/data + expect(validatePath('/srv/database/x', { rootScope: '/srv/data' }).ok).toBe(false); + }); +}); + +describe('validatePath — extra hook', () => { + test('extra reject takes precedence', () => { + const result = validatePath('/secret/foo', { + extra: (p) => (p.includes('secret') ? 'reject' : 'allow'), + }); + expect(result.ok).toBe(false); + }); + + test('extra allow is the default', () => { + expect( + validatePath('/foo', { + extra: () => 'allow', + }).ok, + ).toBe(true); + }); +}); + +describe('validatePath — property-based', () => { + test('any string with a forbidden control byte is rejected', () => { + fc.assert( + fc.property( + fc.string({ minLength: 1, maxLength: 100 }), + fc.constantFrom('\x00', '\x07', '\x0a', '\x0d', '\x7f', '\\'), + (prefix, bad) => { + const p = `/${prefix}${bad}`; + expect(validatePath(p).ok).toBe(false); + }, + ), + { numRuns: 200 }, + ); + }); + + test('any path inside rootScope normalizes within rootScope', () => { + fc.assert( + fc.property( + fc.array(fc.string({ minLength: 1, maxLength: 20 }).filter( + (s) => /^[A-Za-z0-9_-]+$/.test(s), + ), { minLength: 1, maxLength: 5 }), + (segments) => { + const root = '/srv'; + const path = `${root}/${segments.join('/')}`; + const r = validatePath(path, { rootScope: root }); + if (r.ok) { + expect(r.normalized.startsWith(root)).toBe(true); + } + }, + ), + { numRuns: 200 }, + ); + }); +}); diff --git a/packages/shade-files/tests/unit/rate-limiter.test.ts b/packages/shade-files/tests/unit/rate-limiter.test.ts new file mode 100644 index 0000000..9ec9e09 --- /dev/null +++ b/packages/shade-files/tests/unit/rate-limiter.test.ts @@ -0,0 +1,91 @@ +import { describe, test, expect } from 'bun:test'; +import { + FsRateLimitError, + QuotaExceededError, + RateLimiter, +} from '../../src/index.js'; + +describe('RateLimiter — op bucket', () => { + test('allows up to capacity then rejects', () => { + const rl = new RateLimiter({ maxOpsPerMinutePerSender: 5, opCost: { default: 1 } }); + for (let i = 0; i < 5; i++) rl.acquire('alice', 'list'); + expect(() => rl.acquire('alice', 'list')).toThrow(FsRateLimitError); + }); + + test('rejection includes retryAfterMs', () => { + const rl = new RateLimiter({ maxOpsPerMinutePerSender: 1, opCost: { default: 1 } }); + rl.acquire('alice', 'list'); + try { + rl.acquire('alice', 'list'); + throw new Error('expected throw'); + } catch (err) { + expect(err).toBeInstanceOf(FsRateLimitError); + const payload = (err as FsRateLimitError).payload; + expect(payload.retryAfterMs).toBeGreaterThan(0); + } + }); + + test('different op costs respected', () => { + const rl = new RateLimiter({ + maxOpsPerMinutePerSender: 10, + opCost: { write: 5, default: 1 }, + }); + rl.acquire('alice', 'write'); // 10 - 5 = 5 left + rl.acquire('alice', 'write'); // 5 - 5 = 0 left + expect(() => rl.acquire('alice', 'write')).toThrow(FsRateLimitError); + }); + + test('per-sender isolation', () => { + const rl = new RateLimiter({ maxOpsPerMinutePerSender: 1 }); + rl.acquire('alice', 'list'); + rl.acquire('bob', 'list'); // bob's bucket independent + expect(() => rl.acquire('alice', 'list')).toThrow(FsRateLimitError); + expect(() => rl.acquire('bob', 'list')).toThrow(FsRateLimitError); + }); + + test('release returns tokens', () => { + const rl = new RateLimiter({ maxOpsPerMinutePerSender: 1 }); + rl.acquire('alice', 'list'); + rl.release('alice', 'list'); + rl.acquire('alice', 'list'); // should succeed again + }); +}); + +describe('RateLimiter — byte bucket', () => { + test('quota exceeded triggers QuotaExceededError', () => { + const rl = new RateLimiter({ maxBytesPerHourPerSender: 1000 }); + rl.acquire('alice', 'write', 600); + rl.acquire('alice', 'write', 400); + expect(() => rl.acquire('alice', 'write', 1)).toThrow(QuotaExceededError); + }); + + test('reconcile returns over-reserved bytes', () => { + const rl = new RateLimiter({ maxBytesPerHourPerSender: 1000 }); + rl.acquire('alice', 'write', 800); + rl.reconcile('alice', 800, 200); // we only used 200 of the 800 reserved + rl.acquire('alice', 'write', 600); // capacity now 400 + 600 = 1000, fits 600 + }); + + test('release returns reserved bytes', () => { + const rl = new RateLimiter({ maxBytesPerHourPerSender: 100 }); + rl.acquire('alice', 'read', 80); + rl.release('alice', 'read', 80); + rl.acquire('alice', 'read', 80); + }); +}); + +describe('RateLimiter — refill', () => { + test('tokens refill over time', async () => { + const rl = new RateLimiter({ + // 6000/min = 100/sec = 0.1/ms; in 50ms we refill 5 + maxOpsPerMinutePerSender: 6000, + opCost: { default: 1 }, + }); + // Drain by 5 + for (let i = 0; i < 5; i++) rl.acquire('alice', 'list'); + const before = rl.snapshot('alice')!.ops; + await new Promise((r) => setTimeout(r, 50)); + const after = rl.snapshot('alice')!.ops; + expect(after).toBeGreaterThan(before); + }); +}); diff --git a/packages/shade-files/tests/unit/schemas.test.ts b/packages/shade-files/tests/unit/schemas.test.ts new file mode 100644 index 0000000..e6f2e9d --- /dev/null +++ b/packages/shade-files/tests/unit/schemas.test.ts @@ -0,0 +1,350 @@ +import { describe, test, expect } from 'bun:test'; +import { + PathSchema, + RequestIdSchema, + CursorSchema, + Sha256HexSchema, + FileEntrySchema, + FileKindSchema, + ListPageSchema, + ListArgsSchema, + StatArgsSchema, + MkdirArgsSchema, + DeleteArgsSchema, + MoveArgsSchema, + ReadArgsSchema, + ReadResultSchema, + WriteArgsSchema, + GetThumbnailArgsSchema, + GetThumbnailResultSchema, + CustomArgsSchema, + RpcRequestSchema, + RpcResponseSchema, + RpcErrorSchema, + RpcCancelSchema, + FileErrorCodeSchema, + FileErrorPayloadSchema, +} from '../../src/index.js'; + +describe('PathSchema', () => { + test('accepts absolute paths', () => { + expect(PathSchema.parse('/foo')).toBe('/foo'); + expect(PathSchema.parse('/foo/bar/baz.txt')).toBe('/foo/bar/baz.txt'); + expect(PathSchema.parse('/')).toBe('/'); + }); + test('rejects relative paths', () => { + expect(() => PathSchema.parse('foo')).toThrow(); + expect(() => PathSchema.parse('./foo')).toThrow(); + }); + test('rejects NUL/CR/LF/DEL/backslash', () => { + expect(() => PathSchema.parse('/foo\x00bar')).toThrow(); + expect(() => PathSchema.parse('/foo\r\n')).toThrow(); + expect(() => PathSchema.parse('/foo\x7f')).toThrow(); + expect(() => PathSchema.parse('/foo\\bar')).toThrow(); + }); + test('rejects empty + over-length', () => { + expect(() => PathSchema.parse('')).toThrow(); + expect(() => PathSchema.parse('/' + 'a'.repeat(4096))).toThrow(); + }); + test('accepts UTF-8 in filenames', () => { + expect(PathSchema.parse('/Документы/файл.txt')).toBe('/Документы/файл.txt'); + expect(PathSchema.parse('/πρόβλημα/αρχείο')).toBe('/πρόβλημα/αρχείο'); + }); +}); + +describe('RequestIdSchema', () => { + test('accepts 22-char base64url', () => { + expect(RequestIdSchema.parse('AbCdEfGhIjKlMnOpQrStUv')).toBe('AbCdEfGhIjKlMnOpQrStUv'); + }); + test('rejects wrong length', () => { + expect(() => RequestIdSchema.parse('AbCd')).toThrow(); + expect(() => RequestIdSchema.parse('AbCdEfGhIjKlMnOpQrStUvWxYz')).toThrow(); + }); + test('rejects non-base64url chars', () => { + expect(() => RequestIdSchema.parse('AbCd/EfGh+IjKlMnOpQrST')).toThrow(); + expect(() => RequestIdSchema.parse('AbCd=EfGhIjKlMnOpQrSTUV')).toThrow(); + }); +}); + +describe('CursorSchema', () => { + test('accepts up to 2048 chars', () => { + expect(CursorSchema.parse('a')).toBe('a'); + expect(CursorSchema.parse('x'.repeat(2048)).length).toBe(2048); + }); + test('rejects empty + over-length', () => { + expect(() => CursorSchema.parse('')).toThrow(); + expect(() => CursorSchema.parse('x'.repeat(2049))).toThrow(); + }); +}); + +describe('Sha256HexSchema', () => { + test('accepts 64 hex chars', () => { + const h = '0'.repeat(64); + expect(Sha256HexSchema.parse(h)).toBe(h); + }); + test('rejects wrong length / non-hex', () => { + expect(() => Sha256HexSchema.parse('0'.repeat(63))).toThrow(); + expect(() => Sha256HexSchema.parse('0'.repeat(65))).toThrow(); + expect(() => Sha256HexSchema.parse('g'.repeat(64))).toThrow(); + expect(() => Sha256HexSchema.parse('A'.repeat(64))).toThrow(); // uppercase rejected + }); +}); + +describe('FileKind / FileEntry', () => { + test('FileKind accepts file/dir', () => { + expect(FileKindSchema.parse('file')).toBe('file'); + expect(FileKindSchema.parse('dir')).toBe('dir'); + expect(() => FileKindSchema.parse('symlink')).toThrow(); + }); + test('FileEntry roundtrip', () => { + const e = { + name: 'foo.txt', + kind: 'file' as const, + size: 1024, + mtime: 1730000000000, + contentType: 'text/plain', + }; + const parsed = FileEntrySchema.parse(e); + expect(parsed.name).toBe('foo.txt'); + expect(parsed.metadata).toEqual({}); // default + }); + test('FileEntry rejects path separators in name', () => { + expect(() => + FileEntrySchema.parse({ name: 'foo/bar', kind: 'file', size: 0, mtime: 0 }), + ).toThrow(); + expect(() => + FileEntrySchema.parse({ name: 'foo\\bar', kind: 'file', size: 0, mtime: 0 }), + ).toThrow(); + }); + test('FileEntry rejects negative size', () => { + expect(() => + FileEntrySchema.parse({ name: 'a', kind: 'file', size: -1, mtime: 0 }), + ).toThrow(); + }); + test('FileEntry passes through metadata', () => { + const parsed = FileEntrySchema.parse({ + name: 'mod.jar', + kind: 'file', + size: 100, + mtime: 0, + metadata: { modrinthId: 'abc', version: '1.0' }, + }); + expect(parsed.metadata).toEqual({ modrinthId: 'abc', version: '1.0' }); + }); +}); + +describe('ListPage', () => { + test('hasMore + nextCursor when more pages', () => { + const p = ListPageSchema.parse({ + entries: [], + hasMore: true, + nextCursor: 'abc', + }); + expect(p.hasMore).toBe(true); + expect(p.nextCursor).toBe('abc'); + }); + test('no nextCursor when hasMore false', () => { + const p = ListPageSchema.parse({ entries: [], hasMore: false }); + expect(p.nextCursor).toBeUndefined(); + }); +}); + +describe('ListArgs', () => { + test('defaults pageSize to 100', () => { + const a = ListArgsSchema.parse({ path: '/foo' }); + expect(a.pageSize).toBe(100); + }); + test('rejects pageSize > 1000', () => { + expect(() => ListArgsSchema.parse({ path: '/foo', pageSize: 1001 })).toThrow(); + }); + test('rejects pageSize < 1', () => { + expect(() => ListArgsSchema.parse({ path: '/foo', pageSize: 0 })).toThrow(); + }); +}); + +describe('StatArgs', () => { + test('requires path', () => { + expect(() => StatArgsSchema.parse({})).toThrow(); + expect(StatArgsSchema.parse({ path: '/foo' }).path).toBe('/foo'); + }); +}); + +describe('Mkdir/Delete/Move args', () => { + test('Mkdir defaults recursive=false', () => { + expect(MkdirArgsSchema.parse({ path: '/a' }).recursive).toBe(false); + }); + test('Delete defaults recursive=false', () => { + expect(DeleteArgsSchema.parse({ path: '/a' }).recursive).toBe(false); + }); + test('Move defaults overwrite=false', () => { + expect( + MoveArgsSchema.parse({ src: '/a', dst: '/b' }).overwrite, + ).toBe(false); + }); +}); + +describe('ReadArgs / ReadResult', () => { + test('Read accepts optional range', () => { + const a = ReadArgsSchema.parse({ path: '/a', range: { start: 0, end: 100 } }); + expect(a.range).toEqual({ start: 0, end: 100 }); + }); + test('ReadResult inline', () => { + const r = ReadResultSchema.parse({ + kind: 'inline', + bytesB64: 'YQ==', + size: 1, + sha256: 'a'.repeat(64), + }); + expect(r.kind).toBe('inline'); + }); + test('ReadResult streams', () => { + const r = ReadResultSchema.parse({ + kind: 'streams', + streamId: 'sid-123', + size: 1024, + sha256: 'b'.repeat(64), + }); + expect(r.kind).toBe('streams'); + }); + test('ReadResult rejects unknown kind', () => { + expect(() => + ReadResultSchema.parse({ kind: 'magic', bytesB64: 'YQ==' }), + ).toThrow(); + }); +}); + +describe('WriteArgs', () => { + test('inline shape', () => { + const w = WriteArgsSchema.parse({ + kind: 'inline', + path: '/foo', + bytesB64: 'YQ==', + }); + expect(w.kind).toBe('inline'); + if (w.kind === 'inline') expect(w.overwrite).toBe(false); + }); + test('streams shape', () => { + const w = WriteArgsSchema.parse({ + kind: 'streams', + path: '/foo', + size: 1000, + writeId: 'AbCdEfGhIjKlMnOpQrStUv', + }); + if (w.kind === 'streams') expect(w.size).toBe(1000); + }); +}); + +describe('GetThumbnailArgs / Result', () => { + test('size must be enum 64/128/256/512', () => { + expect(() => GetThumbnailArgsSchema.parse({ path: '/a', size: 100 })).toThrow(); + expect(GetThumbnailArgsSchema.parse({ path: '/a', size: 64 }).size).toBe(64); + }); + test('format defaults to png', () => { + const a = GetThumbnailArgsSchema.parse({ path: '/a', size: 128 }); + expect(a.format).toBe('png'); + }); + test('Result requires width/height/sha256', () => { + const r = GetThumbnailResultSchema.parse({ + bytesB64: 'YQ==', + format: 'png', + width: 128, + height: 128, + sha256: 'c'.repeat(64), + }); + expect(r.width).toBe(128); + }); +}); + +describe('CustomArgs', () => { + test('accepts dotted op names', () => { + expect( + CustomArgsSchema.parse({ name: 'dispatch.deploy-mod', payload: {} }).name, + ).toBe('dispatch.deploy-mod'); + }); + test('rejects names with spaces or slashes', () => { + expect(() => CustomArgsSchema.parse({ name: 'foo bar', payload: {} })).toThrow(); + expect(() => CustomArgsSchema.parse({ name: 'foo/bar', payload: {} })).toThrow(); + }); +}); + +describe('RPC envelopes', () => { + test('RpcRequest valid', () => { + const env = RpcRequestSchema.parse({ + kind: 'shade.fs.list/v1', + id: 'AbCdEfGhIjKlMnOpQrStUv', + args: { path: '/' }, + sig: 'sig', + signedAt: 1730000000000, + }); + expect(env.kind).toBe('shade.fs.list/v1'); + }); + test('RpcRequest rejects malformed kind', () => { + expect(() => + RpcRequestSchema.parse({ + kind: 'shade.fs.list', + id: 'AbCdEfGhIjKlMnOpQrStUv', + args: {}, + sig: 's', + signedAt: 1, + }), + ).toThrow(); + }); + test('RpcResponse expects .response suffix', () => { + const env = RpcResponseSchema.parse({ + kind: 'shade.fs.list/v1.response', + id: 'AbCdEfGhIjKlMnOpQrStUv', + result: { entries: [], hasMore: false }, + }); + expect(env.id.length).toBe(22); + }); + test('RpcError fixed kind', () => { + const env = RpcErrorSchema.parse({ + kind: 'shade.fs.error/v1', + id: 'AbCdEfGhIjKlMnOpQrStUv', + error: { code: 'NOT_FOUND', message: 'gone' }, + }); + expect(env.error.code).toBe('NOT_FOUND'); + }); + test('RpcCancel fixed kind', () => { + const env = RpcCancelSchema.parse({ + kind: 'shade.fs.cancel/v1', + id: 'AbCdEfGhIjKlMnOpQrStUv', + reason: 'user-cancel', + }); + expect(env.reason).toBe('user-cancel'); + }); +}); + +describe('FileError envelope', () => { + test('all known codes parse', () => { + const codes: ReadonlyArray = [ + 'NOT_FOUND', + 'PERMISSION_DENIED', + 'CONFLICT', + 'QUOTA_EXCEEDED', + 'RATE_LIMIT', + 'PATH_VALIDATION', + 'FINGERPRINT_REQUIRED', + 'OPERATION_TIMEOUT', + 'IDEMPOTENCY_CONFLICT', + 'CANCELLED', + 'INTERNAL', + 'NOT_IMPLEMENTED', + 'CUSTOM_OP_REJECTED', + 'INVALID_SIGNATURE', + 'INVALID_ARGS', + ]; + for (const c of codes) expect(FileErrorCodeSchema.parse(c)).toBe(c); + }); + test('rejects unknown code', () => { + expect(() => FileErrorCodeSchema.parse('NOT_REAL')).toThrow(); + }); + test('FileErrorPayload includes optional fields', () => { + const p = FileErrorPayloadSchema.parse({ + code: 'RATE_LIMIT', + message: 'too fast', + retryAfterMs: 1000, + }); + expect(p.retryAfterMs).toBe(1000); + }); +}); diff --git a/packages/shade-files/tsconfig.json b/packages/shade-files/tsconfig.json new file mode 100644 index 0000000..ba504d9 --- /dev/null +++ b/packages/shade-files/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "jsx": "react" + }, + "include": ["src"] +} diff --git a/packages/shade-observer/package.json b/packages/shade-observer/package.json index ef50abf..f9dbff0 100644 --- a/packages/shade-observer/package.json +++ b/packages/shade-observer/package.json @@ -1,6 +1,6 @@ { "name": "@shade/observer", - "version": "0.1.0", + "version": "0.3.0", "type": "module", "main": "src/index.ts", "types": "src/index.ts", diff --git a/packages/shade-observer/src/state.ts b/packages/shade-observer/src/state.ts index bbc3c9c..ce8707d 100644 --- a/packages/shade-observer/src/state.ts +++ b/packages/shade-observer/src/state.ts @@ -76,10 +76,10 @@ export class StateAggregator { }; constructor( - private readonly clientEvents?: ShadeEventEmitter, - private readonly serverEvents?: PrekeyServerEvents, - private readonly manager?: ShadeSessionManager, - private readonly store?: PrekeyStore, + clientEvents?: ShadeEventEmitter, + serverEvents?: PrekeyServerEvents, + _manager?: ShadeSessionManager, + _store?: PrekeyStore, ) { if (clientEvents) { // Replay any events that fired before subscription, then subscribe diff --git a/packages/shade-proto/package.json b/packages/shade-proto/package.json index eb286c8..c366bb2 100644 --- a/packages/shade-proto/package.json +++ b/packages/shade-proto/package.json @@ -1,6 +1,6 @@ { "name": "@shade/proto", - "version": "0.1.0", + "version": "0.3.0", "type": "module", "main": "src/index.ts", "types": "src/index.ts", diff --git a/packages/shade-proto/src/index.ts b/packages/shade-proto/src/index.ts index 49153f5..eec42d1 100644 --- a/packages/shade-proto/src/index.ts +++ b/packages/shade-proto/src/index.ts @@ -1 +1,11 @@ -export { encodeEnvelope, decodeEnvelope, encodePreKeyMessage, encodeRatchetMessage } from './wire.js'; +export { + encodeEnvelope, + decodeEnvelope, + encodePreKeyMessage, + encodeRatchetMessage, + encodeStreamChunk, + decodeStreamChunk, + inspectEnvelopeType, + TYPE_STREAM_CHUNK, +} from './wire.js'; +export type { StreamChunkWire } from './wire.js'; diff --git a/packages/shade-proto/src/wire.ts b/packages/shade-proto/src/wire.ts index 7256506..56d6a5e 100644 --- a/packages/shade-proto/src/wire.ts +++ b/packages/shade-proto/src/wire.ts @@ -6,17 +6,45 @@ * Types: * 0x01 = PreKeyMessage * 0x02 = RatchetMessage + * 0x11 = StreamChunk * * All multi-byte integers are big-endian. - * All byte arrays are length-prefixed (2-byte length + data). + * + * Length prefixes are 4-byte (u32) since wire VERSION 0x02. (VERSION 0x01 + * used 2-byte/u16 length prefixes which silently truncated payloads larger + * than 64 KiB — a hard correctness ceiling that blocked inline file ops + * up to 256 KiB. The bump is incompatible with 0.2.x peers.) */ import type { PreKeyMessage, RatchetMessage, ShadeEnvelope } from '@shade/core'; -const VERSION = 0x01; +const VERSION = 0x02; const TYPE_PREKEY = 0x01; const TYPE_RATCHET = 0x02; +export const TYPE_STREAM_CHUNK = 0x11; + +// ─── Stream chunk types ────────────────────────────────────── + +/** + * Wire-decoded stream-chunk envelope (type 0x11). + * + * See spec §2.2. The nonce is deterministic — derived from + * (laneId, seq) on both sides — but is also serialized over the wire for + * self-description and validated by the receiver. + */ +export interface StreamChunkWire { + streamId: Uint8Array; // 16 bytes + laneId: number; // u32 + seq: number | bigint; // u64 + isLast: boolean; + nonce: Uint8Array; // 12 bytes + aad: Uint8Array; // additional bound data (length=0 in v0.2.0, reserved) + ciphertext: Uint8Array; // AES-GCM ciphertext including 16-byte tag +} + +const STREAM_ID_BYTES = 16; +const STREAM_NONCE_BYTES = 12; // ─── Encode ────────────────────────────────────────────────── @@ -72,6 +100,59 @@ function encodeRatchetMessageInner(msg: RatchetMessage): Uint8Array { return concat(parts); } +/** + * Encode a stream-chunk envelope to wire bytes (type 0x11). + * + * Layout: see `shade-streams-spec.md` §2.2. + */ +export function encodeStreamChunk(c: StreamChunkWire): Uint8Array { + if (c.streamId.length !== STREAM_ID_BYTES) { + throw new Error(`streamId must be ${STREAM_ID_BYTES} bytes`); + } + if (c.nonce.length !== STREAM_NONCE_BYTES) { + throw new Error(`nonce must be ${STREAM_NONCE_BYTES} bytes`); + } + if (!Number.isInteger(c.laneId) || c.laneId < 0 || c.laneId > 0xffff_ffff) { + throw new Error(`laneId out of u32 range: ${c.laneId}`); + } + const seqBig = typeof c.seq === 'bigint' ? c.seq : BigInt(c.seq); + if (seqBig < 0n || seqBig > 0xffff_ffff_ffff_ffffn) { + throw new Error(`seq out of u64 range: ${c.seq}`); + } + + const headerSize = 1 + 1 + STREAM_ID_BYTES + 4 + 8 + 1 + STREAM_NONCE_BYTES + 4 + c.aad.length + 4; + const out = new Uint8Array(headerSize + c.ciphertext.length); + const view = new DataView(out.buffer); + let offset = 0; + + out[offset++] = VERSION; + out[offset++] = TYPE_STREAM_CHUNK; + out.set(c.streamId, offset); + offset += STREAM_ID_BYTES; + + view.setUint32(offset, c.laneId, false); + offset += 4; + + view.setBigUint64(offset, seqBig, false); + offset += 8; + + out[offset++] = c.isLast ? 0x01 : 0x00; + + out.set(c.nonce, offset); + offset += STREAM_NONCE_BYTES; + + view.setUint32(offset, c.aad.length, false); + offset += 4; + out.set(c.aad, offset); + offset += c.aad.length; + + view.setUint32(offset, c.ciphertext.length, false); + offset += 4; + out.set(c.ciphertext, offset); + + return out; +} + // ─── Decode ────────────────────────────────────────────────── export function decodeEnvelope(data: Uint8Array): ShadeEnvelope { @@ -93,6 +174,80 @@ export function decodeEnvelope(data: Uint8Array): ShadeEnvelope { throw new Error(`Unknown type: ${type}`); } +/** + * Decode a stream-chunk envelope from wire bytes (type 0x11). + * Throws if the data is malformed or the type tag is wrong. + */ +export function decodeStreamChunk(data: Uint8Array): StreamChunkWire { + const minHeaderSize = 2 + STREAM_ID_BYTES + 4 + 8 + 1 + STREAM_NONCE_BYTES + 4 + 4; + if (data.length < minHeaderSize) { + throw new Error(`stream-chunk too short: ${data.length} < ${minHeaderSize}`); + } + if (data[0] !== VERSION) { + throw new Error(`Unknown version: ${data[0]}`); + } + if (data[1] !== TYPE_STREAM_CHUNK) { + throw new Error(`Not a stream-chunk: type=${data[1]}`); + } + + const view = new DataView(data.buffer, data.byteOffset); + let offset = 2; + + const streamId = data.slice(offset, offset + STREAM_ID_BYTES); + offset += STREAM_ID_BYTES; + + const laneId = view.getUint32(offset, false); + offset += 4; + + const seq = view.getBigUint64(offset, false); + offset += 8; + + const isLast = data[offset] === 0x01; + offset += 1; + + const nonce = data.slice(offset, offset + STREAM_NONCE_BYTES); + offset += STREAM_NONCE_BYTES; + + const aadLen = view.getUint32(offset, false); + offset += 4; + if (offset + aadLen + 4 > data.length) { + throw new Error('stream-chunk truncated in aad/ctLen'); + } + const aad = data.slice(offset, offset + aadLen); + offset += aadLen; + + const ctLen = view.getUint32(offset, false); + offset += 4; + if (offset + ctLen !== data.length) { + throw new Error( + `stream-chunk length mismatch: declared ${offset + ctLen}, actual ${data.length}`, + ); + } + const ciphertext = data.slice(offset, offset + ctLen); + + return { streamId, laneId, seq, isLast, nonce, aad, ciphertext }; +} + +/** + * Inspect the type tag of an arbitrary envelope without full parsing. + * Returns one of `'prekey' | 'ratchet' | 'stream-chunk' | 'unknown'`. + */ +export function inspectEnvelopeType( + data: Uint8Array, +): 'prekey' | 'ratchet' | 'stream-chunk' | 'unknown' { + if (data.length < 2 || data[0] !== VERSION) return 'unknown'; + switch (data[1]) { + case TYPE_PREKEY: + return 'prekey'; + case TYPE_RATCHET: + return 'ratchet'; + case TYPE_STREAM_CHUNK: + return 'stream-chunk'; + default: + return 'unknown'; + } +} + function decodePreKeyMessageInner(data: Uint8Array): PreKeyMessage { let offset = 0; @@ -107,14 +262,15 @@ function decodePreKeyMessageInner(data: Uint8Array): PreKeyMessage { const ratchet = decodeRatchetMessageInner(ratchetData.value, 0); - return { + const msg: PreKeyMessage = { registrationId, - preKeyId, signedPreKeyId, ephemeralKey: ephemeral.value, identityDHKey: identityDH.value, message: ratchet.value, }; + if (preKeyId !== undefined) msg.preKeyId = preKeyId; + return msg; } function decodeRatchetMessageInner(data: Uint8Array, offset: number): { value: RatchetMessage; end: number } { @@ -145,8 +301,8 @@ function uint32(n: number): Uint8Array { } function lpBytes(data: Uint8Array): Uint8Array { - const len = new Uint8Array(2); - new DataView(len.buffer).setUint16(0, data.length, false); + const len = new Uint8Array(4); + new DataView(len.buffer).setUint32(0, data.length, false); return concat([len, data]); } @@ -155,9 +311,9 @@ function readUint32(data: Uint8Array, offset: number): number { } function readLP(data: Uint8Array, offset: number): { value: Uint8Array; end: number } { - const len = new DataView(data.buffer, data.byteOffset + offset).getUint16(0, false); - const value = data.slice(offset + 2, offset + 2 + len); - return { value, end: offset + 2 + len }; + const len = new DataView(data.buffer, data.byteOffset + offset).getUint32(0, false); + const value = data.slice(offset + 4, offset + 4 + len); + return { value, end: offset + 4 + len }; } function concat(parts: Uint8Array[]): Uint8Array { diff --git a/packages/shade-proto/tests/wire.test.ts b/packages/shade-proto/tests/wire.test.ts index b9d6d36..c12b22e 100644 --- a/packages/shade-proto/tests/wire.test.ts +++ b/packages/shade-proto/tests/wire.test.ts @@ -1,5 +1,14 @@ import { describe, test, expect } from 'bun:test'; -import { encodeEnvelope, decodeEnvelope, encodePreKeyMessage, encodeRatchetMessage } from '../src/index.js'; +import { + encodeEnvelope, + decodeEnvelope, + encodeRatchetMessage, + encodeStreamChunk, + decodeStreamChunk, + inspectEnvelopeType, + TYPE_STREAM_CHUNK, +} from '../src/index.js'; +import type { StreamChunkWire } from '../src/index.js'; import type { PreKeyMessage, RatchetMessage, ShadeEnvelope } from '@shade/core'; function randBytes(n: number): Uint8Array { @@ -173,13 +182,124 @@ describe('Wire Format', () => { }); test('rejects unknown type', () => { - const data = new Uint8Array([0x01, 0xFF]); + const data = new Uint8Array([0x02, 0xFF]); expect(() => decodeEnvelope(data)).toThrow('Unknown type'); }); test('rejects too-short data', () => { - expect(() => decodeEnvelope(new Uint8Array([0x01]))).toThrow('Too short'); + expect(() => decodeEnvelope(new Uint8Array([0x02]))).toThrow('Too short'); expect(() => decodeEnvelope(new Uint8Array([]))).toThrow('Too short'); }); }); + + // ─── StreamChunk (type 0x11) ─────────────────────────────── + + describe('StreamChunk', () => { + function makeChunk(overrides: Partial = {}): StreamChunkWire { + return { + streamId: randBytes(16), + laneId: 0, + seq: 0n, + isLast: false, + nonce: randBytes(12), + aad: new Uint8Array(0), + ciphertext: randBytes(64), + ...overrides, + }; + } + + test('encode/decode roundtrip', () => { + const c = makeChunk({ laneId: 7, seq: 42n, isLast: true }); + const encoded = encodeStreamChunk(c); + const decoded = decodeStreamChunk(encoded); + expect(decoded.streamId).toEqual(c.streamId); + expect(decoded.laneId).toBe(7); + expect(decoded.seq).toBe(42n); + expect(decoded.isLast).toBe(true); + expect(decoded.nonce).toEqual(c.nonce); + expect(decoded.aad.length).toBe(0); + expect(decoded.ciphertext).toEqual(c.ciphertext); + }); + + test('emits the correct type tag', () => { + const encoded = encodeStreamChunk(makeChunk()); + expect(encoded[0]).toBe(0x02); // version + expect(encoded[1]).toBe(TYPE_STREAM_CHUNK); + }); + + test('handles empty ciphertext', () => { + const c = makeChunk({ ciphertext: new Uint8Array(0) }); + const decoded = decodeStreamChunk(encodeStreamChunk(c)); + expect(decoded.ciphertext.length).toBe(0); + }); + + test('handles 16 MiB ciphertext (max chunk + tag)', () => { + const c = makeChunk({ ciphertext: randBytes(16 * 1024 * 1024 + 16) }); + const encoded = encodeStreamChunk(c); + const decoded = decodeStreamChunk(encoded); + expect(decoded.ciphertext.length).toBe(c.ciphertext.length); + }); + + test('handles MAX_SEQ', () => { + const max = 0xffff_ffff_ffff_ffffn; + const c = makeChunk({ seq: max }); + const decoded = decodeStreamChunk(encodeStreamChunk(c)); + expect(decoded.seq).toBe(max); + }); + + test('rejects wrong-length streamId', () => { + expect(() => encodeStreamChunk(makeChunk({ streamId: randBytes(15) }))).toThrow(); + }); + + test('rejects wrong-length nonce', () => { + expect(() => encodeStreamChunk(makeChunk({ nonce: randBytes(11) }))).toThrow(); + }); + + test('rejects out-of-range laneId', () => { + expect(() => encodeStreamChunk(makeChunk({ laneId: 0x1_0000_0000 }))).toThrow(); + expect(() => encodeStreamChunk(makeChunk({ laneId: -1 }))).toThrow(); + }); + + test('decode rejects wrong type tag', () => { + const valid = encodeStreamChunk(makeChunk()); + const tampered = valid.slice(); + tampered[1] = 0x02; // ratchet + expect(() => decodeStreamChunk(tampered)).toThrow(); + }); + + test('decode rejects wrong version', () => { + const valid = encodeStreamChunk(makeChunk()); + const tampered = valid.slice(); + tampered[0] = 0x01; // bumped to 0x02 in v0.3.0; 0x01 is the legacy version + expect(() => decodeStreamChunk(tampered)).toThrow(); + }); + + test('decode rejects truncated body', () => { + const valid = encodeStreamChunk(makeChunk()); + expect(() => decodeStreamChunk(valid.slice(0, valid.length - 5))).toThrow(); + }); + + test('decode rejects body extended beyond ctLen', () => { + const valid = encodeStreamChunk(makeChunk()); + const tampered = new Uint8Array(valid.length + 1); + tampered.set(valid); + expect(() => decodeStreamChunk(tampered)).toThrow(); + }); + + test('inspectEnvelopeType identifies stream-chunk', () => { + expect(inspectEnvelopeType(encodeStreamChunk(makeChunk()))).toBe('stream-chunk'); + }); + + test('inspectEnvelopeType identifies ratchet/prekey/unknown', () => { + const ratchet: ShadeEnvelope = { + type: 'ratchet', + content: makeRatchetMessage(), + timestamp: 0, + senderAddress: '', + }; + expect(inspectEnvelopeType(encodeEnvelope(ratchet))).toBe('ratchet'); + expect(inspectEnvelopeType(new Uint8Array([0x02, 0x99]))).toBe('unknown'); + expect(inspectEnvelopeType(new Uint8Array([]))).toBe('unknown'); + }); + }); }); diff --git a/packages/shade-sdk/package.json b/packages/shade-sdk/package.json index 8f56095..385231f 100644 --- a/packages/shade-sdk/package.json +++ b/packages/shade-sdk/package.json @@ -1,16 +1,19 @@ { "name": "@shade/sdk", - "version": "0.1.0", + "version": "0.3.0", "type": "module", "main": "src/index.ts", "types": "src/index.ts", "dependencies": { "@shade/core": "workspace:*", "@shade/crypto-web": "workspace:*", - "@shade/storage-sqlite": "workspace:*", - "@shade/server": "workspace:*", - "@shade/transport": "workspace:*", + "@shade/files": "workspace:*", "@shade/observer": "workspace:*", - "@shade/proto": "workspace:*" + "@shade/proto": "workspace:*", + "@shade/server": "workspace:*", + "@shade/storage-sqlite": "workspace:*", + "@shade/streams": "workspace:*", + "@shade/transfer": "workspace:*", + "@shade/transport": "workspace:*" } } diff --git a/packages/shade-sdk/src/background.ts b/packages/shade-sdk/src/background.ts index 154f2e7..5b84ad6 100644 --- a/packages/shade-sdk/src/background.ts +++ b/packages/shade-sdk/src/background.ts @@ -16,14 +16,29 @@ export interface BackgroundHooks { onReplenish?: (count: number, total: number) => void; /** Called after identity rotation completes and new bundle is published */ onRotate?: () => void; + /** + * Called periodically to prune `@shade/files` retention state (idempotency + * cache, parked transfers). Wired by `Shade.files.serve()`. Synchronous — + * heavy retention work should self-yield. + */ + onPruneFiles?: () => void; /** Called if a background task throws */ - onError?: (error: Error, task: 'replenish' | 'rotate') => void; + onError?: (error: Error, task: 'replenish' | 'rotate' | 'prune-files') => void; +} + +/** Options forwarded to `BackgroundTasks` (in addition to `ResolvedConfig`). */ +export interface BackgroundOptions { + hooks?: BackgroundHooks; + /** Override the file-prune interval. Default 5 min. */ + pruneFilesIntervalMs?: number; } export class BackgroundTasks { private replenishTimer: ReturnType | null = null; private rotateTimer: ReturnType | null = null; + private pruneFilesTimer: ReturnType | null = null; private running = false; + private readonly pruneFilesIntervalMs: number; constructor( private readonly manager: ShadeSessionManager, @@ -31,7 +46,21 @@ export class BackgroundTasks { private readonly address: string, private readonly config: ResolvedConfig, private readonly hooks: BackgroundHooks = {}, - ) {} + options: BackgroundOptions = {}, + ) { + this.pruneFilesIntervalMs = options.pruneFilesIntervalMs ?? 5 * 60 * 1000; + } + + /** + * Update or replace background hooks at runtime. Used by + * `Shade.files.serve()` to register `onPruneFiles` after construction. + */ + setHook(name: K, fn: BackgroundHooks[K]): void { + (this.hooks as BackgroundHooks)[name] = fn; + if (name === 'onPruneFiles' && this.running && fn !== undefined) { + this.startPruneFilesTimer(); + } + } start(): void { if (this.running) return; @@ -51,6 +80,23 @@ export class BackgroundTasks { if (this.config.autoRotate) { this.scheduleNextRotation(); } + + // Files retention timer + if (this.hooks.onPruneFiles !== undefined) { + this.startPruneFilesTimer(); + } + } + + private startPruneFilesTimer(): void { + if (this.pruneFilesTimer !== null) clearInterval(this.pruneFilesTimer); + this.pruneFilesTimer = setInterval(() => { + try { + this.hooks.onPruneFiles?.(); + } catch (err) { + this.hooks.onError?.(err as Error, 'prune-files'); + } + }, this.pruneFilesIntervalMs); + this.pruneFilesTimer.unref?.(); } stop(): void { @@ -63,6 +109,10 @@ export class BackgroundTasks { clearTimeout(this.rotateTimer); this.rotateTimer = null; } + if (this.pruneFilesTimer) { + clearInterval(this.pruneFilesTimer); + this.pruneFilesTimer = null; + } } /** @@ -82,12 +132,11 @@ export class BackgroundTasks { // we instead ask the manager to expose them. For now, we just re-publish // a fresh bundle + upload everything fresh. try { - const newKeys = await this.manager.generateOneTimePreKeys(0); // no-op to keep types - // Fetch all current one-time prekeys via the storage and upload - // (the manager doesn't expose them directly; we work around by using the - // public newly-generated array returned above, but that was empty.) - // TODO: improve ShadeSessionManager to expose recent prekeys for re-upload. - // For now, simply log — correct upload will be handled on next rotate. + // No-op call to keep the manager warm. The proper re-upload path is + // exposed once ShadeSessionManager surfaces newly-generated prekeys + // (TODO: improve manager API). Until then, correct upload is handled + // on the next rotate. + await this.manager.generateOneTimePreKeys(0); } catch { // ignore } diff --git a/packages/shade-sdk/src/backup.ts b/packages/shade-sdk/src/backup.ts index c49bbd1..b93adb2 100644 --- a/packages/shade-sdk/src/backup.ts +++ b/packages/shade-sdk/src/backup.ts @@ -1,4 +1,4 @@ -import type { CryptoProvider, StorageProvider, IdentityKeyPair, SignedPreKey, OneTimePreKey, SessionState } from '@shade/core'; +import type { CryptoProvider, StorageProvider } from '@shade/core'; import { toBase64, fromBase64, diff --git a/packages/shade-sdk/src/config.ts b/packages/shade-sdk/src/config.ts index dcb11b0..9ef9e98 100644 --- a/packages/shade-sdk/src/config.ts +++ b/packages/shade-sdk/src/config.ts @@ -24,9 +24,9 @@ export interface ShadeConfig { /** * Your address on the prekey server (e.g. "alice@example.com" or "device:abc123"). - * If omitted, a random UUID is generated and persisted. + * If omitted (undefined), a random UUID is generated and persisted. */ - address?: string; + address?: string | undefined; /** * Auto-replenish configuration. When the one-time prekey stock drops @@ -61,12 +61,12 @@ export interface ShadeConfig { export interface ResolvedConfig { prekeyServer: string; storage: string | StorageProvider | { type: 'postgres'; url: string }; - address?: string; + address?: string | undefined; autoReplenish: { min: number; target: number; intervalMs: number } | false; autoRotate: false | '1d' | '7d' | '30d' | '90d'; observer?: { token: string; - port?: number; + port?: number | undefined; basePath: string; }; } diff --git a/packages/shade-sdk/src/index.ts b/packages/shade-sdk/src/index.ts index b2d334a..b79ce30 100644 --- a/packages/shade-sdk/src/index.ts +++ b/packages/shade-sdk/src/index.ts @@ -11,3 +11,47 @@ export { export type { ShadeConfig, ResolvedConfig } from './config.js'; export type { BackgroundHooks } from './background.js'; export type { BackupBlob, BackupPayload } from './backup.js'; + +// ─── Stream transfers (v0.2.0) ───────────────────────────── +export { + ShadeControlChannel, + ShadeTransferAuthenticator, + canonicalChunkBytes, + canonicalControlBytes, +} from './streams-bridge.js'; +export type { ControlEnvelopeTransport } from './streams-bridge.js'; +export { + TransferEngine, + MemoryControlChannel, + MemoryTransferTransport, + ShadeTransferHttpTransport, + createTransferRoutes, + PermissiveAuthenticator, + TransferError, + TransferAbortError, + TransferIntegrityError, + TransferProtocolError, + TransferOfflineError, + TransferResumeError, +} from '@shade/transfer'; +export type { + TransferOptions, + TransferProgress, + TransferEvent, + TransferHandle, + TransferResult, + TransferInput, + TransferOutput, + IncomingTransfer, + IncomingTransferAcceptOptions, + TransferSummary, + TransferRouteOptions, + TransferRouteAuthenticator, + ITransferTransport, + IControlChannel, + TransferAuthenticator, + ChunkAck, + TransferResumeState, + LaneProgress, +} from '@shade/transfer'; +export type { StreamMetadata, LaneInitSpec, LanePartition } from '@shade/streams'; diff --git a/packages/shade-sdk/src/shade.ts b/packages/shade-sdk/src/shade.ts index da35744..4056aec 100644 --- a/packages/shade-sdk/src/shade.ts +++ b/packages/shade-sdk/src/shade.ts @@ -5,10 +5,27 @@ import { NoSessionError, } from '@shade/core'; import { SubtleCryptoProvider, MemoryStorage } from '@shade/crypto-web'; +import { encodeEnvelope, decodeEnvelope, inspectEnvelopeType } from '@shade/proto'; import { ShadeFetchTransport } from '@shade/transport'; -import { BackgroundTasks, type BackgroundHooks } from './background.js'; -import { exportBackup, importBackup, backupToString, backupFromString, type BackupBlob } from './backup.js'; +import { + TransferEngine, + ShadeTransferHttpTransport, + type ITransferTransport, + type IncomingTransfer, + type TransferHandle, + type TransferOptions, + type TransferSummary, +} from '@shade/transfer'; +import type { Hono } from 'hono'; +import { BackgroundTasks } from './background.js'; +import { exportBackup, importBackup, backupToString, backupFromString } from './backup.js'; import type { ResolvedConfig } from './config.js'; +import { + ShadeControlChannel, + ShadeTransferAuthenticator, + type ControlEnvelopeTransport, +} from './streams-bridge.js'; +import { createFilesNamespace, type FilesNamespace } from '@shade/files'; /** * The high-level Shade API. @@ -23,7 +40,7 @@ export class Shade { private storage!: StorageProvider; private manager!: ShadeSessionManager; private transport!: ShadeFetchTransport; - private background: BackgroundTasks | null = null; + private _background: BackgroundTasks | null = null; private address!: string; private initialized = false; @@ -32,8 +49,19 @@ export class Shade { // Per-address encrypt queue to serialize ratchet mutations private encryptChains = new Map>(); - // Message handlers - private messageHandlers: Array<(from: string, plaintext: string) => void> = []; + // Message handlers — may be sync or async; receive() awaits each. + private messageHandlers: Array< + (from: string, plaintext: string) => void | Promise + > = []; + + // Stream-transfer engine, lazily constructed on first use. + private transferEngine: TransferEngine | null = null; + private controlChannel: ShadeControlChannel | null = null; + private peerBaseUrlResolver: ((peerAddress: string) => Promise) | null = null; + private envelopeOutboxes: ControlEnvelopeTransport | null = null; + + // `@shade/files` namespace, lazy + memoized. + private filesNamespace: FilesNamespace | null = null; constructor(private readonly config: ResolvedConfig) {} @@ -88,13 +116,13 @@ export class Shade { } // Step 6: Background tasks - this.background = new BackgroundTasks( + this._background = new BackgroundTasks( this.manager, this.transport, this.address, this.config, ); - this.background.start(); + this._background.start(); this.initialized = true; } @@ -111,6 +139,35 @@ export class Shade { return this.address; } + /** + * `@shade/files` namespace — high-level entry point for E2EE filesystem + * RPC. Lazily creates the underlying channel + streams bridges on first + * access; subsequent accesses return the same instance. + * + * ```ts + * const files = shade.files; + * const stop = await files.serve({ list: ..., write: ..., ... }); + * const fs = await files.client('bob'); + * await fs.list('/'); + * ``` + * + * Requires `configureTransfers({ resolveBaseUrl })` to be called first + * (same as `upload`/`onIncomingTransfer`). + */ + get files(): FilesNamespace { + if (!this.initialized) throw new Error('Not initialized'); + if (this.filesNamespace !== null) return this.filesNamespace; + // `@shade/files` only imports `Shade` as a type, so the cyclic ESM + // import is type-only at the value layer — safe to bind synchronously. + this.filesNamespace = createFilesNamespace(this); + return this.filesNamespace; + } + + /** Internal — exposes the BackgroundTasks for `@shade/files` to wire prune. */ + get background(): BackgroundTasks | null { + return this._background; + } + /** Access the underlying event emitter (for observer integration) */ getEvents(): ShadeEventEmitter { return this.events; @@ -164,7 +221,7 @@ export class Shade { const plaintext = await this.manager.decrypt(from, envelope); for (const handler of this.messageHandlers) { try { - handler(from, plaintext); + await handler(from, plaintext); } catch (err) { console.error('[Shade] Message handler threw:', err); } @@ -172,8 +229,10 @@ export class Shade { return plaintext; } - /** Register a handler for incoming messages */ - onMessage(handler: (from: string, plaintext: string) => void): () => void { + /** Register a handler for incoming messages. Async handlers are awaited. */ + onMessage( + handler: (from: string, plaintext: string) => void | Promise, + ): () => void { this.messageHandlers.push(handler); return () => { this.messageHandlers = this.messageHandlers.filter((h) => h !== handler); @@ -218,23 +277,23 @@ export class Shade { ); // Rebuild background tasks so they use the new transport - if (this.background) { - this.background.stop(); - this.background = new BackgroundTasks( + if (this._background) { + this._background.stop(); + this._background = new BackgroundTasks( this.manager, this.transport, this.address, this.config, ); - this.background.start(); + this._background.start(); } } /** Manually trigger replenishment (normally background task handles this) */ async replenish(): Promise { if (!this.initialized) throw new Error('Not initialized'); - if (!this.background) return 0; - return this.background.runReplenish(); + if (!this._background) return 0; + return this._background.runReplenish(); } /** @@ -273,7 +332,9 @@ export class Shade { /** Clean shutdown: stop timers, close storage if it supports it */ async shutdown(): Promise { - this.background?.stop(); + this._background?.stop(); + if (this.transferEngine !== null) this.transferEngine.destroy(); + if (this.controlChannel !== null) this.controlChannel.destroy(); // Close storage if it has a close method (SQLite) const closable = this.storage as unknown as { close?: () => void | Promise }; if (typeof closable.close === 'function') { @@ -282,8 +343,166 @@ export class Shade { this.initialized = false; } + // ─── Stream transfers (v0.2.0) ───────────────────────────── + + /** + * Configure how stream-transfer chunks reach peers. Provide a resolver + * that returns the peer's HTTP base URL (e.g. by looking up a + * `transfer.baseUrl` field in your prekey-bundle metadata or a static + * directory map). If unset, `upload()` rejects with a clear error. + * + * Optionally also override the control-envelope transport (defaults to + * HTTP POSTs to `/v1/transfer/control`). + */ + configureTransfers(opts: { + resolveBaseUrl: (peerAddress: string) => Promise; + envelopeTransport?: ControlEnvelopeTransport; + }): void { + this.peerBaseUrlResolver = opts.resolveBaseUrl; + this.envelopeOutboxes = + opts.envelopeTransport ?? new HttpEnvelopeTransport(opts.resolveBaseUrl, this.address); + } + + /** + * Deliver a freshly-encrypted ratchet envelope to a peer using the + * configured envelope transport (HTTP POST to `/v1/transfer/control` by + * default). Used by `@shade/files` for RPC plaintext delivery. + */ + async deliverControlEnvelope(peerAddress: string, envelope: ShadeEnvelope): Promise { + if (this.envelopeOutboxes === null) { + throw new Error( + 'Call shade.configureTransfers({ resolveBaseUrl }) before deliverControlEnvelope()', + ); + } + await this.envelopeOutboxes.send(peerAddress, envelope); + } + + /** + * Upload bytes to a peer. Returns a `TransferHandle` that can be paused/ + * aborted and awaited. Requires `configureTransfers` to be called first. + */ + async upload(opts: TransferOptions): Promise { + return (await this.engine()).upload(opts); + } + + /** + * Subscribe to incoming transfers from peers. Handler is invoked when a + * `stream-init` arrives; the handler MUST call `incoming.accept(...)` to + * begin receiving (or `incoming.decline(...)` to reject). + */ + async onIncomingTransfer( + handler: (incoming: IncomingTransfer) => void | Promise, + ): Promise<() => void> { + return (await this.engine()).onIncomingTransfer(handler); + } + + /** + * Mount the receiver-side HTTP routes on a Hono app. Mount under any + * base path: `app.route('/shade', await shade.transferRoute())`. + * + * Routes: + * POST /v1/transfer/:streamId/chunk — wire-encoded 0x11 chunks + * GET /v1/transfer/:streamId/state — resume-state lookup + * POST /v1/transfer/control — wire-encoded 0x02 control envelopes + * GET /v1/transfer/health — peer reachability probe + */ + async transferRoute(): Promise { + const engine = await this.engine(); + const { createTransferRoutes, PermissiveAuthenticator } = await import('@shade/transfer'); + const app = await createTransferRoutes(engine, { + authenticator: PermissiveAuthenticator, + }); + // Add the control-envelope POST route on top. + app.post('/v1/transfer/control', async (c) => { + const senderAddress = c.req.header('X-Shade-Sender-Address'); + if (senderAddress === undefined || senderAddress === '') { + return c.json({ error: 'missing X-Shade-Sender-Address' }, 400); + } + const ab = await c.req.arrayBuffer(); + const bytes = new Uint8Array(ab); + try { + await this.acceptTransferEnvelope(senderAddress, bytes); + } catch (err) { + return c.json({ error: (err as Error).message }, 400); + } + return c.json({ ok: true }); + }); + return app; + } + + /** + * Low-level entry for custom transports: hand a `0x02` ratchet envelope + * (control-plane) or a `0x11` stream-chunk envelope to the engine. + * Used internally by `transferRoute()`. + */ + async acceptTransferEnvelope(from: string, env: ShadeEnvelope | Uint8Array): Promise { + const engine = await this.engine(); + if (env instanceof Uint8Array) { + const kind = inspectEnvelopeType(env); + if (kind === 'stream-chunk') { + // Engine extracts laneId/seq from the wire bytes via decodeStreamChunk. + const headers = parseChunkHeader(env); + await engine.receiveChunk(from, headers.streamId, headers.laneId, headers.seq, env); + return; + } + if (kind === 'ratchet' || kind === 'prekey') { + const decoded = decodeEnvelope(env); + await this.controlChannel!.acceptEnvelope(from, decoded); + return; + } + throw new Error(`Unknown envelope type ${kind}`); + } + // Already-decoded envelope (ratchet or prekey) + await this.controlChannel!.acceptEnvelope(from, env); + } + // ─── Internals ───────────────────────────────────────────── + private async engine(): Promise { + if (this.transferEngine !== null) return this.transferEngine; + if (this.peerBaseUrlResolver === null || this.envelopeOutboxes === null) { + throw new Error( + 'Call shade.configureTransfers({ resolveBaseUrl }) before using upload()/onIncomingTransfer()', + ); + } + this.controlChannel = new ShadeControlChannel(this, this.envelopeOutboxes); + const transport: ITransferTransport = new ShadeTransferHttpTransport({ + resolveBaseUrl: this.peerBaseUrlResolver, + authenticator: await this.makeAuthenticator(), + }); + this.transferEngine = new TransferEngine({ + crypto: this.crypto, + controlChannel: this.controlChannel, + transport, + myAddress: this.address, + }); + return this.transferEngine; + } + + private async makeAuthenticator(): Promise { + const identity = await this.storage.getIdentityKeyPair(); + if (identity === null) throw new Error('Identity not initialized'); + return new ShadeTransferAuthenticator(this.crypto, this.address, identity.signingPrivateKey); + } + + /** Returns a list of in-flight stream transfers from storage (resume support). */ + async listTransfers(filter?: { + direction?: 'send' | 'receive'; + }): Promise { + if (this.storage.listActiveStreamStates === undefined) return []; + const rows = await this.storage.listActiveStreamStates(filter?.direction); + return rows.map((s) => ({ + streamId: s.streamId, + direction: s.direction, + peerAddress: s.peerAddress, + status: s.status, + bytesProcessed: 0, // computed from laneState + createdAt: s.createdAt, + updatedAt: s.updatedAt, + metadata: tryParseMetadata(s.metadataJson), + })); + } + private async ensureSession(address: string): Promise { // Deduplicate concurrent establishment requests const existing = this.establishing.get(address); @@ -306,6 +525,57 @@ export class Shade { } } +function tryParseMetadata(json: string): import('@shade/streams').StreamMetadata | null { + try { + return JSON.parse(json) as import('@shade/streams').StreamMetadata; + } catch { + return null; + } +} + +function parseChunkHeader(bytes: Uint8Array): { + streamId: string; + laneId: number; + seq: bigint; +} { + // [0]=ver [1]=type [2..18]=streamId(16) [18..22]=laneId u32 [22..30]=seq u64 + if (bytes.length < 30) throw new Error('truncated stream-chunk header'); + const view = new DataView(bytes.buffer, bytes.byteOffset); + const sidBytes = bytes.slice(2, 18); + const laneId = view.getUint32(18, false); + const seq = view.getBigUint64(22, false); + // Encode streamId as base64url + let bin = ''; + for (let i = 0; i < sidBytes.length; i++) bin += String.fromCharCode(sidBytes[i]!); + const streamId = btoa(bin).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); + return { streamId, laneId, seq }; +} + +// ─── Default HTTP envelope transport ────────────────────────── + +class HttpEnvelopeTransport implements ControlEnvelopeTransport { + constructor( + private readonly resolveBaseUrl: (peerAddress: string) => Promise, + private readonly myAddress: string, + ) {} + async send(peerAddress: string, envelope: ShadeEnvelope): Promise { + const base = (await this.resolveBaseUrl(peerAddress)).replace(/\/$/, ''); + const url = `${base}/v1/transfer/control`; + const bytes = encodeEnvelope(envelope); + const res = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/octet-stream', + 'X-Shade-Sender-Address': this.myAddress, + }, + body: bytes as unknown as never, + }); + if (!res.ok) { + throw new Error(`control envelope POST failed: ${res.status} ${await res.text()}`); + } + } +} + // ─── Helpers ───────────────────────────────────────────────── async function resolveStorage( @@ -326,8 +596,14 @@ async function resolveStorage( } if (typeof spec === 'object' && spec.type === 'postgres') { - const { PostgresStorage } = await import('@shade/storage-postgres'); - return PostgresStorage.create(spec.url); + // Dynamic import keeps @shade/storage-postgres optional — consumers that + // never use postgres don't need to install it. The string-form import + // path makes the resolver lazy at type-check time too. + const moduleId = '@shade/storage-postgres'; + const mod = (await import(moduleId)) as { + PostgresStorage: { create(url: string): Promise }; + }; + return mod.PostgresStorage.create(spec.url); } throw new Error(`Unsupported storage spec: ${JSON.stringify(spec)}`); diff --git a/packages/shade-sdk/src/streams-bridge.ts b/packages/shade-sdk/src/streams-bridge.ts new file mode 100644 index 0000000..af163aa --- /dev/null +++ b/packages/shade-sdk/src/streams-bridge.ts @@ -0,0 +1,215 @@ +import type { CryptoProvider, ShadeEnvelope } from '@shade/core'; +import { ValidationError } from '@shade/core'; +import { + encodeStreamControl, + parseStreamControl, + type StreamControlMessage, +} from '@shade/streams'; +import { + type IControlChannel, + type TransferAuthenticator, +} from '@shade/transfer'; +import type { Shade } from './shade.js'; + +/** + * Adapter contract for shipping ratchet envelopes between two Shade + * instances. The SDK provides an HTTP implementation that POSTs to the + * peer's `transferRoute()` control endpoint; tests use a memory pair. + */ +export interface ControlEnvelopeTransport { + send(peerAddress: string, envelope: ShadeEnvelope): Promise; +} + +/** + * `IControlChannel` implementation that rides on top of an existing Shade + * instance's `send`/`receive` API. Each control message is JSON-encoded + * plaintext and shipped through the Double Ratchet — the same path as + * regular chat messages. + * + * Receiver-side: the SDK installs a `Shade.onMessage` handler at + * construction. Plaintext is parsed for the `shade.stream-` `kind` prefix; + * matches are dispatched to subscribed handlers, non-matches are passed + * through to the consumer's regular onMessage handlers via `passthrough`. + */ +export class ShadeControlChannel implements IControlChannel { + private readonly handlers = new Set< + (from: string, message: StreamControlMessage) => void | Promise + >(); + private readonly unsubscribeShadeMessages: () => void; + + constructor( + private readonly shade: Shade, + private readonly envelopeTransport: ControlEnvelopeTransport, + private readonly passthrough?: (from: string, plaintext: string) => void, + ) { + this.unsubscribeShadeMessages = shade.onMessage(async (from, plaintext) => { + if (!plaintext.includes('shade.stream-')) { + this.passthrough?.(from, plaintext); + return; + } + let msg: StreamControlMessage; + try { + msg = parseStreamControl(plaintext); + } catch { + // Not a stream control message; treat as ordinary plaintext. + this.passthrough?.(from, plaintext); + return; + } + // Await each handler. The engine relies on this for sequencing: + // sender's `controlChannel.send` only resolves once the receiver has + // fully processed `stream-init`, so chunks never race ahead. + for (const handler of [...this.handlers]) { + try { + await handler(from, msg); + } catch (err) { + console.error('[ShadeControlChannel] handler error:', err); + } + } + }); + } + + async send(peerAddress: string, message: StreamControlMessage): Promise { + const envelope = await this.shade.send(peerAddress, encodeStreamControl(message)); + await this.envelopeTransport.send(peerAddress, envelope); + } + + /** + * Receive-side: hand a freshly-decrypted envelope from the peer to this + * channel. The plaintext is fed through the same `onMessage` pipeline. + * Used by `Shade.acceptTransferEnvelope` for control-plane messages. + */ + async acceptEnvelope(from: string, envelope: ShadeEnvelope): Promise { + await this.shade.receive(from, envelope); + } + + onMessage( + handler: (from: string, message: StreamControlMessage) => void | Promise, + ): () => void { + this.handlers.add(handler); + return () => this.handlers.delete(handler); + } + + destroy(): void { + this.unsubscribeShadeMessages(); + this.handlers.clear(); + } +} + +/** + * Ed25519-signing authenticator for `ShadeTransferHttpTransport`. + * + * Header format on `/chunk`: + * `X-Shade-Sender-Address:
` + * `X-Shade-Signed-At: ` + * `X-Shade-Signature: ` + * + * The receiver-side `ShadeTransferRouteAuthenticator` (below) verifies this + * using the sender's public identity key fetched from the prekey server. + */ +export class ShadeTransferAuthenticator implements TransferAuthenticator { + constructor( + private readonly crypto: CryptoProvider, + private readonly senderAddress: string, + private readonly signingPrivateKey: Uint8Array, + ) { + if (signingPrivateKey.length !== 32) { + throw new ValidationError('signingPrivateKey must be 32 bytes', 'signingPrivateKey'); + } + } + + async signChunk(args: { + streamId: string; + laneId: number; + seq: bigint; + bodyHash: Uint8Array; + }): Promise> { + const signedAt = Date.now(); + const message = canonicalChunkBytes({ + address: this.senderAddress, + signedAt, + streamId: args.streamId, + laneId: args.laneId, + seq: args.seq, + bodyHash: args.bodyHash, + }); + const sig = await this.crypto.sign(this.signingPrivateKey, message); + return { + 'X-Shade-Sender-Address': this.senderAddress, + 'X-Shade-Signed-At': String(signedAt), + 'X-Shade-Signature': bytesToBase64(sig), + }; + } + + async signControl(args: { + streamId: string; + method: string; + path: string; + }): Promise> { + const signedAt = Date.now(); + const message = canonicalControlBytes({ + address: this.senderAddress, + signedAt, + streamId: args.streamId, + method: args.method, + path: args.path, + }); + const sig = await this.crypto.sign(this.signingPrivateKey, message); + return { + 'X-Shade-Sender-Address': this.senderAddress, + 'X-Shade-Signed-At': String(signedAt), + 'X-Shade-Signature': bytesToBase64(sig), + }; + } +} + +/** Helper: canonical bytes for chunk signature. */ +export function canonicalChunkBytes(args: { + address: string; + signedAt: number; + streamId: string; + laneId: number; + seq: bigint; + bodyHash: Uint8Array; +}): Uint8Array { + const enc = new TextEncoder(); + const fields = [ + `chunk\0`, + `addr=${args.address}\0`, + `at=${args.signedAt}\0`, + `sid=${args.streamId}\0`, + `lane=${args.laneId}\0`, + `seq=${args.seq.toString()}\0`, + `bodyHash=${bytesToHex(args.bodyHash)}\0`, + ]; + return enc.encode(fields.join('')); +} + +/** Helper: canonical bytes for control signature. */ +export function canonicalControlBytes(args: { + address: string; + signedAt: number; + streamId: string; + method: string; + path: string; +}): Uint8Array { + const enc = new TextEncoder(); + const fields = [ + `control\0`, + `addr=${args.address}\0`, + `at=${args.signedAt}\0`, + `sid=${args.streamId}\0`, + `method=${args.method}\0`, + `path=${args.path}\0`, + ]; + return enc.encode(fields.join('')); +} + +function bytesToBase64(b: Uint8Array): string { + let bin = ''; + for (let i = 0; i < b.length; i++) bin += String.fromCharCode(b[i]!); + return btoa(bin); +} + +function bytesToHex(b: Uint8Array): string { + return Array.from(b, (x) => x.toString(16).padStart(2, '0')).join(''); +} diff --git a/packages/shade-sdk/tests/streams-integration.test.ts b/packages/shade-sdk/tests/streams-integration.test.ts new file mode 100644 index 0000000..52269da --- /dev/null +++ b/packages/shade-sdk/tests/streams-integration.test.ts @@ -0,0 +1,159 @@ +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { createShade, type Shade, type TransferHandle, type TransferResult } from '../src/index.js'; +import { + createPrekeyServer, + MemoryPrekeyStore, + PrekeyServerEvents, +} from '@shade/server'; +import { SubtleCryptoProvider } from '@shade/crypto-web'; +import { sha256Once } from '@shade/streams'; + +const crypto = new SubtleCryptoProvider(); + +interface TestRig { + alice: Shade; + bob: Shade; + bobBaseUrl: string; + prekeyStop: () => void; + bobServerStop: () => void; +} + +async function startPrekeyServer(): Promise<{ url: string; stop: () => void }> { + const events = new PrekeyServerEvents(); + const server = createPrekeyServer({ + crypto, + store: new MemoryPrekeyStore(), + disableRateLimit: true, + events, + }); + const port = 21000 + Math.floor(Math.random() * 500); + const handle = Bun.serve({ port, fetch: server.fetch }); + return { url: `http://localhost:${port}`, stop: () => handle.stop() }; +} + +async function setupRig(): Promise { + const prekey = await startPrekeyServer(); + const alice = await createShade({ prekeyServer: prekey.url, address: 'alice' }); + const bob = await createShade({ prekeyServer: prekey.url, address: 'bob' }); + + // Bob's transferRoute lazily creates a TransferEngine, which requires + // configureTransfers first. + bob.configureTransfers({ + resolveBaseUrl: async () => { + throw new Error('bob is receive-only in this test'); + }, + }); + + // Spin up Bob's HTTP transfer endpoint. + const bobApp = await bob.transferRoute(); + const port = 21500 + Math.floor(Math.random() * 500); + const bobServer = Bun.serve({ port, fetch: bobApp.fetch }); + const bobBaseUrl = `http://localhost:${port}`; + + // Wire up Alice's outgoing transfer routing. + alice.configureTransfers({ + resolveBaseUrl: async (addr) => { + if (addr === 'bob') return bobBaseUrl; + throw new Error(`unknown peer ${addr}`); + }, + }); + + return { + alice, + bob, + bobBaseUrl, + prekeyStop: prekey.stop, + bobServerStop: () => bobServer.stop(), + }; +} + +async function teardownRig(rig: TestRig): Promise { + await rig.alice.shutdown(); + await rig.bob.shutdown(); + rig.bobServerStop(); + rig.prekeyStop(); +} + +function hex(b: Uint8Array): string { + return Array.from(b, (x) => x.toString(16).padStart(2, '0')).join(''); +} + +async function uploadAndAwait( + rig: TestRig, + input: Uint8Array, + opts?: { lanes?: number; chunkSize?: number }, +): Promise<{ senderResult: TransferResult; received: Uint8Array }> { + let resolveRecv!: (h: TransferHandle) => void; + const recvHandlePromise = new Promise((r) => { + resolveRecv = r; + }); + const unsubscribe = await rig.bob.onIncomingTransfer(async (incoming) => { + const h = await incoming.accept({ output: { kind: 'buffer' } }); + resolveRecv(h); + }); + + const handle = await rig.alice.upload({ + to: 'bob', + input, + ...(opts?.lanes !== undefined ? { lanes: opts.lanes } : {}), + ...(opts?.chunkSize !== undefined ? { chunkSize: opts.chunkSize } : {}), + metadata: { name: 'integration-test.bin' }, + }); + const recvHandle = await recvHandlePromise; + const [senderResult, recvResult] = await Promise.all([handle.done(), recvHandle.done()]); + unsubscribe(); + const received = + (recvResult as TransferResult & { bytes?: Uint8Array }).bytes ?? new Uint8Array(); + return { senderResult, received }; +} + +describe('Shade SDK end-to-end E2EE transfer', () => { + let rig: TestRig; + beforeAll(async () => { + rig = await setupRig(); + }); + afterAll(async () => { + await teardownRig(rig); + }); + + test('64 KiB payload — 1 lane', async () => { + const input = crypto.randomBytes(64 * 1024); + const { senderResult, received } = await uploadAndAwait(rig, input, { + lanes: 1, + chunkSize: 16 * 1024, + }); + expect(received).toEqual(input); + expect(senderResult.sha256).toBe(hex(sha256Once(input))); + }); + + test('512 KiB payload — 4 lanes range partition', async () => { + const input = crypto.randomBytes(512 * 1024); + const { senderResult, received } = await uploadAndAwait(rig, input, { + lanes: 4, + chunkSize: 32 * 1024, + }); + expect(received).toEqual(input); + expect(senderResult.sha256).toBe(hex(sha256Once(input))); + }); + + test( + '4 MiB payload — 4 lanes, simulates Dispatch upload', + async () => { + const input = crypto.randomBytes(4 * 1024 * 1024); + const { senderResult, received } = await uploadAndAwait(rig, input, { + lanes: 4, + chunkSize: 128 * 1024, + }); + expect(received).toEqual(input); + expect(senderResult.sha256).toBe(hex(sha256Once(input))); + }, + 30_000, + ); + + test('listTransfers returns empty before transfer (memory storage)', async () => { + // MemoryStorage's listActiveStreamStates is implemented but never + // populated by the engine in v0.2.0 (resume persistence is M-Stream-6). + const list = await rig.alice.listTransfers(); + expect(Array.isArray(list)).toBe(true); + }); +}); diff --git a/packages/shade-server/package.json b/packages/shade-server/package.json index e6643f6..3577f13 100644 --- a/packages/shade-server/package.json +++ b/packages/shade-server/package.json @@ -1,6 +1,6 @@ { "name": "@shade/server", - "version": "0.1.0", + "version": "0.3.0", "type": "module", "main": "src/index.ts", "types": "src/index.ts", @@ -9,10 +9,10 @@ "hono": "^4.12.12" }, "optionalDependencies": { + "@shade/crypto-web": "workspace:*", "@shade/observer": "workspace:*", - "@shade/storage-sqlite": "workspace:*", "@shade/storage-postgres": "workspace:*", - "@shade/crypto-web": "workspace:*" + "@shade/storage-sqlite": "workspace:*" }, "devDependencies": { "@shade/crypto-web": "workspace:*" diff --git a/packages/shade-server/src/events.ts b/packages/shade-server/src/events.ts index 5f8cd0b..2b3d756 100644 --- a/packages/shade-server/src/events.ts +++ b/packages/shade-server/src/events.ts @@ -86,7 +86,9 @@ export class PrekeyServerEvents { * Uses SHA-256 via crypto.subtle directly. */ export async function shortHash(key: Uint8Array): Promise { - const buf = await globalThis.crypto.subtle.digest('SHA-256', key); + // Cast through ArrayBuffer (rather than DOM `BufferSource`) so this file + // type-checks without DOM lib in the consumer's tsconfig. + const buf = await globalThis.crypto.subtle.digest('SHA-256', key as unknown as ArrayBuffer); const arr = new Uint8Array(buf).slice(0, 8); return Array.from(arr, (b) => b.toString(16).padStart(2, '0')).join(''); } diff --git a/packages/shade-server/src/index.ts b/packages/shade-server/src/index.ts index 7a74f5e..9df8477 100644 --- a/packages/shade-server/src/index.ts +++ b/packages/shade-server/src/index.ts @@ -31,10 +31,14 @@ export function createPrekeyServer(options: { events?: PrekeyServerEvents; }): Hono { const store = options.store ?? new MemoryPrekeyStore(); - return createPrekeyRoutes(store, options.crypto, { - disableRateLimit: options.disableRateLimit, - events: options.events, - }); + const routesOptions: Parameters[2] = {}; + if (options.disableRateLimit !== undefined) { + routesOptions.disableRateLimit = options.disableRateLimit; + } + if (options.events !== undefined) { + routesOptions.events = options.events; + } + return createPrekeyRoutes(store, options.crypto, routesOptions); } export { RateLimiter, MemoryRateLimitStore } from './rate-limit.js'; diff --git a/packages/shade-server/src/routes.ts b/packages/shade-server/src/routes.ts index fff4c96..9c9f044 100644 --- a/packages/shade-server/src/routes.ts +++ b/packages/shade-server/src/routes.ts @@ -1,6 +1,6 @@ import { Hono } from 'hono'; import type { CryptoProvider } from '@shade/core'; -import { fromBase64, errorToHttpStatus, ShadeError, ValidationError, RateLimitError } from '@shade/core'; +import { errorToHttpStatus, ShadeError, ValidationError, RateLimitError } from '@shade/core'; import type { PrekeyStore } from './store.js'; import { verifyPayload, validateAddress } from './auth.js'; import { RateLimiter, MemoryRateLimitStore, REGISTER_LIMIT, FETCH_LIMIT, REPLENISH_LIMIT, DELETE_LIMIT } from './rate-limit.js'; diff --git a/packages/shade-storage-postgres/package.json b/packages/shade-storage-postgres/package.json index 7cc81e9..a5a1fbe 100644 --- a/packages/shade-storage-postgres/package.json +++ b/packages/shade-storage-postgres/package.json @@ -1,6 +1,6 @@ { "name": "@shade/storage-postgres", - "version": "0.1.0", + "version": "0.3.0", "type": "module", "main": "src/index.ts", "types": "src/index.ts", diff --git a/packages/shade-storage-postgres/src/ensure-tables.ts b/packages/shade-storage-postgres/src/ensure-tables.ts index aeab2d0..605e9aa 100644 --- a/packages/shade-storage-postgres/src/ensure-tables.ts +++ b/packages/shade-storage-postgres/src/ensure-tables.ts @@ -59,6 +59,35 @@ export async function ensureClientTables(sql: Sql): Promise { await sql` CREATE INDEX IF NOT EXISTS shade_retired_at_idx ON shade_retired_identities(retired_at) `; + await sql` + CREATE TABLE IF NOT EXISTS shade_stream_state ( + stream_id TEXT PRIMARY KEY, + direction TEXT NOT NULL CHECK (direction IN ('send','receive')), + peer_address TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('active','paused','finished','aborted')), + metadata_json TEXT NOT NULL, + partition_json TEXT NOT NULL, + lane_state_json TEXT NOT NULL, + io_descriptor_json TEXT NOT NULL, + secret_enc BYTEA NOT NULL, + secret_nonce BYTEA NOT NULL, + overall_hash_state TEXT, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL + ) + `; + await sql` + CREATE INDEX IF NOT EXISTS shade_stream_state_peer_idx + ON shade_stream_state(peer_address) + `; + await sql` + CREATE INDEX IF NOT EXISTS shade_stream_state_updated_idx + ON shade_stream_state(updated_at) + `; + await sql` + CREATE INDEX IF NOT EXISTS shade_stream_state_status_idx + ON shade_stream_state(status, direction) + `; } export async function ensurePrekeyServerTables(sql: Sql): Promise { diff --git a/packages/shade-storage-postgres/src/postgres-storage.ts b/packages/shade-storage-postgres/src/postgres-storage.ts index 9e21f00..2735bef 100644 --- a/packages/shade-storage-postgres/src/postgres-storage.ts +++ b/packages/shade-storage-postgres/src/postgres-storage.ts @@ -1,5 +1,5 @@ import postgres, { type Sql } from 'postgres'; -import type { StorageProvider, IdentityKeyPair, SignedPreKey, OneTimePreKey, SessionState, RetiredIdentity } from '@shade/core'; +import type { StorageProvider, IdentityKeyPair, SignedPreKey, OneTimePreKey, SessionState, RetiredIdentity, PersistedStreamState } from '@shade/core'; import { toBase64, fromBase64, constantTimeEqual, @@ -200,4 +200,89 @@ export class PostgresStorage implements StorageProvider { async pruneRetiredIdentities(olderThan: number): Promise { await this.sql`DELETE FROM shade_retired_identities WHERE retired_at < ${olderThan}`; } + + // ─── Stream-transfer resume state (v0.2.0) ──────────────── + + async saveStreamState(state: PersistedStreamState): Promise { + await this.sql` + INSERT INTO shade_stream_state ( + stream_id, direction, peer_address, status, + metadata_json, partition_json, lane_state_json, io_descriptor_json, + secret_enc, secret_nonce, overall_hash_state, created_at, updated_at + ) VALUES ( + ${state.streamId}, ${state.direction}, ${state.peerAddress}, ${state.status}, + ${state.metadataJson}, ${state.partitionJson}, ${state.laneStateJson}, ${state.ioDescriptorJson}, + ${state.secretEnc}, ${state.secretNonce}, ${state.overallHashState ?? null}, + ${state.createdAt}, ${state.updatedAt} + ) + ON CONFLICT (stream_id) DO UPDATE SET + direction = EXCLUDED.direction, + peer_address = EXCLUDED.peer_address, + status = EXCLUDED.status, + metadata_json = EXCLUDED.metadata_json, + partition_json = EXCLUDED.partition_json, + lane_state_json = EXCLUDED.lane_state_json, + io_descriptor_json = EXCLUDED.io_descriptor_json, + secret_enc = EXCLUDED.secret_enc, + secret_nonce = EXCLUDED.secret_nonce, + overall_hash_state = EXCLUDED.overall_hash_state, + updated_at = EXCLUDED.updated_at + `; + } + + async getStreamState(streamId: string): Promise { + const rows = await this.sql>>` + SELECT * FROM shade_stream_state WHERE stream_id = ${streamId} + `; + if (rows.length === 0) return null; + return rowToStreamState(rows[0]!); + } + + async removeStreamState(streamId: string): Promise { + await this.sql`DELETE FROM shade_stream_state WHERE stream_id = ${streamId}`; + } + + async listActiveStreamStates(direction?: 'send' | 'receive'): Promise { + const rows = + direction === undefined + ? await this.sql>>` + SELECT * FROM shade_stream_state + WHERE status IN ('active','paused') + ORDER BY updated_at DESC + ` + : await this.sql>>` + SELECT * FROM shade_stream_state + WHERE status IN ('active','paused') AND direction = ${direction} + ORDER BY updated_at DESC + `; + return rows.map(rowToStreamState); + } + + async pruneStreamStates(olderThan: number): Promise { + await this.sql` + DELETE FROM shade_stream_state + WHERE status IN ('finished','aborted') AND updated_at < ${olderThan} + `; + } +} + +function rowToStreamState(row: Record): PersistedStreamState { + const out: PersistedStreamState = { + streamId: String(row.stream_id), + direction: row.direction as 'send' | 'receive', + peerAddress: String(row.peer_address), + status: row.status as 'active' | 'paused' | 'finished' | 'aborted', + metadataJson: String(row.metadata_json), + partitionJson: String(row.partition_json), + laneStateJson: String(row.lane_state_json), + ioDescriptorJson: String(row.io_descriptor_json), + secretEnc: row.secret_enc as Uint8Array, + secretNonce: row.secret_nonce as Uint8Array, + createdAt: Number(row.created_at), + updatedAt: Number(row.updated_at), + }; + if (row.overall_hash_state !== null && row.overall_hash_state !== undefined) { + out.overallHashState = String(row.overall_hash_state); + } + return out; } diff --git a/packages/shade-storage-sqlite/package.json b/packages/shade-storage-sqlite/package.json index 8361765..edfd7cf 100644 --- a/packages/shade-storage-sqlite/package.json +++ b/packages/shade-storage-sqlite/package.json @@ -1,6 +1,6 @@ { "name": "@shade/storage-sqlite", - "version": "0.1.0", + "version": "0.3.0", "type": "module", "main": "src/index.ts", "types": "src/index.ts", diff --git a/packages/shade-storage-sqlite/src/sqlite-storage.ts b/packages/shade-storage-sqlite/src/sqlite-storage.ts index e9e7f59..8d83f89 100644 --- a/packages/shade-storage-sqlite/src/sqlite-storage.ts +++ b/packages/shade-storage-sqlite/src/sqlite-storage.ts @@ -1,5 +1,5 @@ import { Database } from 'bun:sqlite'; -import type { StorageProvider, IdentityKeyPair, SignedPreKey, OneTimePreKey, SessionState, RetiredIdentity } from '@shade/core'; +import type { StorageProvider, IdentityKeyPair, SignedPreKey, OneTimePreKey, SessionState, RetiredIdentity, PersistedStreamState } from '@shade/core'; import { toBase64, fromBase64, constantTimeEqual, @@ -42,6 +42,12 @@ export class SQLiteStorage implements StorageProvider { addRetired: ReturnType; listRetired: ReturnType; pruneRetired: ReturnType; + saveStreamState: ReturnType; + getStreamState: ReturnType; + removeStreamState: ReturnType; + listActiveStreamStates: ReturnType; + listActiveStreamStatesByDirection: ReturnType; + pruneStreamStates: ReturnType; }; constructor(dbPath?: string) { @@ -87,6 +93,24 @@ export class SQLiteStorage implements StorageProvider { retired_at INTEGER NOT NULL ); CREATE INDEX IF NOT EXISTS idx_retired_at ON retired_identities(retired_at); + CREATE TABLE IF NOT EXISTS stream_state ( + stream_id TEXT PRIMARY KEY, + direction TEXT NOT NULL, + peer_address TEXT NOT NULL, + status TEXT NOT NULL, + metadata_json TEXT NOT NULL, + partition_json TEXT NOT NULL, + lane_state_json TEXT NOT NULL, + io_descriptor_json TEXT NOT NULL, + secret_enc BLOB NOT NULL, + secret_nonce BLOB NOT NULL, + overall_hash_state TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_stream_state_peer ON stream_state(peer_address); + CREATE INDEX IF NOT EXISTS idx_stream_state_updated ON stream_state(updated_at); + CREATE INDEX IF NOT EXISTS idx_stream_state_status ON stream_state(status, direction); `); } @@ -111,6 +135,24 @@ export class SQLiteStorage implements StorageProvider { addRetired: this.db.prepare('INSERT INTO retired_identities (data_json, retired_at) VALUES (?, ?)'), listRetired: this.db.prepare('SELECT data_json, retired_at FROM retired_identities ORDER BY retired_at DESC'), pruneRetired: this.db.prepare('DELETE FROM retired_identities WHERE retired_at < ?'), + saveStreamState: this.db.prepare( + `INSERT OR REPLACE INTO stream_state ( + stream_id, direction, peer_address, status, + metadata_json, partition_json, lane_state_json, io_descriptor_json, + secret_enc, secret_nonce, overall_hash_state, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ), + getStreamState: this.db.prepare('SELECT * FROM stream_state WHERE stream_id = ?'), + removeStreamState: this.db.prepare('DELETE FROM stream_state WHERE stream_id = ?'), + listActiveStreamStates: this.db.prepare( + "SELECT * FROM stream_state WHERE status IN ('active', 'paused') ORDER BY updated_at DESC", + ), + listActiveStreamStatesByDirection: this.db.prepare( + "SELECT * FROM stream_state WHERE status IN ('active', 'paused') AND direction = ? ORDER BY updated_at DESC", + ), + pruneStreamStates: this.db.prepare( + "DELETE FROM stream_state WHERE status IN ('finished', 'aborted') AND updated_at < ?", + ), }; } @@ -235,4 +277,78 @@ export class SQLiteStorage implements StorageProvider { async pruneRetiredIdentities(olderThan: number): Promise { this.stmts.pruneRetired.run(olderThan); } + + // ─── Stream-transfer resume state (v0.2.0) ──────────────── + + async saveStreamState(state: PersistedStreamState): Promise { + this.stmts.saveStreamState.run( + state.streamId, + state.direction, + state.peerAddress, + state.status, + state.metadataJson, + state.partitionJson, + state.laneStateJson, + state.ioDescriptorJson, + state.secretEnc, + state.secretNonce, + state.overallHashState ?? null, + state.createdAt, + state.updatedAt, + ); + } + + async getStreamState(streamId: string): Promise { + const row = this.stmts.getStreamState.get(streamId) as any; + if (!row) return null; + return rowToStreamState(row); + } + + async removeStreamState(streamId: string): Promise { + this.stmts.removeStreamState.run(streamId); + } + + async listActiveStreamStates(direction?: 'send' | 'receive'): Promise { + const rows = ( + direction === undefined + ? (this.stmts.listActiveStreamStates.all() as any[]) + : (this.stmts.listActiveStreamStatesByDirection.all(direction) as any[]) + ); + return rows.map(rowToStreamState); + } + + async pruneStreamStates(olderThan: number): Promise { + this.stmts.pruneStreamStates.run(olderThan); + } +} + +function rowToStreamState(row: any): PersistedStreamState { + const out: PersistedStreamState = { + streamId: row.stream_id, + direction: row.direction, + peerAddress: row.peer_address, + status: row.status, + metadataJson: row.metadata_json, + partitionJson: row.partition_json, + laneStateJson: row.lane_state_json, + ioDescriptorJson: row.io_descriptor_json, + secretEnc: toBytes(row.secret_enc), + secretNonce: toBytes(row.secret_nonce), + createdAt: Number(row.created_at), + updatedAt: Number(row.updated_at), + }; + if (row.overall_hash_state !== null && row.overall_hash_state !== undefined) { + out.overallHashState = row.overall_hash_state; + } + return out; +} + +function toBytes(value: unknown): Uint8Array { + if (value instanceof Uint8Array) return value; + if (value instanceof ArrayBuffer) return new Uint8Array(value); + if (Array.isArray(value)) return new Uint8Array(value); + if (typeof value === 'string') { + return new Uint8Array(value.split('').map((c) => c.charCodeAt(0))); + } + throw new Error(`Unsupported BLOB representation: ${typeof value}`); } diff --git a/packages/shade-streams/package.json b/packages/shade-streams/package.json new file mode 100644 index 0000000..43c0d34 --- /dev/null +++ b/packages/shade-streams/package.json @@ -0,0 +1,13 @@ +{ + "name": "@shade/streams", + "version": "0.3.0", + "type": "module", + "main": "src/index.ts", + "types": "src/index.ts", + "dependencies": { + "@noble/hashes": "^2.0.1", + "@shade/core": "workspace:*", + "@shade/crypto-web": "workspace:*", + "@shade/proto": "workspace:*" + } +} diff --git a/packages/shade-streams/src/aead.ts b/packages/shade-streams/src/aead.ts new file mode 100644 index 0000000..d8cd031 --- /dev/null +++ b/packages/shade-streams/src/aead.ts @@ -0,0 +1,76 @@ +import { StreamDecryptionError } from './errors.js'; +import { STREAM_NONCE_BYTES } from './nonce.js'; + +/** Authentication tag length for AES-256-GCM (always 16 bytes). */ +export const AEAD_TAG_BYTES = 16; + +/** + * SubtleCrypto-style typed buffer source bridge. The DOM `BufferSource` alias + * isn't available without `lib: ["DOM"]`, but SubtleCrypto runtime accepts + * `ArrayBuffer` or `ArrayBufferView` interchangeably. Cast through `unknown` + * to satisfy TS without dragging in DOM lib (matches the pattern in + * `@shade/crypto-web/src/provider.ts:14`). + */ +function bs(u: Uint8Array): ArrayBuffer { + return u as unknown as ArrayBuffer; +} + +function resolveSubtle(subtle?: SubtleCrypto): SubtleCrypto { + return subtle ?? globalThis.crypto.subtle; +} + +/** + * AES-256-GCM encrypt with a CALLER-SUPPLIED 12-byte nonce. + * + * Unlike `CryptoProvider.aesGcmEncrypt` (which generates a random nonce + * internally), streams require deterministic nonces derived from + * `(laneId, seq)`. Returns the ciphertext concatenated with the 16-byte + * authentication tag (SubtleCrypto's standard layout). + */ +export async function aesGcmEncryptWithNonce( + key: Uint8Array, + nonce: Uint8Array, + plaintext: Uint8Array, + aad: Uint8Array, + subtle?: SubtleCrypto, +): Promise { + if (nonce.length !== STREAM_NONCE_BYTES) { + throw new Error(`AES-GCM nonce must be ${STREAM_NONCE_BYTES} bytes`); + } + const s = resolveSubtle(subtle); + const aesKey = await s.importKey('raw', bs(key), 'AES-GCM', false, ['encrypt']); + const out = await s.encrypt( + { name: 'AES-GCM', iv: bs(nonce), additionalData: bs(aad) }, + aesKey, + bs(plaintext), + ); + return new Uint8Array(out); +} + +/** + * AES-256-GCM decrypt with a CALLER-SUPPLIED nonce. Throws + * `StreamDecryptionError` on authentication failure. + */ +export async function aesGcmDecryptWithNonce( + key: Uint8Array, + nonce: Uint8Array, + ciphertext: Uint8Array, + aad: Uint8Array, + subtle?: SubtleCrypto, +): Promise { + if (nonce.length !== STREAM_NONCE_BYTES) { + throw new Error(`AES-GCM nonce must be ${STREAM_NONCE_BYTES} bytes`); + } + const s = resolveSubtle(subtle); + const aesKey = await s.importKey('raw', bs(key), 'AES-GCM', false, ['decrypt']); + try { + const out = await s.decrypt( + { name: 'AES-GCM', iv: bs(nonce), additionalData: bs(aad) }, + aesKey, + bs(ciphertext), + ); + return new Uint8Array(out); + } catch { + throw new StreamDecryptionError(); + } +} diff --git a/packages/shade-streams/src/coordinator.ts b/packages/shade-streams/src/coordinator.ts new file mode 100644 index 0000000..c916228 --- /dev/null +++ b/packages/shade-streams/src/coordinator.ts @@ -0,0 +1,265 @@ +import type { CryptoProvider } from '@shade/core'; +import { ValidationError } from '@shade/core'; +import { decodeStreamChunk } from '@shade/proto'; +import { StreamFinishedError, StreamProtocolError } from './errors.js'; +import { StreamingSha256 } from './hash.js'; +import { StreamReceiver } from './receiver.js'; +import { StreamSender } from './sender.js'; +import type { DecryptedChunk, EncryptedChunk, LaneInitSpec } from './types.js'; + +export interface MultiLaneSenderInit { + crypto: CryptoProvider; + subtle?: SubtleCrypto; + streamId: Uint8Array; + streamSecret: Uint8Array; + lanes: LaneInitSpec[]; + /** Per-lane resume offsets; defaults to 0 for each lane. */ + startSeqByLane?: Map; +} + +export interface MultiLaneReceiverInit extends MultiLaneSenderInit {} + +/** Per-lane sha256 fingerprint for the stream-finish envelope. */ +export interface LaneFingerprint { + laneId: number; + /** sha256 over plaintext bytes carried by this lane in seq order. */ + sha256: Uint8Array; + chunkCount: number; + byteCount: number; +} + +/** + * Multi-lane stream sender. + * + * Wraps N independent `StreamSender`s sharing the same (streamSecret, streamId) + * and tracks an `overallSha256` over the original plaintext as the consumer + * dispatches bytes to lanes. + * + * The coordinator is partition-AGNOSTIC — the consumer (transfer layer) + * decides which bytes go to which lane. The coordinator only enforces: + * - lane lookup by laneId + * - update of overallSha256 from `appendOverall(...)` (called BEFORE bytes + * are dispatched to lanes, in original byte order) + */ +export class MultiLaneSender { + private readonly senders = new Map(); + private readonly overallHasher = new StreamingSha256(); + private destroyed = false; + + private constructor(senders: StreamSender[]) { + for (const s of senders) { + this.senders.set(s.laneId, s); + } + } + + static async create(opts: MultiLaneSenderInit): Promise { + if (opts.lanes.length === 0) { + throw new ValidationError('at least one lane is required', 'lanes'); + } + const seen = new Set(); + for (const l of opts.lanes) { + if (seen.has(l.laneId)) { + throw new ValidationError(`duplicate laneId ${l.laneId}`, 'lanes'); + } + seen.add(l.laneId); + } + const senders: StreamSender[] = []; + for (const lane of opts.lanes) { + const startSeq = opts.startSeqByLane?.get(lane.laneId); + senders.push( + await StreamSender.create({ + crypto: opts.crypto, + ...(opts.subtle !== undefined ? { subtle: opts.subtle } : {}), + streamId: opts.streamId, + streamSecret: opts.streamSecret, + laneId: lane.laneId, + ...(startSeq !== undefined ? { startSeq } : {}), + }), + ); + } + return new MultiLaneSender(senders); + } + + /** + * Update the overall (cross-lane, original-order) sha256 hasher. + * + * The consumer must call this once with each plaintext slice IN ORIGINAL + * BYTE ORDER (regardless of which lane the slice ends up on). For range + * partitioning this is straightforward (lanes are contiguous); for + * round-robin the consumer hashes each chunk in the order it was read + * from the input. + */ + appendOverall(plaintext: Uint8Array): void { + if (this.destroyed) throw new StreamFinishedError('MultiLaneSender: destroyed'); + this.overallHasher.update(plaintext); + } + + /** Encrypt `plaintext` for the given lane. */ + async encryptForLane( + laneId: number, + plaintext: Uint8Array, + isLast: boolean, + ): Promise { + const lane = this.senders.get(laneId); + if (lane === undefined) { + throw new StreamProtocolError(`Unknown laneId ${laneId}`); + } + return lane.encryptChunk(plaintext, isLast); + } + + /** Resume helper: feed plaintext into a lane's hasher without advancing seq. */ + preHashForLane(laneId: number, plaintext: Uint8Array): void { + const lane = this.senders.get(laneId); + if (lane === undefined) { + throw new StreamProtocolError(`Unknown laneId ${laneId}`); + } + lane.preHash(plaintext); + } + + /** Snapshot per-lane fingerprints (call once when all lanes are finished). */ + getLaneFingerprints(): LaneFingerprint[] { + const out: LaneFingerprint[] = []; + for (const [laneId, sender] of this.senders) { + out.push({ + laneId, + sha256: sender.getLaneSha256Digest(), + chunkCount: Number(sender.nextSequence), + byteCount: Number(sender.bytesSent), + }); + } + out.sort((a, b) => a.laneId - b.laneId); + return out; + } + + /** Snapshot overall sha256 over original-order plaintext. */ + getOverallSha256(): Uint8Array { + return this.overallHasher.digest(); + } + + /** All lanes have emitted their `isLast` chunks. */ + get allLanesFinished(): boolean { + for (const lane of this.senders.values()) { + if (!lane.isFinished) return false; + } + return true; + } + + destroy(): void { + if (this.destroyed) return; + for (const sender of this.senders.values()) sender.destroy(); + this.destroyed = true; + } +} + +/** + * Multi-lane stream receiver. + * + * Routes incoming wire envelopes to per-lane `StreamReceiver`s by laneId, + * and exposes a hook for the consumer to update the overall sha256 in + * original byte order (the consumer's reorder logic feeds plaintext here + * after collecting it from individual lanes). + */ +export class MultiLaneReceiver { + private readonly receivers = new Map(); + private readonly overallHasher = new StreamingSha256(); + private destroyed = false; + + private constructor(receivers: StreamReceiver[]) { + for (const r of receivers) { + this.receivers.set(r.laneId, r); + } + } + + static async create(opts: MultiLaneReceiverInit): Promise { + if (opts.lanes.length === 0) { + throw new ValidationError('at least one lane is required', 'lanes'); + } + const seen = new Set(); + for (const l of opts.lanes) { + if (seen.has(l.laneId)) { + throw new ValidationError(`duplicate laneId ${l.laneId}`, 'lanes'); + } + seen.add(l.laneId); + } + const receivers: StreamReceiver[] = []; + for (const lane of opts.lanes) { + const startSeq = opts.startSeqByLane?.get(lane.laneId); + receivers.push( + await StreamReceiver.create({ + crypto: opts.crypto, + ...(opts.subtle !== undefined ? { subtle: opts.subtle } : {}), + streamId: opts.streamId, + streamSecret: opts.streamSecret, + laneId: lane.laneId, + ...(startSeq !== undefined ? { startSeq } : {}), + }), + ); + } + return new MultiLaneReceiver(receivers); + } + + /** + * Decrypt one wire-level chunk. Routes to the lane indicated by the + * envelope's laneId. Throws if no receiver exists for that laneId. + */ + async decryptChunk(wireBytes: Uint8Array): Promise { + if (this.destroyed) throw new StreamFinishedError('MultiLaneReceiver: destroyed'); + const env = decodeStreamChunk(wireBytes); + const lane = this.receivers.get(env.laneId); + if (lane === undefined) { + throw new StreamProtocolError(`Unknown laneId ${env.laneId}`); + } + const dec = await lane.decryptChunk(wireBytes); + return { ...dec, laneId: env.laneId }; + } + + /** + * Update the overall sha256 hasher with plaintext IN ORIGINAL BYTE ORDER. + * The consumer's reorder logic decides when to call this (as bytes leave + * the reorder buffer in original order, OR for range-partitioning, + * concatenated lane outputs in laneId order). + */ + appendOverall(plaintext: Uint8Array): void { + if (this.destroyed) throw new StreamFinishedError('MultiLaneReceiver: destroyed'); + this.overallHasher.update(plaintext); + } + + /** Snapshot per-lane fingerprints (call once when all lanes are finished). */ + getLaneFingerprints(): LaneFingerprint[] { + const out: LaneFingerprint[] = []; + for (const [laneId, receiver] of this.receivers) { + out.push({ + laneId, + sha256: receiver.getLaneSha256Digest(), + chunkCount: Number(receiver.nextExpectedSequence), + byteCount: Number(receiver.bytesReceived), + }); + } + out.sort((a, b) => a.laneId - b.laneId); + return out; + } + + /** Snapshot overall sha256 over original-order plaintext. */ + getOverallSha256(): Uint8Array { + return this.overallHasher.digest(); + } + + /** All lanes have received their `isLast` chunks. */ + get allLanesFinished(): boolean { + for (const lane of this.receivers.values()) { + if (!lane.isFinished) return false; + } + return true; + } + + /** Get a specific lane receiver (for inspection/resume). */ + getLane(laneId: number): StreamReceiver | undefined { + return this.receivers.get(laneId); + } + + destroy(): void { + if (this.destroyed) return; + for (const receiver of this.receivers.values()) receiver.destroy(); + this.destroyed = true; + } +} diff --git a/packages/shade-streams/src/envelope.ts b/packages/shade-streams/src/envelope.ts new file mode 100644 index 0000000..34df672 --- /dev/null +++ b/packages/shade-streams/src/envelope.ts @@ -0,0 +1,103 @@ +/** + * Plaintext schemas for stream control messages. + * + * Stream-init / -finish / -abort / -resume ride the existing 0x02 ratchet + * envelope — Double Ratchet AEAD already authenticates them and rejects + * replays. Each carries a JSON plaintext with a `kind` discriminator. + * Stream-chunk uses dedicated wire type 0x11 (see `@shade/proto/wire.ts`). + */ +import { ValidationError } from '@shade/core'; +import type { LaneInitSpec, StreamMetadata } from './types.js'; + +export type StreamControlKind = + | 'shade.stream-init/v1' + | 'shade.stream-finish/v1' + | 'shade.stream-abort/v1' + | 'shade.stream-resume-request/v1' + | 'shade.stream-resume-state/v1'; + +export interface StreamInitMessage { + kind: 'shade.stream-init/v1'; + /** base64url-encoded 16-byte streamId. */ + streamId: string; + /** base64url-encoded 32-byte streamSecret. */ + streamSecret: string; + metadata: StreamMetadata; + lanes: LaneInitSpec[]; +} + +export interface StreamFinishMessage { + kind: 'shade.stream-finish/v1'; + streamId: string; + /** Per-lane integrity fingerprints, base64-encoded sha256 of plaintext bytes the lane carried. */ + laneSha256: Array<{ + laneId: number; + sha256: string; + chunkCount: number; + byteCount: number; + }>; + /** base64-encoded sha256 over the original plaintext file in original byte order. */ + overallSha256: string; + finishedAt: number; +} + +export type StreamAbortReason = 'sender-cancel' | 'receiver-cancel' | 'fatal-error'; + +export interface StreamAbortMessage { + kind: 'shade.stream-abort/v1'; + streamId: string; + reason: StreamAbortReason; + message?: string; + abortedAt: number; +} + +export interface StreamResumeRequestMessage { + kind: 'shade.stream-resume-request/v1'; + streamId: string; + requestedAt: number; +} + +export interface StreamResumeStateMessage { + kind: 'shade.stream-resume-state/v1'; + streamId: string; + /** lastSeqAcked = -1 means no chunk for this lane has been received yet. */ + lanes: Array<{ laneId: number; lastSeqAcked: number }>; + resumedAt: number; +} + +export type StreamControlMessage = + | StreamInitMessage + | StreamFinishMessage + | StreamAbortMessage + | StreamResumeRequestMessage + | StreamResumeStateMessage; + +/** Type guard: does the value look like a stream control message? */ +export function isStreamControlMessage(value: unknown): value is StreamControlMessage { + if (typeof value !== 'object' || value === null) return false; + const kind = (value as { kind?: unknown }).kind; + if (typeof kind !== 'string') return false; + return kind.startsWith('shade.stream-'); +} + +/** Parse JSON plaintext; throw ValidationError on shape mismatch. */ +export function parseStreamControl(plaintext: string): StreamControlMessage { + let parsed: unknown; + try { + parsed = JSON.parse(plaintext); + } catch (err) { + throw new ValidationError( + `stream control plaintext is not valid JSON: ${(err as Error).message}`, + 'plaintext', + ); + } + if (!isStreamControlMessage(parsed)) { + throw new ValidationError('plaintext is not a stream control message', 'plaintext'); + } + return parsed; +} + +/** Encode a stream control message as JSON; throws on circular refs. */ +export function encodeStreamControl(msg: StreamControlMessage): string { + return JSON.stringify(msg); +} diff --git a/packages/shade-streams/src/errors.ts b/packages/shade-streams/src/errors.ts new file mode 100644 index 0000000..13e7168 --- /dev/null +++ b/packages/shade-streams/src/errors.ts @@ -0,0 +1,53 @@ +import { ShadeError } from '@shade/core'; + +export class StreamError extends ShadeError { + constructor(code: string, message: string) { + super(code, message); + this.name = 'StreamError'; + } +} + +export class StreamProtocolError extends StreamError { + constructor(message: string) { + super('SHADE_STREAM_PROTOCOL', message); + this.name = 'StreamProtocolError'; + } +} + +export class StreamIntegrityError extends StreamError { + constructor(message: string) { + super('SHADE_STREAM_INTEGRITY', message); + this.name = 'StreamIntegrityError'; + } +} + +export class StreamReplayError extends StreamError { + constructor(message = 'Stream chunk replay detected') { + super('SHADE_STREAM_REPLAY', message); + this.name = 'StreamReplayError'; + } +} + +export class StreamOutOfOrderError extends StreamError { + constructor(expected: number, received: number) { + super( + 'SHADE_STREAM_OUT_OF_ORDER', + `Out-of-order chunk: expected seq=${expected}, got ${received}`, + ); + this.name = 'StreamOutOfOrderError'; + } +} + +export class StreamDecryptionError extends StreamError { + constructor(message = 'Stream chunk authenticated decryption failed') { + super('SHADE_STREAM_DECRYPTION', message); + this.name = 'StreamDecryptionError'; + } +} + +export class StreamFinishedError extends StreamError { + constructor(message = 'Stream is already finished or aborted') { + super('SHADE_STREAM_FINISHED', message); + this.name = 'StreamFinishedError'; + } +} diff --git a/packages/shade-streams/src/hash.ts b/packages/shade-streams/src/hash.ts new file mode 100644 index 0000000..3116b07 --- /dev/null +++ b/packages/shade-streams/src/hash.ts @@ -0,0 +1,41 @@ +import { sha256 } from '@noble/hashes/sha2.js'; + +/** + * Streaming SHA-256 wrapper. + * + * SubtleCrypto exposes only one-shot `digest`. Streams need incremental + * hashing so the receiver doesn't materialize the full plaintext before + * verifying integrity (NF2: O(chunkSize) memory). `@noble/hashes/sha2` + * provides a streaming API that works in Bun, Node, and browsers without + * any native bindings. + */ +export class StreamingSha256 { + private h = sha256.create(); + private finalized = false; + + /** Feed bytes into the hasher. No-op on empty input. */ + update(data: Uint8Array): this { + if (this.finalized) throw new Error('StreamingSha256: already finalized'); + if (data.length > 0) this.h.update(data); + return this; + } + + /** + * Produce the 32-byte digest. After `digest()` the hasher is finalized; + * subsequent `update()` calls will throw. + */ + digest(): Uint8Array { + this.finalized = true; + return this.h.digest(); + } + + /** Whether `digest()` has been called. */ + get isFinalized(): boolean { + return this.finalized; + } +} + +/** Convenience: hash a single buffer in one shot. */ +export function sha256Once(data: Uint8Array): Uint8Array { + return sha256(data); +} diff --git a/packages/shade-streams/src/ids.ts b/packages/shade-streams/src/ids.ts new file mode 100644 index 0000000..8996334 --- /dev/null +++ b/packages/shade-streams/src/ids.ts @@ -0,0 +1,47 @@ +import type { CryptoProvider } from '@shade/core'; +import { ValidationError } from '@shade/core'; + +export const STREAM_ID_BYTES = 16; +export const STREAM_SECRET_BYTES = 32; + +/** Generate a fresh 16-byte random streamId. */ +export function generateStreamId(crypto: CryptoProvider): Uint8Array { + return crypto.randomBytes(STREAM_ID_BYTES); +} + +/** Generate a fresh 32-byte random streamSecret. */ +export function generateStreamSecret(crypto: CryptoProvider): Uint8Array { + return crypto.randomBytes(STREAM_SECRET_BYTES); +} + +/** Encode a streamId as URL-safe base64 (no padding). */ +export function streamIdToString(streamId: Uint8Array): string { + if (streamId.length !== STREAM_ID_BYTES) { + throw new ValidationError(`streamId must be ${STREAM_ID_BYTES} bytes`, 'streamId'); + } + return base64UrlEncode(streamId); +} + +/** Decode a URL-safe base64 streamId back to bytes. */ +export function streamIdFromString(s: string): Uint8Array { + const bytes = base64UrlDecode(s); + if (bytes.length !== STREAM_ID_BYTES) { + throw new ValidationError(`streamId must decode to ${STREAM_ID_BYTES} bytes`, 'streamId'); + } + return bytes; +} + +function base64UrlEncode(bytes: Uint8Array): string { + let bin = ''; + for (let i = 0; i < bytes.length; i++) bin += String.fromCharCode(bytes[i]!); + return btoa(bin).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); +} + +function base64UrlDecode(s: string): Uint8Array { + const padded = s.replace(/-/g, '+').replace(/_/g, '/'); + const pad = padded.length % 4 === 0 ? '' : '='.repeat(4 - (padded.length % 4)); + const bin = atob(padded + pad); + const out = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); + return out; +} diff --git a/packages/shade-streams/src/index.ts b/packages/shade-streams/src/index.ts new file mode 100644 index 0000000..c1d942a --- /dev/null +++ b/packages/shade-streams/src/index.ts @@ -0,0 +1,12 @@ +export * from './errors.js'; +export * from './types.js'; +export * from './ids.js'; +export * from './kdf.js'; +export * from './nonce.js'; +export * from './aead.js'; +export * from './hash.js'; +export * from './envelope.js'; +export * from './sender.js'; +export * from './receiver.js'; +export * from './partition.js'; +export * from './coordinator.js'; diff --git a/packages/shade-streams/src/kdf.ts b/packages/shade-streams/src/kdf.ts new file mode 100644 index 0000000..250700e --- /dev/null +++ b/packages/shade-streams/src/kdf.ts @@ -0,0 +1,66 @@ +import type { CryptoProvider } from '@shade/core'; +import { ValidationError } from '@shade/core'; +import { STREAM_ID_BYTES, STREAM_SECRET_BYTES } from './ids.js'; + +const TEXT = new TextEncoder(); +const STREAM_KEY_INFO = TEXT.encode('shade-stream/v1\0master'); +const LANE_KEY_INFO_PREFIX = TEXT.encode('shade-stream/v1\0lane\0'); + +const STREAM_KEY_BYTES = 32; +export const LANE_KEY_BYTES = 32; + +/** + * Derive the master streamKey from a streamSecret + streamId. + * + * streamKey = HKDF(ikm=streamSecret, salt=streamId, + * info="shade-stream/v1\0master", length=32) + * + * The streamKey is NEVER used to encrypt chunks directly — it is a root for + * per-lane key derivation (see `deriveLaneKey`). + */ +export async function deriveStreamKey( + crypto: CryptoProvider, + streamSecret: Uint8Array, + streamId: Uint8Array, +): Promise { + if (streamSecret.length !== STREAM_SECRET_BYTES) { + throw new ValidationError( + `streamSecret must be ${STREAM_SECRET_BYTES} bytes`, + 'streamSecret', + ); + } + if (streamId.length !== STREAM_ID_BYTES) { + throw new ValidationError(`streamId must be ${STREAM_ID_BYTES} bytes`, 'streamId'); + } + return crypto.hkdf(streamSecret, streamId, STREAM_KEY_INFO, STREAM_KEY_BYTES); +} + +/** + * Derive a lane-specific AEAD key. + * + * laneKey[i] = HKDF(ikm=streamKey, salt=streamId, + * info="shade-stream/v1\0lane\0" || u32_be(laneId), length=32) + * + * Distinct laneIds produce independent keys; receiver derives the same key + * given the same (streamSecret, streamId, laneId). + */ +export async function deriveLaneKey( + crypto: CryptoProvider, + streamKey: Uint8Array, + streamId: Uint8Array, + laneId: number, +): Promise { + if (streamKey.length !== STREAM_KEY_BYTES) { + throw new ValidationError(`streamKey must be ${STREAM_KEY_BYTES} bytes`, 'streamKey'); + } + if (streamId.length !== STREAM_ID_BYTES) { + throw new ValidationError(`streamId must be ${STREAM_ID_BYTES} bytes`, 'streamId'); + } + if (!Number.isInteger(laneId) || laneId < 0 || laneId > 0xffff_ffff) { + throw new ValidationError(`laneId must fit in u32: ${laneId}`, 'laneId'); + } + const info = new Uint8Array(LANE_KEY_INFO_PREFIX.length + 4); + info.set(LANE_KEY_INFO_PREFIX, 0); + new DataView(info.buffer).setUint32(LANE_KEY_INFO_PREFIX.length, laneId, false); + return crypto.hkdf(streamKey, streamId, info, LANE_KEY_BYTES); +} diff --git a/packages/shade-streams/src/nonce.ts b/packages/shade-streams/src/nonce.ts new file mode 100644 index 0000000..8cde282 --- /dev/null +++ b/packages/shade-streams/src/nonce.ts @@ -0,0 +1,62 @@ +import { ValidationError } from '@shade/core'; + +export const STREAM_NONCE_BYTES = 12; + +/** Maximum chunk seq value (u64 max). Hard-spec'd hard limit. */ +export const MAX_SEQ = 0xffff_ffff_ffff_ffffn; + +/** + * Construct the deterministic AES-GCM nonce for a stream chunk. + * + * nonce[0..4] = u32_be(laneId) + * nonce[4..12] = u64_be(seq) + * + * Per (laneId, seq) is unique — combined with the lane-specific key, this + * guarantees AES-GCM nonce-uniqueness even across multiple parallel lanes. + */ +export function buildChunkNonce(laneId: number, seq: number | bigint): Uint8Array { + if (!Number.isInteger(laneId) || laneId < 0 || laneId > 0xffff_ffff) { + throw new ValidationError(`laneId must fit in u32: ${laneId}`, 'laneId'); + } + const seqBig = typeof seq === 'bigint' ? seq : BigInt(seq); + if (seqBig < 0n || seqBig > MAX_SEQ) { + throw new ValidationError(`seq must fit in u64 (>= 0): ${seq}`, 'seq'); + } + const out = new Uint8Array(STREAM_NONCE_BYTES); + const view = new DataView(out.buffer); + view.setUint32(0, laneId, false); + view.setBigUint64(4, seqBig, false); + return out; +} + +/** + * Build the AAD bound to a stream chunk. Computed implicitly on both sides + * from the chunk header (never transmitted as-is). Tampering with any header + * field invalidates the AEAD tag. + * + * aad = streamId(16) || u32_be(laneId) || u64_be(seq) || u8(isLast) + */ +export function buildChunkAad( + streamId: Uint8Array, + laneId: number, + seq: number | bigint, + isLast: boolean, +): Uint8Array { + if (streamId.length !== 16) { + throw new ValidationError('streamId must be 16 bytes', 'streamId'); + } + if (!Number.isInteger(laneId) || laneId < 0 || laneId > 0xffff_ffff) { + throw new ValidationError(`laneId must fit in u32: ${laneId}`, 'laneId'); + } + const seqBig = typeof seq === 'bigint' ? seq : BigInt(seq); + if (seqBig < 0n || seqBig > MAX_SEQ) { + throw new ValidationError(`seq must fit in u64 (>= 0): ${seq}`, 'seq'); + } + const out = new Uint8Array(16 + 4 + 8 + 1); + out.set(streamId, 0); + const view = new DataView(out.buffer); + view.setUint32(16, laneId, false); + view.setBigUint64(20, seqBig, false); + out[28] = isLast ? 0x01 : 0x00; + return out; +} diff --git a/packages/shade-streams/src/partition.ts b/packages/shade-streams/src/partition.ts new file mode 100644 index 0000000..ef2f72a --- /dev/null +++ b/packages/shade-streams/src/partition.ts @@ -0,0 +1,87 @@ +import { ValidationError } from '@shade/core'; +import type { LaneInitSpec, LanePartition } from './types.js'; + +/** Build a range-partition plan: contiguous byte ranges, one per lane. */ +export function planRangePartition( + totalBytes: number, + laneCount: number, +): LaneInitSpec[] { + if (!Number.isInteger(totalBytes) || totalBytes < 0) { + throw new ValidationError(`totalBytes must be a non-negative integer`, 'totalBytes'); + } + if (!Number.isInteger(laneCount) || laneCount < 1) { + throw new ValidationError(`laneCount must be >= 1`, 'laneCount'); + } + const lanes: LaneInitSpec[] = []; + // Use ceil so the first lanes get the extra byte when not evenly divisible. + // Each lane's start = previous end. Last lane's end = totalBytes. + const baseSize = Math.floor(totalBytes / laneCount); + const remainder = totalBytes - baseSize * laneCount; + let cursor = 0; + for (let i = 0; i < laneCount; i++) { + const extra = i < remainder ? 1 : 0; + const size = baseSize + extra; + const startByte = cursor; + const endByte = cursor + size; + lanes.push({ + laneId: i, + partition: { kind: 'range', startByte, endByte, startChunk: 0 }, + }); + cursor = endByte; + } + return lanes; +} + +/** Build a round-robin partition plan: chunk i goes to lane (i mod count). */ +export function planRoundRobinPartition(laneCount: number): LaneInitSpec[] { + if (!Number.isInteger(laneCount) || laneCount < 1) { + throw new ValidationError(`laneCount must be >= 1`, 'laneCount'); + } + const lanes: LaneInitSpec[] = []; + for (let i = 0; i < laneCount; i++) { + lanes.push({ + laneId: i, + partition: { kind: 'round-robin', lane: i, count: laneCount }, + }); + } + return lanes; +} + +/** + * Split an arbitrary byte range into chunkSize-sized slices. + * Returns an array of [startByte, endByte) tuples for each chunk. + */ +export function chunkRange( + startByte: number, + endByte: number, + chunkSize: number, +): Array<{ start: number; end: number }> { + if (chunkSize <= 0) { + throw new ValidationError(`chunkSize must be positive`, 'chunkSize'); + } + const out: Array<{ start: number; end: number }> = []; + if (endByte === startByte) { + // Empty range: emit one empty chunk so isLast can be carried. + out.push({ start: startByte, end: startByte }); + return out; + } + for (let off = startByte; off < endByte; off += chunkSize) { + out.push({ start: off, end: Math.min(off + chunkSize, endByte) }); + } + return out; +} + +/** + * Validate that a `LanePartition` matches the global stream parameters. + * Used by the receiver to detect partition-mismatch on resume. + */ +export function partitionsEqual(a: LanePartition, b: LanePartition): boolean { + if (a.kind !== b.kind) return false; + if (a.kind === 'range' && b.kind === 'range') { + return a.startByte === b.startByte && a.endByte === b.endByte && a.startChunk === b.startChunk; + } + if (a.kind === 'round-robin' && b.kind === 'round-robin') { + return a.lane === b.lane && a.count === b.count; + } + return false; +} diff --git a/packages/shade-streams/src/receiver.ts b/packages/shade-streams/src/receiver.ts new file mode 100644 index 0000000..1777217 --- /dev/null +++ b/packages/shade-streams/src/receiver.ts @@ -0,0 +1,169 @@ +import type { CryptoProvider } from '@shade/core'; +import { ValidationError, constantTimeEqual } from '@shade/core'; +import { decodeStreamChunk } from '@shade/proto'; +import { aesGcmDecryptWithNonce } from './aead.js'; +import { + StreamFinishedError, + StreamOutOfOrderError, + StreamProtocolError, + StreamReplayError, +} from './errors.js'; +import { StreamingSha256 } from './hash.js'; +import { deriveLaneKey, deriveStreamKey } from './kdf.js'; +import { buildChunkAad, buildChunkNonce, MAX_SEQ } from './nonce.js'; +import type { DecryptedChunk } from './types.js'; + +export interface StreamReceiverInit { + crypto: CryptoProvider; + subtle?: SubtleCrypto; + streamId: Uint8Array; + streamSecret: Uint8Array; + laneId: number; + /** First seq this receiver will accept; defaults to 0. Used for resume. */ + startSeq?: number | bigint; +} + +/** + * Per-lane stream receiver state machine. + * + * Verifies AEAD, enforces strict in-order seq (rejects replay + out-of-order), + * and updates a running lane sha256 over the decrypted plaintext. + */ +export class StreamReceiver { + private constructor( + private readonly subtle: SubtleCrypto | undefined, + private readonly crypto: CryptoProvider, + private readonly streamIdBytes: Uint8Array, + public readonly laneId: number, + private laneKey: Uint8Array | null, + private expectedSeq: bigint, + private readonly hasher: StreamingSha256, + private finished: boolean, + private bytesReceivedInternal: bigint, + ) {} + + /** 16-byte streamId this receiver decrypts under. Defensive copy. */ + get streamId(): Uint8Array { + return this.streamIdBytes.slice(); + } + + static async create(opts: StreamReceiverInit): Promise { + const streamKey = await deriveStreamKey(opts.crypto, opts.streamSecret, opts.streamId); + try { + const laneKey = await deriveLaneKey(opts.crypto, streamKey, opts.streamId, opts.laneId); + const startSeq = + opts.startSeq === undefined + ? 0n + : typeof opts.startSeq === 'bigint' + ? opts.startSeq + : BigInt(opts.startSeq); + if (startSeq < 0n || startSeq > MAX_SEQ) { + throw new ValidationError(`startSeq out of range: ${opts.startSeq}`, 'startSeq'); + } + return new StreamReceiver( + opts.subtle, + opts.crypto, + opts.streamId.slice(), + opts.laneId, + laneKey, + startSeq, + new StreamingSha256(), + false, + 0n, + ); + } finally { + opts.crypto.zeroize(streamKey); + } + } + + /** + * Decrypt and authenticate a wire-level chunk envelope. Throws on: + * - mismatched streamId / laneId + * - replayed seq (already accepted) + * - out-of-order seq (gap or backwards) + * - tampered nonce / ciphertext / aad + * - any chunk after `isLast` + */ + async decryptChunk(wireBytes: Uint8Array): Promise { + if (this.finished) { + throw new StreamFinishedError('StreamReceiver: lane already finished'); + } + if (this.laneKey === null) { + throw new StreamFinishedError('StreamReceiver: destroyed'); + } + + const env = decodeStreamChunk(wireBytes); + + if (!constantTimeEqual(env.streamId, this.streamIdBytes)) { + throw new StreamProtocolError('stream-chunk streamId does not match this receiver'); + } + if (env.laneId !== this.laneId) { + throw new StreamProtocolError( + `stream-chunk laneId=${env.laneId} routed to laneId=${this.laneId}`, + ); + } + + const seqBig = typeof env.seq === 'bigint' ? env.seq : BigInt(env.seq); + if (seqBig < this.expectedSeq) { + throw new StreamReplayError(`Replay: seq=${seqBig}, already accepted < ${this.expectedSeq}`); + } + if (seqBig > this.expectedSeq) { + throw new StreamOutOfOrderError(Number(this.expectedSeq), Number(seqBig)); + } + + // Defense-in-depth: the wire nonce MUST equal the deterministically derived one. + const expectedNonce = buildChunkNonce(this.laneId, seqBig); + if (!constantTimeEqual(env.nonce, expectedNonce)) { + throw new StreamProtocolError('stream-chunk nonce mismatch'); + } + if (env.aad.length !== 0) { + throw new StreamProtocolError('stream-chunk aad must be empty in v0.2.0'); + } + + const aad = buildChunkAad(this.streamIdBytes, this.laneId, seqBig, env.isLast); + const plaintext = await aesGcmDecryptWithNonce( + this.laneKey, + env.nonce, + env.ciphertext, + aad, + this.subtle, + ); + + this.hasher.update(plaintext); + this.bytesReceivedInternal += BigInt(plaintext.length); + this.expectedSeq = seqBig + 1n; + + if (env.isLast) this.finished = true; + + return { plaintext, seq: Number(seqBig), isLast: env.isLast }; + } + + /** Snapshot the lane sha256 digest. Hasher is frozen after this call. */ + getLaneSha256Digest(): Uint8Array { + return this.hasher.digest(); + } + + /** Total plaintext bytes accepted so far in this lane. */ + get bytesReceived(): bigint { + return this.bytesReceivedInternal; + } + + /** Next sequence number this receiver expects. */ + get nextExpectedSequence(): bigint { + return this.expectedSeq; + } + + /** Has this lane received its `isLast` chunk? */ + get isFinished(): boolean { + return this.finished; + } + + /** Zero the lane key in memory. After destroy, decrypt calls throw. */ + destroy(): void { + if (this.laneKey !== null) { + this.crypto.zeroize(this.laneKey); + this.laneKey = null; + } + this.finished = true; + } +} diff --git a/packages/shade-streams/src/sender.ts b/packages/shade-streams/src/sender.ts new file mode 100644 index 0000000..05ce844 --- /dev/null +++ b/packages/shade-streams/src/sender.ts @@ -0,0 +1,165 @@ +import type { CryptoProvider } from '@shade/core'; +import { ValidationError } from '@shade/core'; +import { encodeStreamChunk } from '@shade/proto'; +import { aesGcmEncryptWithNonce } from './aead.js'; +import { StreamFinishedError } from './errors.js'; +import { StreamingSha256 } from './hash.js'; +import { deriveLaneKey, deriveStreamKey } from './kdf.js'; +import { buildChunkAad, buildChunkNonce, MAX_SEQ } from './nonce.js'; +import type { EncryptedChunk, StreamChunkEnvelope } from './types.js'; + +export interface StreamSenderInit { + crypto: CryptoProvider; + /** Optional SubtleCrypto for AEAD; defaults to globalThis.crypto.subtle. */ + subtle?: SubtleCrypto; + streamId: Uint8Array; + /** 32-byte streamSecret. The lane key is derived internally and stays in this instance. */ + streamSecret: Uint8Array; + laneId: number; + /** First seq this sender will emit; defaults to 0. Used for resume. */ + startSeq?: number | bigint; +} + +/** + * Per-lane stream sender state machine. + * + * Encapsulates the lane-specific AEAD key + monotonic seq counter + running + * lane-sha256. Multiple senders (one per lane) sharing the same + * `(streamSecret, streamId)` make up a parallel transfer. + */ +export class StreamSender { + private constructor( + private readonly subtle: SubtleCrypto | undefined, + private readonly crypto: CryptoProvider, + private readonly streamIdBytes: Uint8Array, + public readonly laneId: number, + private laneKey: Uint8Array | null, + private nextSeq: bigint, + private readonly hasher: StreamingSha256, + private finished: boolean, + private bytesSentInternal: bigint, + ) {} + + /** 16-byte streamId this sender encrypts under. Defensive copy. */ + get streamId(): Uint8Array { + return this.streamIdBytes.slice(); + } + + static async create(opts: StreamSenderInit): Promise { + const streamKey = await deriveStreamKey(opts.crypto, opts.streamSecret, opts.streamId); + try { + const laneKey = await deriveLaneKey(opts.crypto, streamKey, opts.streamId, opts.laneId); + const startSeq = + opts.startSeq === undefined + ? 0n + : typeof opts.startSeq === 'bigint' + ? opts.startSeq + : BigInt(opts.startSeq); + if (startSeq < 0n || startSeq > MAX_SEQ) { + throw new ValidationError(`startSeq out of range: ${opts.startSeq}`, 'startSeq'); + } + return new StreamSender( + opts.subtle, + opts.crypto, + opts.streamId.slice(), + opts.laneId, + laneKey, + startSeq, + new StreamingSha256(), + false, + 0n, + ); + } finally { + opts.crypto.zeroize(streamKey); + } + } + + /** + * Encrypt one plaintext chunk. Updates lane sha256 with the plaintext bytes + * BEFORE encryption (so receiver can independently verify by hashing + * decrypted plaintext in the same order). + */ + async encryptChunk(plaintext: Uint8Array, isLast: boolean): Promise { + if (this.finished) { + throw new StreamFinishedError('StreamSender: lane already finished'); + } + if (this.laneKey === null) { + throw new StreamFinishedError('StreamSender: destroyed'); + } + if (this.nextSeq > MAX_SEQ) { + throw new ValidationError('seq overflow', 'seq'); + } + + const seq = this.nextSeq; + const nonce = buildChunkNonce(this.laneId, seq); + const aad = buildChunkAad(this.streamIdBytes, this.laneId, seq, isLast); + const ciphertext = await aesGcmEncryptWithNonce( + this.laneKey, + nonce, + plaintext, + aad, + this.subtle, + ); + + this.hasher.update(plaintext); + this.bytesSentInternal += BigInt(plaintext.length); + this.nextSeq = seq + 1n; + + const envelope: StreamChunkEnvelope = { + streamId: this.streamIdBytes, + laneId: this.laneId, + seq, + isLast, + nonce, + aad: new Uint8Array(0), + ciphertext, + }; + const bytes = encodeStreamChunk(envelope); + + if (isLast) this.finished = true; + + return { envelope, bytes, seq: Number(seq) }; + } + + /** Snapshot the lane sha256 digest. Hasher is frozen after this call. */ + getLaneSha256Digest(): Uint8Array { + return this.hasher.digest(); + } + + /** + * Feed plaintext into the lane sha256 WITHOUT advancing seq or + * producing wire bytes. Used by resume flows to re-build the running + * lane hash from already-shipped bytes (since `@noble/hashes` v2 doesn't + * expose serializable state in v0.2.0). + */ + preHash(plaintext: Uint8Array): void { + if (this.finished) { + throw new StreamFinishedError('StreamSender: lane already finished'); + } + this.hasher.update(plaintext); + } + + /** Number of plaintext bytes encrypted so far in this lane. */ + get bytesSent(): bigint { + return this.bytesSentInternal; + } + + /** Next sequence number this sender will emit. */ + get nextSequence(): bigint { + return this.nextSeq; + } + + /** Has this lane emitted its `isLast` chunk? */ + get isFinished(): boolean { + return this.finished; + } + + /** Zero the lane key in memory. After destroy, encrypt calls throw. */ + destroy(): void { + if (this.laneKey !== null) { + this.crypto.zeroize(this.laneKey); + this.laneKey = null; + } + this.finished = true; + } +} diff --git a/packages/shade-streams/src/types.ts b/packages/shade-streams/src/types.ts new file mode 100644 index 0000000..0c9c091 --- /dev/null +++ b/packages/shade-streams/src/types.ts @@ -0,0 +1,72 @@ +/** + * Public types for @shade/streams. + * + * Higher-level wrappers for transfer-orchestration (parallel lanes, resume, + * progress, etc.) live in @shade/transfer. + */ + +/** Plaintext metadata sent in a stream-init control envelope. */ +export interface StreamMetadata { + name?: string; + /** Total plaintext size in bytes. Omit for streams of unknown length. */ + sizeBytes?: number; + contentType?: string; + /** Plaintext bytes per chunk for this stream. */ + chunkSize: number; + /** Total chunk count across all lanes. Omit for unknown-length streams. */ + totalChunks?: number; + /** Sender's local clock at init time (advisory; never used for security decisions). */ + sentAt: number; + /** + * Optional application-level metadata, JSON-stringified-safe key/value + * pairs. Round-tripped verbatim through stream-init plaintext. Used by + * higher layers (e.g. `@shade/files` writes a `shadeFilesWriteId` here so + * a server-side bridge can correlate an inbound transfer with a pending + * RPC). The transport itself does not interpret these values. + */ + userMetadata?: Record; +} + +/** Per-lane partition assignment carried in stream-init. */ +export type LanePartition = + | { + kind: 'range'; + /** Inclusive start byte of this lane's slice. */ + startByte: number; + /** Exclusive end byte of this lane's slice. */ + endByte: number; + /** First chunk seq this lane's region begins at (always 0 for per-lane numbering). */ + startChunk: number; + } + | { + kind: 'round-robin'; + /** Chunk i goes to lane (i mod count). */ + lane: number; + count: number; + }; + +/** Per-lane state included in stream-init plaintext. laneKey is NOT shipped — it is derived. */ +export interface LaneInitSpec { + laneId: number; + partition: LanePartition; +} + +import type { StreamChunkWire } from '@shade/proto'; + +/** Wire-decoded stream-chunk envelope (alias for @shade/proto's `StreamChunkWire`). */ +export type StreamChunkEnvelope = StreamChunkWire; + +/** Result returned from `StreamSender.encryptChunk`. */ +export interface EncryptedChunk { + envelope: StreamChunkEnvelope; + /** Raw bytes ready to ship — output of `encodeStreamChunk`. */ + bytes: Uint8Array; + seq: number; +} + +/** Result returned from `StreamReceiver.decryptChunk`. */ +export interface DecryptedChunk { + plaintext: Uint8Array; + seq: number; + isLast: boolean; +} diff --git a/packages/shade-streams/tests/aead.test.ts b/packages/shade-streams/tests/aead.test.ts new file mode 100644 index 0000000..e69ff79 --- /dev/null +++ b/packages/shade-streams/tests/aead.test.ts @@ -0,0 +1,145 @@ +import { describe, test, expect } from 'bun:test'; +import { SubtleCryptoProvider } from '@shade/crypto-web'; +import { + aesGcmEncryptWithNonce, + aesGcmDecryptWithNonce, + buildChunkNonce, + buildChunkAad, + deriveStreamKey, + deriveLaneKey, + StreamDecryptionError, +} from '../src/index.js'; + +const crypto = new SubtleCryptoProvider(); + +async function laneKey(): Promise<{ key: Uint8Array; streamId: Uint8Array }> { + const secret = new Uint8Array(32).fill(0x42); + const streamId = new Uint8Array(16).fill(0x99); + const sk = await deriveStreamKey(crypto, secret, streamId); + const lk = await deriveLaneKey(crypto, sk, streamId, 0); + return { key: lk, streamId }; +} + +describe('aesGcmEncryptWithNonce / aesGcmDecryptWithNonce', () => { + test('encrypt → decrypt roundtrip', async () => { + const { key, streamId } = await laneKey(); + const nonce = buildChunkNonce(0, 0); + const aad = buildChunkAad(streamId, 0, 0, false); + const plaintext = new TextEncoder().encode('hello shade streams'); + + const ct = await aesGcmEncryptWithNonce(key, nonce, plaintext, aad); + const pt = await aesGcmDecryptWithNonce(key, nonce, ct, aad); + expect(new TextDecoder().decode(pt)).toBe('hello shade streams'); + }); + + test('produces ciphertext length = plaintext + 16-byte tag', async () => { + const { key, streamId } = await laneKey(); + const plaintext = new Uint8Array(1024); + const ct = await aesGcmEncryptWithNonce( + key, + buildChunkNonce(0, 0), + plaintext, + buildChunkAad(streamId, 0, 0, false), + ); + expect(ct.length).toBe(1024 + 16); + }); + + test('handles empty plaintext', async () => { + const { key, streamId } = await laneKey(); + const nonce = buildChunkNonce(0, 0); + const aad = buildChunkAad(streamId, 0, 0, true); + const ct = await aesGcmEncryptWithNonce(key, nonce, new Uint8Array(0), aad); + expect(ct.length).toBe(16); // tag only + const pt = await aesGcmDecryptWithNonce(key, nonce, ct, aad); + expect(pt.length).toBe(0); + }); + + test('handles 1 MiB plaintext (default chunk size)', async () => { + const { key, streamId } = await laneKey(); + const nonce = buildChunkNonce(0, 0); + const aad = buildChunkAad(streamId, 0, 0, false); + const plaintext = crypto.randomBytes(1024 * 1024); + const ct = await aesGcmEncryptWithNonce(key, nonce, plaintext, aad); + const pt = await aesGcmDecryptWithNonce(key, nonce, ct, aad); + expect(pt).toEqual(plaintext); + }); + + test('different nonces with same key produce different ciphertexts', async () => { + const { key, streamId } = await laneKey(); + const aad = buildChunkAad(streamId, 0, 0, false); + const plaintext = new TextEncoder().encode('same plaintext'); + const ct1 = await aesGcmEncryptWithNonce(key, buildChunkNonce(0, 0), plaintext, aad); + const ct2 = await aesGcmEncryptWithNonce(key, buildChunkNonce(0, 1), plaintext, aad); + expect(ct1).not.toEqual(ct2); + }); + + test('tampered ciphertext byte → StreamDecryptionError', async () => { + const { key, streamId } = await laneKey(); + const nonce = buildChunkNonce(0, 0); + const aad = buildChunkAad(streamId, 0, 0, false); + const ct = await aesGcmEncryptWithNonce(key, nonce, new TextEncoder().encode('hi'), aad); + ct[0] ^= 0x01; + await expect(aesGcmDecryptWithNonce(key, nonce, ct, aad)).rejects.toThrow( + StreamDecryptionError, + ); + }); + + test('tampered tag byte → StreamDecryptionError', async () => { + const { key, streamId } = await laneKey(); + const nonce = buildChunkNonce(0, 0); + const aad = buildChunkAad(streamId, 0, 0, false); + const ct = await aesGcmEncryptWithNonce(key, nonce, new TextEncoder().encode('hi'), aad); + ct[ct.length - 1] ^= 0x80; + await expect(aesGcmDecryptWithNonce(key, nonce, ct, aad)).rejects.toThrow( + StreamDecryptionError, + ); + }); + + test('wrong AAD → StreamDecryptionError', async () => { + const { key, streamId } = await laneKey(); + const nonce = buildChunkNonce(0, 0); + const aadEnc = buildChunkAad(streamId, 0, 0, false); + const aadDec = buildChunkAad(streamId, 0, 0, true); // isLast flipped + const ct = await aesGcmEncryptWithNonce(key, nonce, new TextEncoder().encode('hi'), aadEnc); + await expect(aesGcmDecryptWithNonce(key, nonce, ct, aadDec)).rejects.toThrow( + StreamDecryptionError, + ); + }); + + test('wrong nonce → StreamDecryptionError', async () => { + const { key, streamId } = await laneKey(); + const aad = buildChunkAad(streamId, 0, 0, false); + const ct = await aesGcmEncryptWithNonce( + key, + buildChunkNonce(0, 0), + new TextEncoder().encode('hi'), + aad, + ); + await expect( + aesGcmDecryptWithNonce(key, buildChunkNonce(0, 1), ct, aad), + ).rejects.toThrow(StreamDecryptionError); + }); + + test('wrong key → StreamDecryptionError', async () => { + const { streamId } = await laneKey(); + const nonce = buildChunkNonce(0, 0); + const aad = buildChunkAad(streamId, 0, 0, false); + const k1 = new Uint8Array(32).fill(1); + const k2 = new Uint8Array(32).fill(2); + const ct = await aesGcmEncryptWithNonce(k1, nonce, new TextEncoder().encode('hi'), aad); + await expect(aesGcmDecryptWithNonce(k2, nonce, ct, aad)).rejects.toThrow( + StreamDecryptionError, + ); + }); + + test('rejects non-12-byte nonce', async () => { + const { key, streamId } = await laneKey(); + const aad = buildChunkAad(streamId, 0, 0, false); + await expect( + aesGcmEncryptWithNonce(key, new Uint8Array(11), new Uint8Array(0), aad), + ).rejects.toThrow(); + await expect( + aesGcmDecryptWithNonce(key, new Uint8Array(13), new Uint8Array(16), aad), + ).rejects.toThrow(); + }); +}); diff --git a/packages/shade-streams/tests/coordinator.test.ts b/packages/shade-streams/tests/coordinator.test.ts new file mode 100644 index 0000000..cc79b0b --- /dev/null +++ b/packages/shade-streams/tests/coordinator.test.ts @@ -0,0 +1,281 @@ +import { describe, test, expect } from 'bun:test'; +import { SubtleCryptoProvider } from '@shade/crypto-web'; +import { ValidationError } from '@shade/core'; +import { + MultiLaneSender, + MultiLaneReceiver, + StreamProtocolError, + generateStreamId, + generateStreamSecret, + planRangePartition, + planRoundRobinPartition, + chunkRange, + sha256Once, +} from '../src/index.js'; + +const crypto = new SubtleCryptoProvider(); + +function hex(b: Uint8Array): string { + return Array.from(b, (x) => x.toString(16).padStart(2, '0')).join(''); +} + +/** + * Roundtrip a fixed input through `laneCount` lanes using range partitioning. + * Returns the per-side overall sha256 + the reconstructed plaintext. + */ +async function roundtripRange(input: Uint8Array, laneCount: number, chunkSize: number) { + const streamId = generateStreamId(crypto); + const streamSecret = generateStreamSecret(crypto); + const lanes = planRangePartition(input.length, laneCount); + + const sender = await MultiLaneSender.create({ crypto, streamId, streamSecret, lanes }); + const receiver = await MultiLaneReceiver.create({ crypto, streamId, streamSecret, lanes }); + + // Append the entire input to the sender's overall hasher in original order + // (range mode: lane i's slice is contiguous in original order). + sender.appendOverall(input); + + // Encrypt all chunks for all lanes (interleaved as a real consumer would). + const wireChunks: Array<{ laneId: number; bytes: Uint8Array }> = []; + for (const lane of lanes) { + if (lane.partition.kind !== 'range') throw new Error('expected range'); + const slices = chunkRange(lane.partition.startByte, lane.partition.endByte, chunkSize); + for (let i = 0; i < slices.length; i++) { + const s = slices[i]!; + const isLast = i === slices.length - 1; + const plaintext = input.subarray(s.start, s.end); + const { bytes } = await sender.encryptForLane(lane.laneId, plaintext, isLast); + wireChunks.push({ laneId: lane.laneId, bytes }); + } + } + + // Receiver decrypts. Range mode: gather lane outputs in laneId order. + const laneBuffers = new Map(); + for (const { bytes } of wireChunks) { + const dec = await receiver.decryptChunk(bytes); + if (!laneBuffers.has(dec.laneId)) laneBuffers.set(dec.laneId, []); + laneBuffers.get(dec.laneId)!.push(dec.plaintext); + } + // Concatenate lane outputs in laneId order to rebuild original byte order. + const reconstructed: Uint8Array[] = []; + for (let i = 0; i < laneCount; i++) { + for (const piece of laneBuffers.get(i) ?? []) reconstructed.push(piece); + } + // Feed receiver's overall hasher in original byte order. + for (const piece of reconstructed) receiver.appendOverall(piece); + + return { + sender, + receiver, + senderOverall: sender.getOverallSha256(), + receiverOverall: receiver.getOverallSha256(), + reconstructed: concat(reconstructed), + }; +} + +/** Roundtrip via round-robin partitioning. Chunk i goes to lane (i mod L). */ +async function roundtripRoundRobin( + input: Uint8Array, + laneCount: number, + chunkSize: number, +) { + const streamId = generateStreamId(crypto); + const streamSecret = generateStreamSecret(crypto); + const lanes = planRoundRobinPartition(laneCount); + + const sender = await MultiLaneSender.create({ crypto, streamId, streamSecret, lanes }); + const receiver = await MultiLaneReceiver.create({ crypto, streamId, streamSecret, lanes }); + + // Append in original order. + sender.appendOverall(input); + + // Slice into chunks; round-robin assignment. + const slices = chunkRange(0, input.length, chunkSize); + // Determine `isLast` for each lane (last chunk this lane sees). + const lastChunkByLane = new Map(); + for (let i = 0; i < slices.length; i++) { + lastChunkByLane.set(i % laneCount, i); + } + const wireChunks: Array<{ chunkIndex: number; bytes: Uint8Array }> = []; + for (let i = 0; i < slices.length; i++) { + const s = slices[i]!; + const laneId = i % laneCount; + const isLast = lastChunkByLane.get(laneId) === i; + const plaintext = input.subarray(s.start, s.end); + const { bytes } = await sender.encryptForLane(laneId, plaintext, isLast); + wireChunks.push({ chunkIndex: i, bytes }); + } + + // Receiver: collect chunks; reorder by chunkIndex (the original-order index). + const decoded = new Map(); + for (const { chunkIndex, bytes } of wireChunks) { + const dec = await receiver.decryptChunk(bytes); + decoded.set(chunkIndex, dec.plaintext); + } + const reconstructed: Uint8Array[] = []; + for (let i = 0; i < slices.length; i++) { + reconstructed.push(decoded.get(i)!); + } + for (const piece of reconstructed) receiver.appendOverall(piece); + + return { + sender, + receiver, + senderOverall: sender.getOverallSha256(), + receiverOverall: receiver.getOverallSha256(), + reconstructed: concat(reconstructed), + }; +} + +function concat(parts: Uint8Array[]): Uint8Array { + const total = parts.reduce((s, p) => s + p.length, 0); + const out = new Uint8Array(total); + let off = 0; + for (const p of parts) { + out.set(p, off); + off += p.length; + } + return out; +} + +describe('MultiLaneSender / MultiLaneReceiver — basic shape', () => { + test('rejects empty lanes array', async () => { + const streamId = generateStreamId(crypto); + const streamSecret = generateStreamSecret(crypto); + await expect( + MultiLaneSender.create({ crypto, streamId, streamSecret, lanes: [] }), + ).rejects.toThrow(ValidationError); + }); + + test('rejects duplicate laneIds', async () => { + const streamId = generateStreamId(crypto); + const streamSecret = generateStreamSecret(crypto); + await expect( + MultiLaneSender.create({ + crypto, + streamId, + streamSecret, + lanes: [ + { laneId: 0, partition: { kind: 'round-robin', lane: 0, count: 2 } }, + { laneId: 0, partition: { kind: 'round-robin', lane: 1, count: 2 } }, + ], + }), + ).rejects.toThrow(ValidationError); + }); + + test('encryptForLane on unknown laneId throws StreamProtocolError', async () => { + const sender = await MultiLaneSender.create({ + crypto, + streamId: generateStreamId(crypto), + streamSecret: generateStreamSecret(crypto), + lanes: planRoundRobinPartition(2), + }); + await expect(sender.encryptForLane(99, new Uint8Array(0), false)).rejects.toThrow( + StreamProtocolError, + ); + }); +}); + +describe('Range-partition roundtrip', () => { + test('1 KB / 4 lanes / 256 B chunk', async () => { + const input = crypto.randomBytes(1024); + const r = await roundtripRange(input, 4, 256); + expect(r.reconstructed).toEqual(input); + expect(hex(r.senderOverall)).toBe(hex(r.receiverOverall)); + expect(hex(r.senderOverall)).toBe(hex(sha256Once(input))); + }); + + test('exactly chunkSize-aligned input', async () => { + const input = crypto.randomBytes(8 * 256); + const r = await roundtripRange(input, 4, 256); + expect(r.reconstructed).toEqual(input); + expect(hex(r.senderOverall)).toBe(hex(r.receiverOverall)); + }); + + test('input smaller than chunkSize × laneCount', async () => { + const input = crypto.randomBytes(50); + const r = await roundtripRange(input, 4, 64); + expect(r.reconstructed).toEqual(input); + expect(hex(r.senderOverall)).toBe(hex(r.receiverOverall)); + }); +}); + +describe('Round-robin partition roundtrip', () => { + test('4 lanes, 1 KB / 128 B chunks', async () => { + const input = crypto.randomBytes(1024); + const r = await roundtripRoundRobin(input, 4, 128); + expect(r.reconstructed).toEqual(input); + expect(hex(r.senderOverall)).toBe(hex(r.receiverOverall)); + }); +}); + +describe('Lane-parity ship-gate (1 / 4 / 16 lanes → same overallSha256)', () => { + const sizes = [ + { label: '1 KiB', bytes: 1024 }, + { label: '256 KiB', bytes: 256 * 1024 }, + { label: '2 MiB', bytes: 2 * 1024 * 1024 }, + ]; + + for (const { label, bytes } of sizes) { + test(`${label} input — same sha256 across {1, 4, 16} lanes (range)`, async () => { + const input = crypto.randomBytes(bytes); + const expected = hex(sha256Once(input)); + for (const laneCount of [1, 4, 16]) { + const r = await roundtripRange(input, laneCount, 64 * 1024); + expect(r.reconstructed).toEqual(input); + expect(hex(r.senderOverall)).toBe(expected); + expect(hex(r.receiverOverall)).toBe(expected); + } + }); + } + + test('1 MiB input — same sha256 across {1, 4, 16} lanes (round-robin)', async () => { + const input = crypto.randomBytes(1024 * 1024); + const expected = hex(sha256Once(input)); + for (const laneCount of [1, 4, 16]) { + const r = await roundtripRoundRobin(input, laneCount, 32 * 1024); + expect(r.reconstructed).toEqual(input); + expect(hex(r.senderOverall)).toBe(expected); + expect(hex(r.receiverOverall)).toBe(expected); + } + }); + + test('range and round-robin produce the same overall sha256 for the same input', async () => { + const input = crypto.randomBytes(128 * 1024); + const a = await roundtripRange(input, 4, 16 * 1024); + const b = await roundtripRoundRobin(input, 4, 16 * 1024); + expect(hex(a.senderOverall)).toBe(hex(b.senderOverall)); + expect(hex(a.receiverOverall)).toBe(hex(b.receiverOverall)); + }); +}); + +describe('Per-lane fingerprints', () => { + test('match between sender and receiver after roundtrip', async () => { + const input = crypto.randomBytes(64 * 1024); + const r = await roundtripRange(input, 4, 8 * 1024); + const senderFps = r.sender.getLaneFingerprints(); + const receiverFps = r.receiver.getLaneFingerprints(); + expect(senderFps.length).toBe(4); + for (let i = 0; i < 4; i++) { + expect(hex(senderFps[i]!.sha256)).toBe(hex(receiverFps[i]!.sha256)); + expect(senderFps[i]!.byteCount).toBe(receiverFps[i]!.byteCount); + expect(senderFps[i]!.chunkCount).toBe(receiverFps[i]!.chunkCount); + } + }); + + test('byteCount across all lanes equals total input', async () => { + const input = crypto.randomBytes(99 * 1024); // intentionally non-divisible + const r = await roundtripRange(input, 4, 8 * 1024); + const total = r.sender + .getLaneFingerprints() + .reduce((s, l) => s + l.byteCount, 0); + expect(total).toBe(input.length); + }); + + test('allLanesFinished reflects per-lane completion', async () => { + const input = crypto.randomBytes(1024); + const r = await roundtripRange(input, 2, 256); + expect(r.sender.allLanesFinished).toBe(true); + expect(r.receiver.allLanesFinished).toBe(true); + }); +}); diff --git a/packages/shade-streams/tests/envelope.test.ts b/packages/shade-streams/tests/envelope.test.ts new file mode 100644 index 0000000..4301028 --- /dev/null +++ b/packages/shade-streams/tests/envelope.test.ts @@ -0,0 +1,92 @@ +import { describe, test, expect } from 'bun:test'; +import { ValidationError } from '@shade/core'; +import { + encodeStreamControl, + parseStreamControl, + isStreamControlMessage, +} from '../src/index.js'; +import type { + StreamInitMessage, + StreamFinishMessage, + StreamAbortMessage, +} from '../src/index.js'; + +describe('control envelope encode/parse roundtrip', () => { + test('stream-init', () => { + const msg: StreamInitMessage = { + kind: 'shade.stream-init/v1', + streamId: 'AAAAAAAAAAAAAAAAAAAAAA', + streamSecret: 'BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBA', + metadata: { + name: 'world.zip', + sizeBytes: 1024, + contentType: 'application/zip', + chunkSize: 256, + totalChunks: 4, + sentAt: 1730000000000, + }, + lanes: [ + { + laneId: 0, + partition: { kind: 'range', startByte: 0, endByte: 512, startChunk: 0 }, + }, + { + laneId: 1, + partition: { kind: 'range', startByte: 512, endByte: 1024, startChunk: 0 }, + }, + ], + }; + const json = encodeStreamControl(msg); + expect(parseStreamControl(json)).toEqual(msg); + }); + + test('stream-finish', () => { + const msg: StreamFinishMessage = { + kind: 'shade.stream-finish/v1', + streamId: 'AAAAAAAAAAAAAAAAAAAAAA', + laneSha256: [{ laneId: 0, sha256: 'abcd', chunkCount: 1, byteCount: 256 }], + overallSha256: 'efgh', + finishedAt: 1730000001000, + }; + expect(parseStreamControl(encodeStreamControl(msg))).toEqual(msg); + }); + + test('stream-abort', () => { + const msg: StreamAbortMessage = { + kind: 'shade.stream-abort/v1', + streamId: 'AAAAAAAAAAAAAAAAAAAAAA', + reason: 'sender-cancel', + message: 'user clicked cancel', + abortedAt: 1730000002000, + }; + expect(parseStreamControl(encodeStreamControl(msg))).toEqual(msg); + }); + + test('rejects malformed JSON', () => { + expect(() => parseStreamControl('not-json')).toThrow(ValidationError); + }); + + test('rejects messages without a kind field', () => { + expect(() => parseStreamControl(JSON.stringify({ foo: 'bar' }))).toThrow(ValidationError); + }); + + test('rejects messages whose kind does not start with shade.stream-', () => { + expect(() => parseStreamControl(JSON.stringify({ kind: 'other.kind' }))).toThrow( + ValidationError, + ); + }); +}); + +describe('isStreamControlMessage', () => { + test('returns true for valid shapes', () => { + expect(isStreamControlMessage({ kind: 'shade.stream-init/v1' })).toBe(true); + expect(isStreamControlMessage({ kind: 'shade.stream-finish/v1' })).toBe(true); + }); + + test('returns false for non-objects and missing kind', () => { + expect(isStreamControlMessage(null)).toBe(false); + expect(isStreamControlMessage(42)).toBe(false); + expect(isStreamControlMessage({})).toBe(false); + expect(isStreamControlMessage({ kind: 'unrelated' })).toBe(false); + }); +}); diff --git a/packages/shade-streams/tests/hash.test.ts b/packages/shade-streams/tests/hash.test.ts new file mode 100644 index 0000000..e9f5ebf --- /dev/null +++ b/packages/shade-streams/tests/hash.test.ts @@ -0,0 +1,72 @@ +import { describe, test, expect } from 'bun:test'; +import { StreamingSha256, sha256Once } from '../src/index.js'; + +function hex(bytes: Uint8Array): string { + return Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join(''); +} + +async function subtleHashHex(data: Uint8Array): Promise { + const buf = await globalThis.crypto.subtle.digest('SHA-256', data as unknown as ArrayBuffer); + return hex(new Uint8Array(buf)); +} + +describe('StreamingSha256', () => { + test('digest of empty input matches the well-known SHA-256 zero hash', () => { + const h = new StreamingSha256().digest(); + expect(hex(h)).toBe('e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'); + }); + + test('matches one-shot sha256 over the same bytes', () => { + const data = new TextEncoder().encode('the quick brown fox'); + const streaming = new StreamingSha256().update(data).digest(); + expect(hex(streaming)).toBe(hex(sha256Once(data))); + }); + + test('matches SubtleCrypto digest over the same bytes', async () => { + const data = new TextEncoder().encode('cross-impl parity check'); + const streaming = new StreamingSha256().update(data).digest(); + expect(hex(streaming)).toBe(await subtleHashHex(data)); + }); + + test('chunked updates produce identical digest to a single update', () => { + const buf = new Uint8Array(4096); + for (let i = 0; i < buf.length; i++) buf[i] = i & 0xff; + const a = new StreamingSha256().update(buf).digest(); + const b = new StreamingSha256(); + for (let off = 0; off < buf.length; off += 137) { + b.update(buf.slice(off, Math.min(off + 137, buf.length))); + } + expect(hex(a)).toBe(hex(b.digest())); + }); + + test('handles multi-megabyte inputs (memory-bounded streaming)', () => { + const chunk = new Uint8Array(1024 * 1024); + for (let i = 0; i < chunk.length; i++) chunk[i] = (i * 31) & 0xff; + const h = new StreamingSha256(); + for (let i = 0; i < 4; i++) h.update(chunk); + const digest = h.digest(); + expect(digest.length).toBe(32); + }); + + test('throws on update after digest()', () => { + const h = new StreamingSha256(); + h.digest(); + expect(() => h.update(new Uint8Array([1]))).toThrow(); + }); + + test('isFinalized reflects digest()', () => { + const h = new StreamingSha256(); + expect(h.isFinalized).toBe(false); + h.digest(); + expect(h.isFinalized).toBe(true); + }); + + test('skips no-op empty updates', () => { + const h = new StreamingSha256(); + h.update(new Uint8Array(0)); + h.update(new Uint8Array(0)); + expect(hex(h.digest())).toBe( + 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', + ); + }); +}); diff --git a/packages/shade-streams/tests/ids.test.ts b/packages/shade-streams/tests/ids.test.ts new file mode 100644 index 0000000..7c88a6c --- /dev/null +++ b/packages/shade-streams/tests/ids.test.ts @@ -0,0 +1,55 @@ +import { describe, test, expect } from 'bun:test'; +import { SubtleCryptoProvider } from '@shade/crypto-web'; +import { ValidationError } from '@shade/core'; +import { + generateStreamId, + generateStreamSecret, + streamIdToString, + streamIdFromString, + STREAM_ID_BYTES, + STREAM_SECRET_BYTES, +} from '../src/index.js'; + +const crypto = new SubtleCryptoProvider(); + +describe('streamId / streamSecret generators', () => { + test('streamId is 16 bytes', () => { + expect(generateStreamId(crypto).length).toBe(STREAM_ID_BYTES); + }); + + test('streamSecret is 32 bytes', () => { + expect(generateStreamSecret(crypto).length).toBe(STREAM_SECRET_BYTES); + }); + + test('successive generations are not equal (high-entropy)', () => { + const a = generateStreamId(crypto); + const b = generateStreamId(crypto); + expect(a).not.toEqual(b); + }); +}); + +describe('base64url encode/decode roundtrip', () => { + test('roundtrips arbitrary 16-byte streamIds', () => { + for (let i = 0; i < 50; i++) { + const id = generateStreamId(crypto); + const s = streamIdToString(id); + expect(streamIdFromString(s)).toEqual(id); + } + }); + + test('emits URL-safe alphabet (no +, /, =)', () => { + for (let i = 0; i < 50; i++) { + const s = streamIdToString(generateStreamId(crypto)); + expect(s).not.toMatch(/[+/=]/); + } + }); + + test('rejects wrong-length streamId on encode', () => { + expect(() => streamIdToString(new Uint8Array(15))).toThrow(ValidationError); + expect(() => streamIdToString(new Uint8Array(17))).toThrow(ValidationError); + }); + + test('rejects strings that decode to wrong length', () => { + expect(() => streamIdFromString('AAAA')).toThrow(ValidationError); + }); +}); diff --git a/packages/shade-streams/tests/kdf.test.ts b/packages/shade-streams/tests/kdf.test.ts new file mode 100644 index 0000000..6e0e8d1 --- /dev/null +++ b/packages/shade-streams/tests/kdf.test.ts @@ -0,0 +1,110 @@ +import { describe, test, expect } from 'bun:test'; +import { SubtleCryptoProvider } from '@shade/crypto-web'; +import { ValidationError } from '@shade/core'; +import { + deriveStreamKey, + deriveLaneKey, + generateStreamId, + generateStreamSecret, +} from '../src/index.js'; + +const crypto = new SubtleCryptoProvider(); + +function hex(bytes: Uint8Array): string { + return Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join(''); +} + +describe('deriveStreamKey', () => { + test('produces 32-byte output', async () => { + const secret = generateStreamSecret(crypto); + const id = generateStreamId(crypto); + const key = await deriveStreamKey(crypto, secret, id); + expect(key.length).toBe(32); + }); + + test('is deterministic for the same inputs', async () => { + const secret = new Uint8Array(32).fill(7); + const id = new Uint8Array(16).fill(3); + const a = await deriveStreamKey(crypto, secret, id); + const b = await deriveStreamKey(crypto, secret, id); + expect(hex(a)).toBe(hex(b)); + }); + + test('changes with streamSecret', async () => { + const id = new Uint8Array(16).fill(1); + const a = await deriveStreamKey(crypto, new Uint8Array(32).fill(1), id); + const b = await deriveStreamKey(crypto, new Uint8Array(32).fill(2), id); + expect(hex(a)).not.toBe(hex(b)); + }); + + test('changes with streamId', async () => { + const secret = new Uint8Array(32).fill(9); + const a = await deriveStreamKey(crypto, secret, new Uint8Array(16).fill(1)); + const b = await deriveStreamKey(crypto, secret, new Uint8Array(16).fill(2)); + expect(hex(a)).not.toBe(hex(b)); + }); + + test('rejects wrong-length streamSecret', async () => { + const id = new Uint8Array(16); + await expect(deriveStreamKey(crypto, new Uint8Array(31), id)).rejects.toThrow(ValidationError); + await expect(deriveStreamKey(crypto, new Uint8Array(33), id)).rejects.toThrow(ValidationError); + }); + + test('rejects wrong-length streamId', async () => { + const secret = new Uint8Array(32); + await expect(deriveStreamKey(crypto, secret, new Uint8Array(15))).rejects.toThrow(ValidationError); + await expect(deriveStreamKey(crypto, secret, new Uint8Array(17))).rejects.toThrow(ValidationError); + }); +}); + +describe('deriveLaneKey', () => { + test('produces 32-byte output', async () => { + const streamKey = new Uint8Array(32).fill(5); + const id = new Uint8Array(16).fill(2); + const laneKey = await deriveLaneKey(crypto, streamKey, id, 0); + expect(laneKey.length).toBe(32); + }); + + test('is deterministic for the same (streamKey, streamId, laneId)', async () => { + const streamKey = new Uint8Array(32).fill(5); + const id = new Uint8Array(16).fill(2); + const a = await deriveLaneKey(crypto, streamKey, id, 7); + const b = await deriveLaneKey(crypto, streamKey, id, 7); + expect(hex(a)).toBe(hex(b)); + }); + + test('different laneId yields different lane keys', async () => { + const streamKey = new Uint8Array(32).fill(5); + const id = new Uint8Array(16).fill(2); + const a = await deriveLaneKey(crypto, streamKey, id, 0); + const b = await deriveLaneKey(crypto, streamKey, id, 1); + expect(hex(a)).not.toBe(hex(b)); + }); + + test('different streamKey yields different lane keys', async () => { + const id = new Uint8Array(16).fill(2); + const a = await deriveLaneKey(crypto, new Uint8Array(32).fill(5), id, 0); + const b = await deriveLaneKey(crypto, new Uint8Array(32).fill(6), id, 0); + expect(hex(a)).not.toBe(hex(b)); + }); + + test('rejects laneId outside u32 range', async () => { + const streamKey = new Uint8Array(32); + const id = new Uint8Array(16); + await expect(deriveLaneKey(crypto, streamKey, id, -1)).rejects.toThrow(ValidationError); + await expect(deriveLaneKey(crypto, streamKey, id, 0x1_0000_0000)).rejects.toThrow( + ValidationError, + ); + await expect(deriveLaneKey(crypto, streamKey, id, 1.5)).rejects.toThrow(ValidationError); + }); + + test('full pipeline: streamSecret → streamKey → laneKey is deterministic across both sides', async () => { + const secret = new Uint8Array(32).fill(0xab); + const id = new Uint8Array(16).fill(0xcd); + const senderStreamKey = await deriveStreamKey(crypto, secret, id); + const senderLaneKey = await deriveLaneKey(crypto, senderStreamKey, id, 3); + const receiverStreamKey = await deriveStreamKey(crypto, secret, id); + const receiverLaneKey = await deriveLaneKey(crypto, receiverStreamKey, id, 3); + expect(hex(senderLaneKey)).toBe(hex(receiverLaneKey)); + }); +}); diff --git a/packages/shade-streams/tests/nonce.test.ts b/packages/shade-streams/tests/nonce.test.ts new file mode 100644 index 0000000..e9a256c --- /dev/null +++ b/packages/shade-streams/tests/nonce.test.ts @@ -0,0 +1,100 @@ +import { describe, test, expect } from 'bun:test'; +import { ValidationError } from '@shade/core'; +import { buildChunkNonce, buildChunkAad, MAX_SEQ, STREAM_NONCE_BYTES } from '../src/index.js'; + +describe('buildChunkNonce', () => { + test('produces 12-byte output', () => { + expect(buildChunkNonce(0, 0).length).toBe(STREAM_NONCE_BYTES); + }); + + test('encodes laneId as u32 BE in bytes [0..4)', () => { + const n = buildChunkNonce(0x01020304, 0); + expect(n[0]).toBe(0x01); + expect(n[1]).toBe(0x02); + expect(n[2]).toBe(0x03); + expect(n[3]).toBe(0x04); + }); + + test('encodes seq as u64 BE in bytes [4..12)', () => { + const n = buildChunkNonce(0, 0x0102030405060708n); + expect(n[4]).toBe(0x01); + expect(n[5]).toBe(0x02); + expect(n[6]).toBe(0x03); + expect(n[7]).toBe(0x04); + expect(n[8]).toBe(0x05); + expect(n[9]).toBe(0x06); + expect(n[10]).toBe(0x07); + expect(n[11]).toBe(0x08); + }); + + test('different (laneId, seq) yields different nonces', () => { + const seen = new Set(); + for (let lane = 0; lane < 4; lane++) { + for (let seq = 0; seq < 100; seq++) { + const n = buildChunkNonce(lane, seq); + const key = Array.from(n).join(','); + expect(seen.has(key)).toBe(false); + seen.add(key); + } + } + }); + + test('accepts both number and bigint seq', () => { + const a = buildChunkNonce(1, 42); + const b = buildChunkNonce(1, 42n); + expect(a).toEqual(b); + }); + + test('handles MAX_SEQ', () => { + const n = buildChunkNonce(0, MAX_SEQ); + for (let i = 4; i < 12; i++) expect(n[i]).toBe(0xff); + }); + + test('rejects out-of-range laneId', () => { + expect(() => buildChunkNonce(-1, 0)).toThrow(ValidationError); + expect(() => buildChunkNonce(0x1_0000_0000, 0)).toThrow(ValidationError); + expect(() => buildChunkNonce(1.5, 0)).toThrow(ValidationError); + }); + + test('rejects out-of-range seq', () => { + expect(() => buildChunkNonce(0, -1)).toThrow(ValidationError); + expect(() => buildChunkNonce(0, MAX_SEQ + 1n)).toThrow(ValidationError); + }); +}); + +describe('buildChunkAad', () => { + test('produces 29-byte output (16+4+8+1)', () => { + const id = new Uint8Array(16); + expect(buildChunkAad(id, 0, 0, false).length).toBe(29); + }); + + test('embeds streamId, laneId, seq, isLast in canonical layout', () => { + const id = new Uint8Array(16).fill(0xaa); + const aad = buildChunkAad(id, 0x01020304, 0x05060708090a0b0cn, true); + for (let i = 0; i < 16; i++) expect(aad[i]).toBe(0xaa); + expect(aad[16]).toBe(0x01); + expect(aad[17]).toBe(0x02); + expect(aad[18]).toBe(0x03); + expect(aad[19]).toBe(0x04); + expect(aad[20]).toBe(0x05); + expect(aad[27]).toBe(0x0c); + expect(aad[28]).toBe(0x01); + }); + + test('isLast=false sets last byte to 0x00', () => { + const id = new Uint8Array(16); + const aad = buildChunkAad(id, 0, 0, false); + expect(aad[28]).toBe(0x00); + }); + + test('rejects wrong-length streamId', () => { + expect(() => buildChunkAad(new Uint8Array(15), 0, 0, false)).toThrow(ValidationError); + expect(() => buildChunkAad(new Uint8Array(17), 0, 0, false)).toThrow(ValidationError); + }); + + test('rejects out-of-range laneId / seq', () => { + const id = new Uint8Array(16); + expect(() => buildChunkAad(id, -1, 0, false)).toThrow(ValidationError); + expect(() => buildChunkAad(id, 0, -1, false)).toThrow(ValidationError); + }); +}); diff --git a/packages/shade-streams/tests/partition.test.ts b/packages/shade-streams/tests/partition.test.ts new file mode 100644 index 0000000..dc3910f --- /dev/null +++ b/packages/shade-streams/tests/partition.test.ts @@ -0,0 +1,159 @@ +import { describe, test, expect } from 'bun:test'; +import { ValidationError } from '@shade/core'; +import { + planRangePartition, + planRoundRobinPartition, + chunkRange, + partitionsEqual, +} from '../src/index.js'; + +describe('planRangePartition', () => { + test('evenly divisible totalBytes', () => { + const lanes = planRangePartition(100, 4); + expect(lanes).toHaveLength(4); + expect(lanes[0]!.partition).toEqual({ kind: 'range', startByte: 0, endByte: 25, startChunk: 0 }); + expect(lanes[3]!.partition).toEqual({ kind: 'range', startByte: 75, endByte: 100, startChunk: 0 }); + }); + + test('non-divisible: extra bytes go to early lanes', () => { + const lanes = planRangePartition(10, 3); + const ranges = lanes.map((l) => l.partition); + // 10 / 3 = 3 remainder 1 — lane 0 gets 4, lanes 1+2 get 3 each + expect(ranges[0]).toEqual({ kind: 'range', startByte: 0, endByte: 4, startChunk: 0 }); + expect(ranges[1]).toEqual({ kind: 'range', startByte: 4, endByte: 7, startChunk: 0 }); + expect(ranges[2]).toEqual({ kind: 'range', startByte: 7, endByte: 10, startChunk: 0 }); + }); + + test('lanes cover entire range without gaps or overlap', () => { + for (const total of [0, 1, 7, 100, 1024 * 1024]) { + for (const count of [1, 2, 4, 16]) { + const lanes = planRangePartition(total, count); + let cursor = 0; + for (const lane of lanes) { + if (lane.partition.kind !== 'range') throw new Error('expected range'); + expect(lane.partition.startByte).toBe(cursor); + cursor = lane.partition.endByte; + } + expect(cursor).toBe(total); + } + } + }); + + test('1-lane partition spans entire input', () => { + const lanes = planRangePartition(500, 1); + expect(lanes).toHaveLength(1); + expect(lanes[0]!.partition).toEqual({ kind: 'range', startByte: 0, endByte: 500, startChunk: 0 }); + }); + + test('rejects negative totalBytes / fractional / non-positive count', () => { + expect(() => planRangePartition(-1, 1)).toThrow(ValidationError); + expect(() => planRangePartition(1.5, 1)).toThrow(ValidationError); + expect(() => planRangePartition(100, 0)).toThrow(ValidationError); + expect(() => planRangePartition(100, -1)).toThrow(ValidationError); + }); + + test('laneIds are 0..count-1 in order', () => { + const lanes = planRangePartition(64, 16); + for (let i = 0; i < 16; i++) expect(lanes[i]!.laneId).toBe(i); + }); +}); + +describe('planRoundRobinPartition', () => { + test('produces N lanes labeled 0..N-1', () => { + const lanes = planRoundRobinPartition(8); + expect(lanes).toHaveLength(8); + for (let i = 0; i < 8; i++) { + expect(lanes[i]!.laneId).toBe(i); + expect(lanes[i]!.partition).toEqual({ kind: 'round-robin', lane: i, count: 8 }); + } + }); + + test('rejects non-positive count', () => { + expect(() => planRoundRobinPartition(0)).toThrow(ValidationError); + expect(() => planRoundRobinPartition(-1)).toThrow(ValidationError); + }); +}); + +describe('chunkRange', () => { + test('splits an even range into chunkSize slices', () => { + expect(chunkRange(0, 1024, 256)).toEqual([ + { start: 0, end: 256 }, + { start: 256, end: 512 }, + { start: 512, end: 768 }, + { start: 768, end: 1024 }, + ]); + }); + + test('last chunk truncated for non-divisible range', () => { + expect(chunkRange(0, 1000, 256)).toEqual([ + { start: 0, end: 256 }, + { start: 256, end: 512 }, + { start: 512, end: 768 }, + { start: 768, end: 1000 }, + ]); + }); + + test('non-zero start offset is preserved', () => { + expect(chunkRange(100, 350, 100)).toEqual([ + { start: 100, end: 200 }, + { start: 200, end: 300 }, + { start: 300, end: 350 }, + ]); + }); + + test('empty range produces a single empty chunk (so isLast can be carried)', () => { + expect(chunkRange(50, 50, 100)).toEqual([{ start: 50, end: 50 }]); + }); + + test('rejects non-positive chunkSize', () => { + expect(() => chunkRange(0, 100, 0)).toThrow(ValidationError); + expect(() => chunkRange(0, 100, -1)).toThrow(ValidationError); + }); +}); + +describe('partitionsEqual', () => { + test('identical range partitions', () => { + expect( + partitionsEqual( + { kind: 'range', startByte: 0, endByte: 100, startChunk: 0 }, + { kind: 'range', startByte: 0, endByte: 100, startChunk: 0 }, + ), + ).toBe(true); + }); + + test('different range bounds', () => { + expect( + partitionsEqual( + { kind: 'range', startByte: 0, endByte: 100, startChunk: 0 }, + { kind: 'range', startByte: 0, endByte: 200, startChunk: 0 }, + ), + ).toBe(false); + }); + + test('range vs round-robin → false', () => { + expect( + partitionsEqual( + { kind: 'range', startByte: 0, endByte: 100, startChunk: 0 }, + { kind: 'round-robin', lane: 0, count: 1 }, + ), + ).toBe(false); + }); + + test('identical round-robin partitions', () => { + expect( + partitionsEqual( + { kind: 'round-robin', lane: 2, count: 4 }, + { kind: 'round-robin', lane: 2, count: 4 }, + ), + ).toBe(true); + }); + + test('different round-robin lane index', () => { + expect( + partitionsEqual( + { kind: 'round-robin', lane: 1, count: 4 }, + { kind: 'round-robin', lane: 2, count: 4 }, + ), + ).toBe(false); + }); +}); diff --git a/packages/shade-streams/tests/replay.test.ts b/packages/shade-streams/tests/replay.test.ts new file mode 100644 index 0000000..fa1d8d4 --- /dev/null +++ b/packages/shade-streams/tests/replay.test.ts @@ -0,0 +1,71 @@ +import { describe, test, expect } from 'bun:test'; +import { SubtleCryptoProvider } from '@shade/crypto-web'; +import { + StreamSender, + StreamReceiver, + StreamReplayError, + StreamOutOfOrderError, + generateStreamId, + generateStreamSecret, +} from '../src/index.js'; + +const crypto = new SubtleCryptoProvider(); + +async function pair() { + const streamId = generateStreamId(crypto); + const streamSecret = generateStreamSecret(crypto); + const sender = await StreamSender.create({ crypto, streamId, streamSecret, laneId: 0 }); + const receiver = await StreamReceiver.create({ crypto, streamId, streamSecret, laneId: 0 }); + return { sender, receiver }; +} + +describe('Replay and out-of-order detection', () => { + test('replaying the same chunk twice → StreamReplayError', async () => { + const { sender, receiver } = await pair(); + const { bytes } = await sender.encryptChunk(new TextEncoder().encode('first'), false); + await receiver.decryptChunk(bytes); + await expect(receiver.decryptChunk(bytes)).rejects.toThrow(StreamReplayError); + }); + + test('out-of-order chunk (skipping seq) → StreamOutOfOrderError', async () => { + const { sender, receiver } = await pair(); + await sender.encryptChunk(new TextEncoder().encode('a'), false); // seq 0 + const { bytes: c1 } = await sender.encryptChunk(new TextEncoder().encode('b'), false); // seq 1 + // Skip seq 0; send seq 1 first + await expect(receiver.decryptChunk(c1)).rejects.toThrow(StreamOutOfOrderError); + }); + + test('error contains expected and received seq', async () => { + const { sender, receiver } = await pair(); + await sender.encryptChunk(new TextEncoder().encode('skip'), false); // seq 0 produced + const { bytes: c1 } = await sender.encryptChunk(new TextEncoder().encode('next'), false); + try { + await receiver.decryptChunk(c1); + throw new Error('expected throw'); + } catch (err) { + expect((err as Error).message).toContain('expected seq=0'); + expect((err as Error).message).toContain('got 1'); + } + }); + + test('out-of-order then in-order: in-order chunk (after error) still works', async () => { + const { sender, receiver } = await pair(); + const { bytes: c0 } = await sender.encryptChunk(new TextEncoder().encode('a'), false); + const { bytes: c1 } = await sender.encryptChunk(new TextEncoder().encode('b'), true); + await expect(receiver.decryptChunk(c1)).rejects.toThrow(StreamOutOfOrderError); + // Receiver state still expects seq 0 (the error did not advance it) + const dec0 = await receiver.decryptChunk(c0); + expect(dec0.seq).toBe(0); + const dec1 = await receiver.decryptChunk(c1); + expect(dec1.seq).toBe(1); + }); + + test('replay after a different in-order chunk advanced seq → StreamReplayError', async () => { + const { sender, receiver } = await pair(); + const { bytes: c0 } = await sender.encryptChunk(new TextEncoder().encode('a'), false); + const { bytes: c1 } = await sender.encryptChunk(new TextEncoder().encode('b'), false); + await receiver.decryptChunk(c0); + await receiver.decryptChunk(c1); + await expect(receiver.decryptChunk(c0)).rejects.toThrow(StreamReplayError); + }); +}); diff --git a/packages/shade-streams/tests/sender-receiver.test.ts b/packages/shade-streams/tests/sender-receiver.test.ts new file mode 100644 index 0000000..9d801f4 --- /dev/null +++ b/packages/shade-streams/tests/sender-receiver.test.ts @@ -0,0 +1,176 @@ +import { describe, test, expect } from 'bun:test'; +import { SubtleCryptoProvider } from '@shade/crypto-web'; +import { + StreamSender, + StreamReceiver, + StreamFinishedError, + generateStreamId, + generateStreamSecret, +} from '../src/index.js'; + +const crypto = new SubtleCryptoProvider(); + +function hex(b: Uint8Array): string { + return Array.from(b, (x) => x.toString(16).padStart(2, '0')).join(''); +} + +async function makePair(opts?: { laneId?: number; startSeq?: number }) { + const streamId = generateStreamId(crypto); + const streamSecret = generateStreamSecret(crypto); + const laneId = opts?.laneId ?? 0; + const sender = await StreamSender.create({ + crypto, + streamId, + streamSecret, + laneId, + ...(opts?.startSeq !== undefined ? { startSeq: opts.startSeq } : {}), + }); + const receiver = await StreamReceiver.create({ + crypto, + streamId, + streamSecret, + laneId, + ...(opts?.startSeq !== undefined ? { startSeq: opts.startSeq } : {}), + }); + return { sender, receiver, streamId, streamSecret }; +} + +describe('Single-lane sender/receiver roundtrip', () => { + test('basic single-chunk transfer', async () => { + const { sender, receiver } = await makePair(); + const plaintext = new TextEncoder().encode('hello shade'); + const { bytes } = await sender.encryptChunk(plaintext, true); + const decrypted = await receiver.decryptChunk(bytes); + expect(new TextDecoder().decode(decrypted.plaintext)).toBe('hello shade'); + expect(decrypted.seq).toBe(0); + expect(decrypted.isLast).toBe(true); + }); + + test('multi-chunk transfer with monotonic seq', async () => { + const { sender, receiver } = await makePair(); + const chunks = ['alpha', 'beta', 'gamma', 'delta']; + for (let i = 0; i < chunks.length; i++) { + const isLast = i === chunks.length - 1; + const { bytes, seq } = await sender.encryptChunk( + new TextEncoder().encode(chunks[i]!), + isLast, + ); + expect(seq).toBe(i); + const dec = await receiver.decryptChunk(bytes); + expect(new TextDecoder().decode(dec.plaintext)).toBe(chunks[i]); + expect(dec.seq).toBe(i); + expect(dec.isLast).toBe(isLast); + } + }); + + test('lane sha256 matches between sender and receiver', async () => { + const { sender, receiver } = await makePair(); + const data = [ + crypto.randomBytes(1024), + crypto.randomBytes(2048), + crypto.randomBytes(512), + ]; + for (let i = 0; i < data.length; i++) { + const { bytes } = await sender.encryptChunk(data[i]!, i === data.length - 1); + await receiver.decryptChunk(bytes); + } + expect(hex(sender.getLaneSha256Digest())).toBe(hex(receiver.getLaneSha256Digest())); + }); + + test('handles empty chunks', async () => { + const { sender, receiver } = await makePair(); + const { bytes } = await sender.encryptChunk(new Uint8Array(0), true); + const dec = await receiver.decryptChunk(bytes); + expect(dec.plaintext.length).toBe(0); + expect(dec.isLast).toBe(true); + }); + + test('ship-gate: ~10 MiB roundtrip preserves byte-for-byte content', async () => { + const { sender, receiver } = await makePair(); + const total = 10 * 1024 * 1024; + const chunkSize = 256 * 1024; + const allBytes = crypto.randomBytes(total); + + const reconstructed: Uint8Array[] = []; + for (let off = 0; off < total; off += chunkSize) { + const slice = allBytes.subarray(off, Math.min(off + chunkSize, total)); + const isLast = off + chunkSize >= total; + const { bytes } = await sender.encryptChunk(slice, isLast); + const dec = await receiver.decryptChunk(bytes); + reconstructed.push(dec.plaintext); + } + + let off = 0; + for (const piece of reconstructed) { + for (let i = 0; i < piece.length; i++) { + if (piece[i] !== allBytes[off + i]) { + throw new Error(`mismatch at byte ${off + i}`); + } + } + off += piece.length; + } + expect(off).toBe(total); + expect(hex(sender.getLaneSha256Digest())).toBe(hex(receiver.getLaneSha256Digest())); + }); + + test('byte counters track encrypted/decrypted plaintext', async () => { + const { sender, receiver } = await makePair(); + const a = crypto.randomBytes(100); + const b = crypto.randomBytes(250); + const { bytes: ab } = await sender.encryptChunk(a, false); + const { bytes: bb } = await sender.encryptChunk(b, true); + await receiver.decryptChunk(ab); + await receiver.decryptChunk(bb); + expect(sender.bytesSent).toBe(350n); + expect(receiver.bytesReceived).toBe(350n); + }); + + test('finished flag set after isLast', async () => { + const { sender, receiver } = await makePair(); + const { bytes } = await sender.encryptChunk(new Uint8Array(8), true); + expect(sender.isFinished).toBe(true); + await receiver.decryptChunk(bytes); + expect(receiver.isFinished).toBe(true); + }); + + test('encryptChunk after finish throws StreamFinishedError', async () => { + const { sender } = await makePair(); + await sender.encryptChunk(new Uint8Array(0), true); + await expect(sender.encryptChunk(new Uint8Array(0), false)).rejects.toThrow( + StreamFinishedError, + ); + }); + + test('decryptChunk after finish throws StreamFinishedError', async () => { + const { sender, receiver } = await makePair(); + const { bytes: a } = await sender.encryptChunk(new Uint8Array(8), true); + await receiver.decryptChunk(a); + // Try to feed another chunk — sender wouldn't normally produce one, but + // simulate an attacker sending bytes after the legitimate isLast. + const sender2 = await StreamSender.create({ + crypto, + streamId: (sender as unknown as { streamId: Uint8Array }).streamId, + streamSecret: new Uint8Array(32), + laneId: 0, + }); + const { bytes: extra } = await sender2.encryptChunk(new Uint8Array(8), false); + await expect(receiver.decryptChunk(extra)).rejects.toThrow(StreamFinishedError); + }); + + test('destroy zeroes the lane key (subsequent calls throw)', async () => { + const { sender, receiver } = await makePair(); + sender.destroy(); + receiver.destroy(); + await expect(sender.encryptChunk(new Uint8Array(0), false)).rejects.toThrow( + StreamFinishedError, + ); + }); + + test('startSeq enables resume from arbitrary offset', async () => { + const { sender, receiver } = await makePair({ startSeq: 100 }); + const { bytes, seq } = await sender.encryptChunk(new TextEncoder().encode('mid'), false); + expect(seq).toBe(100); + const dec = await receiver.decryptChunk(bytes); + expect(dec.seq).toBe(100); + }); +}); diff --git a/packages/shade-streams/tests/tamper.test.ts b/packages/shade-streams/tests/tamper.test.ts new file mode 100644 index 0000000..303e6c7 --- /dev/null +++ b/packages/shade-streams/tests/tamper.test.ts @@ -0,0 +1,109 @@ +import { describe, test, expect } from 'bun:test'; +import { SubtleCryptoProvider } from '@shade/crypto-web'; +import { decodeStreamChunk, encodeStreamChunk } from '@shade/proto'; +import { + StreamSender, + StreamReceiver, + StreamDecryptionError, + StreamProtocolError, + generateStreamId, + generateStreamSecret, +} from '../src/index.js'; + +const crypto = new SubtleCryptoProvider(); + +async function pair() { + const streamId = generateStreamId(crypto); + const streamSecret = generateStreamSecret(crypto); + const sender = await StreamSender.create({ crypto, streamId, streamSecret, laneId: 0 }); + const receiver = await StreamReceiver.create({ crypto, streamId, streamSecret, laneId: 0 }); + return { sender, receiver }; +} + +describe('Tamper detection', () => { + test('flipping a ciphertext byte → StreamDecryptionError', async () => { + const { sender, receiver } = await pair(); + const { bytes } = await sender.encryptChunk(new TextEncoder().encode('payload'), false); + const env = decodeStreamChunk(bytes); + env.ciphertext[0] ^= 0x01; + const reencoded = encodeStreamChunk(env); + await expect(receiver.decryptChunk(reencoded)).rejects.toThrow(StreamDecryptionError); + }); + + test('flipping the AEAD tag → StreamDecryptionError', async () => { + const { sender, receiver } = await pair(); + const { bytes } = await sender.encryptChunk(new TextEncoder().encode('payload'), false); + const env = decodeStreamChunk(bytes); + env.ciphertext[env.ciphertext.length - 1] ^= 0x80; + await expect(receiver.decryptChunk(encodeStreamChunk(env))).rejects.toThrow( + StreamDecryptionError, + ); + }); + + test('tampering with isLast flag → StreamDecryptionError (AAD mismatch)', async () => { + const { sender, receiver } = await pair(); + const { bytes } = await sender.encryptChunk(new TextEncoder().encode('payload'), false); + const env = decodeStreamChunk(bytes); + env.isLast = true; + await expect(receiver.decryptChunk(encodeStreamChunk(env))).rejects.toThrow( + StreamDecryptionError, + ); + }); + + test('tampering with the wire nonce → StreamProtocolError (deterministic check)', async () => { + const { sender, receiver } = await pair(); + const { bytes } = await sender.encryptChunk(new TextEncoder().encode('payload'), false); + const env = decodeStreamChunk(bytes); + env.nonce[0] ^= 0x01; + await expect(receiver.decryptChunk(encodeStreamChunk(env))).rejects.toThrow( + StreamProtocolError, + ); + }); + + test('tampering with streamId → StreamProtocolError', async () => { + const { sender, receiver } = await pair(); + const { bytes } = await sender.encryptChunk(new TextEncoder().encode('payload'), false); + const env = decodeStreamChunk(bytes); + env.streamId[0] ^= 0xff; + await expect(receiver.decryptChunk(encodeStreamChunk(env))).rejects.toThrow( + StreamProtocolError, + ); + }); + + test('routing a chunk to wrong-lane receiver → StreamProtocolError', async () => { + const streamId = generateStreamId(crypto); + const streamSecret = generateStreamSecret(crypto); + const sender0 = await StreamSender.create({ crypto, streamId, streamSecret, laneId: 0 }); + const receiver1 = await StreamReceiver.create({ crypto, streamId, streamSecret, laneId: 1 }); + const { bytes } = await sender0.encryptChunk(new TextEncoder().encode('payload'), false); + await expect(receiver1.decryptChunk(bytes)).rejects.toThrow(StreamProtocolError); + }); + + test('non-empty AAD on wire → StreamProtocolError (reserved in v0.2.0)', async () => { + const { sender, receiver } = await pair(); + const { bytes } = await sender.encryptChunk(new TextEncoder().encode('payload'), false); + const env = decodeStreamChunk(bytes); + env.aad = new Uint8Array([1, 2, 3]); + await expect(receiver.decryptChunk(encodeStreamChunk(env))).rejects.toThrow( + StreamProtocolError, + ); + }); + + test('different streamSecret → StreamDecryptionError', async () => { + const streamId = generateStreamId(crypto); + const sender = await StreamSender.create({ + crypto, + streamId, + streamSecret: new Uint8Array(32).fill(1), + laneId: 0, + }); + const receiver = await StreamReceiver.create({ + crypto, + streamId, + streamSecret: new Uint8Array(32).fill(2), + laneId: 0, + }); + const { bytes } = await sender.encryptChunk(new TextEncoder().encode('hi'), false); + await expect(receiver.decryptChunk(bytes)).rejects.toThrow(StreamDecryptionError); + }); +}); diff --git a/packages/shade-streams/tsconfig.json b/packages/shade-streams/tsconfig.json new file mode 100644 index 0000000..a086b14 --- /dev/null +++ b/packages/shade-streams/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src"] +} diff --git a/packages/shade-transfer/package.json b/packages/shade-transfer/package.json new file mode 100644 index 0000000..f4743df --- /dev/null +++ b/packages/shade-transfer/package.json @@ -0,0 +1,21 @@ +{ + "name": "@shade/transfer", + "version": "0.3.0", + "type": "module", + "main": "src/index.ts", + "types": "src/index.ts", + "dependencies": { + "@shade/core": "workspace:*", + "@shade/crypto-web": "workspace:*", + "@shade/proto": "workspace:*", + "@shade/streams": "workspace:*" + }, + "peerDependencies": { + "hono": "^4" + }, + "peerDependenciesMeta": { + "hono": { + "optional": true + } + } +} diff --git a/packages/shade-transfer/src/engine.ts b/packages/shade-transfer/src/engine.ts new file mode 100644 index 0000000..64883d7 --- /dev/null +++ b/packages/shade-transfer/src/engine.ts @@ -0,0 +1,1484 @@ +import type { CryptoProvider } from '@shade/core'; +import { ValidationError } from '@shade/core'; +import { + MultiLaneReceiver, + MultiLaneSender, + StreamingSha256, + chunkRange, + generateStreamId, + generateStreamSecret, + isStreamControlMessage, + parseStreamControl, + planRangePartition, + planRoundRobinPartition, + streamIdFromString, + streamIdToString, + type LaneInitSpec, + type StreamControlMessage, + type StreamFinishMessage, + type StreamMetadata, +} from '@shade/streams'; +import { decodeStreamChunk, inspectEnvelopeType } from '@shade/proto'; +import { + TransferAbortError, + TransferIntegrityError, + TransferOfflineError, + TransferProtocolError, +} from './errors.js'; +import { ProgressTracker } from './progress.js'; +import { normalizeInput, type NormalizedInput } from './sender/input.js'; +import { BoundedAsyncQueue } from './sender/lane-queue.js'; +import { resolveOutputSink, type OutputSink } from './receiver/output.js'; +import { + DEFAULT_CHUNK_SIZE, + DEFAULT_LANE_COUNT, + DEFAULT_MAX_CHUNK_SIZE, + type IncomingTransfer, + type IncomingTransferAcceptOptions, + type LaneProgress, + type TransferEvent, + type TransferHandle, + type TransferOptions, + type TransferProgress, + type TransferResult, +} from './types.js'; +import type { + ChunkAck, + IControlChannel, + ITransferTransport, + TransferResumeState, +} from './transport/transport.js'; +import { + type ResumeStore, + wrapStreamSecret, + unwrapStreamSecret, +} from './persistence/resume.js'; +import type { PersistedStreamState } from '@shade/core'; + +export interface TransferEngineDeps { + crypto: CryptoProvider; + controlChannel: IControlChannel; + transport: ITransferTransport; + /** This endpoint's logical address (e.g. `device:server`). */ + myAddress: string; + /** Optional SubtleCrypto instance for AEAD operations. */ + subtle?: SubtleCrypto; + /** Optional resume store; when provided, in-flight state is persisted. */ + resumeStore?: ResumeStore; + /** + * Optional 32-byte deviceKey for at-rest streamSecret encryption. Required + * when `resumeStore` persists across processes (SQLite/Postgres/IndexedDB). + * For in-memory stores it can be omitted (secret stays encrypted with a + * random per-process key). + */ + deviceKey?: Uint8Array; +} + +/** + * The transfer orchestrator. + * + * Wraps an `IControlChannel` (for stream-init/finish/abort messages — usually + * over the existing Shade ratchet) and an `ITransferTransport` (for chunk + * data — usually HTTP). Provides `upload()` and `onIncomingTransfer()` entry + * points; the SDK adds these to the `Shade` class in M-Stream-5. + */ +export class TransferEngine { + private readonly outgoing = new Map(); + private readonly incoming = new Map(); + private readonly incomingHandlers = new Set< + (incoming: IncomingTransfer) => void | Promise + >(); + private readonly unsubscribeControl: () => void; + private readonly persister: Persister | null; + + constructor(private readonly deps: TransferEngineDeps) { + this.persister = + deps.resumeStore !== undefined + ? new Persister(deps.crypto, deps.resumeStore, deps.deviceKey) + : null; + this.unsubscribeControl = deps.controlChannel.onMessage(async (from, message) => { + try { + await this.handleControlMessage(from, message); + } catch (err) { + console.error('[TransferEngine] control message error:', err); + throw err; + } + }); + } + + /** Start an upload. Returns a handle that can be paused/aborted/awaited. */ + async upload(opts: TransferOptions): Promise { + const input = await normalizeInput(opts.input); + const chunkSize = opts.chunkSize ?? DEFAULT_CHUNK_SIZE; + const maxChunkSize = opts.maxChunkSize ?? DEFAULT_MAX_CHUNK_SIZE; + if (chunkSize < 1 || chunkSize > maxChunkSize) { + throw new ValidationError( + `chunkSize ${chunkSize} out of range (1, ${maxChunkSize})`, + 'chunkSize', + ); + } + + const requestedLanes = opts.lanes ?? DEFAULT_LANE_COUNT; + const laneCount = computeEffectiveLaneCount(requestedLanes, input.size, chunkSize); + + const partitionMode = resolvePartition(opts.partition, input); + + const lanes: LaneInitSpec[] = + partitionMode === 'range' + ? planRangePartition(input.size ?? 0, laneCount) + : planRoundRobinPartition(laneCount); + + // Probe peer reachability up front (NF: clear offline error). + try { + await this.deps.transport.probe(opts.to); + } catch (err) { + await input.close(); + throw new TransferOfflineError(opts.to, (err as Error).message); + } + + const streamIdBytes = generateStreamId(this.deps.crypto); + const streamSecret = generateStreamSecret(this.deps.crypto); + const streamId = streamIdToString(streamIdBytes); + + const metadata: StreamMetadata = { + ...(input.name !== undefined ? { name: input.name } : {}), + ...(input.size !== undefined ? { sizeBytes: input.size } : {}), + ...(input.contentType !== undefined ? { contentType: input.contentType } : {}), + chunkSize, + sentAt: Date.now(), + ...(opts.metadata ?? {}), + }; + if (input.size !== undefined) { + metadata.totalChunks = Math.max(1, Math.ceil(input.size / chunkSize)); + } + + const sender = await MultiLaneSender.create({ + crypto: this.deps.crypto, + ...(this.deps.subtle !== undefined ? { subtle: this.deps.subtle } : {}), + streamId: streamIdBytes, + streamSecret, + lanes, + }); + + const state = new OutgoingState( + streamId, + opts.to, + sender, + metadata, + lanes, + input, + chunkSize, + opts.signal, + opts.onProgress, + opts.onEvent, + ); + this.outgoing.set(streamId, state); + + // Persist initial resume state BEFORE sending init, so a crash before + // first chunk leaves a recoverable record on disk. + if (this.persister !== null) { + await this.persister.saveOutgoing(state, streamSecret); + } + + // Send stream-init plaintext over the control channel. + await this.deps.controlChannel.send(opts.to, { + kind: 'shade.stream-init/v1', + streamId, + streamSecret: bytesToBase64Url(streamSecret), + metadata, + lanes, + }); + + // Spawn the upload pipeline (read → encrypt → transport). + state.startedAt = nowMs(); + state.emit({ type: 'start', streamId }); + void this.runUpload(state, partitionMode).catch(async (err) => { + await state.failTransfer(err); + }); + + return state.handle; + } + + /** Subscribe to incoming transfers. */ + onIncomingTransfer( + handler: (incoming: IncomingTransfer) => void | Promise, + ): () => void { + this.incomingHandlers.add(handler); + return () => this.incomingHandlers.delete(handler); + } + + /** + * Resume an interrupted upload. The consumer re-supplies the input + * (browser File handle / Bun path / Uint8Array — must be byte-identical + * to the original) and we pick up where we left off. + * + * Flow: + * 1. Load persisted state, decrypt streamSecret. + * 2. Query the receiver for its `lastSeqAcked` per lane. + * 3. Spawn a new MultiLaneSender with `startSeq = lastSeqAcked + 1` per lane. + * 4. Pre-hash already-shipped bytes through each lane's hasher. + * 5. Continue the upload from the resumption point. + * + * `opts.onProgress` / `opts.onEvent` overrides apply to the resumed run. + */ + async resumeUpload( + streamId: string, + freshInput: TransferOptions['input'], + opts?: { + onProgress?: (p: TransferProgress) => void; + onEvent?: (e: TransferEvent) => void; + signal?: AbortSignal; + }, + ): Promise { + if (this.persister === null) { + throw new TransferProtocolError('resumeUpload requires a resume store'); + } + const loaded = await this.persister.load(streamId); + if (loaded === null) { + throw new TransferProtocolError(`no persisted state for streamId=${streamId}`); + } + if (loaded.record.direction !== 'send') { + throw new TransferProtocolError(`stream ${streamId} is a receive, not a send`); + } + if (loaded.record.status === 'finished' || loaded.record.status === 'aborted') { + throw new TransferProtocolError( + `stream ${streamId} is already ${loaded.record.status}`, + ); + } + const peer = loaded.record.peerAddress; + const metadata = JSON.parse(loaded.record.metadataJson) as import('@shade/streams').StreamMetadata; + const lanes = JSON.parse(loaded.record.partitionJson) as import('@shade/streams').LaneInitSpec[]; + + const remoteState = await this.deps.transport.fetchResumeState(peer, streamId); + const lastSeqByLane = new Map(); + if (remoteState !== null) { + for (const l of remoteState.lanes) lastSeqByLane.set(l.laneId, l.lastSeqAcked); + } + for (const lane of lanes) { + if (!lastSeqByLane.has(lane.laneId)) lastSeqByLane.set(lane.laneId, -1); + } + + const startSeqByLane = new Map(); + for (const [laneId, last] of lastSeqByLane) startSeqByLane.set(laneId, last + 1); + + const input = await normalizeInput(freshInput); + const sender = await MultiLaneSender.create({ + crypto: this.deps.crypto, + ...(this.deps.subtle !== undefined ? { subtle: this.deps.subtle } : {}), + streamId: bytesFromBase64Url(loaded.record.streamId), + streamSecret: loaded.streamSecret, + lanes, + startSeqByLane, + }); + + // For range mode the prefix bytes per lane will be folded into both the + // per-lane hasher AND the overall hasher BEFORE the lane's remainder is + // streamed (see runRangeUploadResumed). For round-robin we re-feed the + // common prefix here so the new sender's overall hasher matches the + // receiver's reconstructed-order view. + if (lanes[0]!.partition.kind === 'round-robin') { + let maxAcked = -1; + for (const last of lastSeqByLane.values()) { + if (last > maxAcked) maxAcked = last; + } + if (maxAcked >= 0 && input.size !== undefined) { + const totalToReHash = (maxAcked + 1) * lanes.length; + let cursor = 0; + for (let i = 0; i < totalToReHash && cursor < input.size; i++) { + const end = Math.min(cursor + metadata.chunkSize, input.size); + const slice = await input.read(cursor, end); + const laneId = i % lanes.length; + sender.preHashForLane(laneId, slice); + sender.appendOverall(slice); + cursor = end; + } + } + } + + const partitionMode = lanes[0]!.partition.kind; + const state = new OutgoingState( + streamId, + peer, + sender, + metadata, + lanes, + input, + metadata.chunkSize, + opts?.signal, + opts?.onProgress, + opts?.onEvent, + ); + // Mark already-shipped lanes as having advanced. + for (const lane of lanes) { + const lp = (state as unknown as { laneProgress: Map }).laneProgress.get( + lane.laneId, + ); + if (lp !== undefined) { + lp.seq = lastSeqByLane.get(lane.laneId)! + 1; + } + } + this.outgoing.set(streamId, state); + state.startedAt = nowMs(); + state.emit({ type: 'start', streamId }); + void this.runResumedUpload(state, partitionMode, lastSeqByLane).catch(async (err) => { + await state.failTransfer(err); + }); + return state.handle; + } + + private async runResumedUpload( + state: OutgoingState, + partitionMode: 'range' | 'round-robin', + lastSeqByLane: Map, + ): Promise { + if (partitionMode === 'range') { + await this.runRangeUploadResumed(state, lastSeqByLane); + } else { + await this.runRoundRobinUploadResumed(state, lastSeqByLane); + } + if (state.aborted) return; + + const laneFps = state.sender.getLaneFingerprints(); + const overall = state.sender.getOverallSha256(); + await this.deps.controlChannel.send(state.peerAddress, { + kind: 'shade.stream-finish/v1', + streamId: state.streamId, + laneSha256: laneFps.map((f) => ({ + laneId: f.laneId, + sha256: bytesToBase64(f.sha256), + chunkCount: f.chunkCount, + byteCount: f.byteCount, + })), + overallSha256: bytesToBase64(overall), + finishedAt: Date.now(), + }); + if (this.persister !== null) { + await this.persister.markStatus(state.streamId, 'finished'); + } + state.complete({ + streamId: state.streamId, + bytesSent: laneFps.reduce((s, l) => s + l.byteCount, 0), + sha256: hex(overall), + durationMs: nowMs() - state.startedAt, + }); + this.outgoing.delete(state.streamId); + } + + private async runRangeUploadResumed( + state: OutgoingState, + lastSeqByLane: Map, + ): Promise { + const queues = new Map>(); + const workers: Promise[] = []; + const flightCap = 4; + for (const lane of state.lanes) { + const q = new BoundedAsyncQueue(flightCap); + queues.set(lane.laneId, q); + workers.push(this.runLaneWorker(state, lane.laneId, q)); + } + try { + // Process lanes in laneId order so the OVERALL hasher receives bytes + // in original order (range mode = lane k owns bytes [k·N/L, (k+1)·N/L)). + // Per lane: re-hash the already-acked prefix, then stream the remainder. + for (const lane of state.lanes) { + if (lane.partition.kind !== 'range') { + throw new ValidationError(`expected range partition`, 'lane'); + } + const lastAcked = lastSeqByLane.get(lane.laneId) ?? -1; + const allSlices = chunkRange( + lane.partition.startByte, + lane.partition.endByte, + state.chunkSize, + ); + // Re-hash prefix (chunks 0..lastAcked) into both per-lane and overall hashers. + for (let i = 0; i <= lastAcked && i < allSlices.length; i++) { + if (state.aborted) break; + const s = allSlices[i]!; + const slice = await state.input.read(s.start, s.end); + state.sender.preHashForLane(lane.laneId, slice); + state.sender.appendOverall(slice); + } + // Stream remainder. + for (let i = lastAcked + 1; i < allSlices.length; i++) { + if (state.aborted) break; + const s = allSlices[i]!; + const isLast = i === allSlices.length - 1; + const plaintext = await state.input.read(s.start, s.end); + state.sender.appendOverall(plaintext); + await queues.get(lane.laneId)!.push({ plaintext, isLast }); + } + queues.get(lane.laneId)!.close(); + } + } catch (err) { + for (const q of queues.values()) q.abort(err); + throw err; + } + await Promise.all(workers); + } + + private async runRoundRobinUploadResumed( + state: OutgoingState, + lastSeqByLane: Map, + ): Promise { + const queues = new Map>(); + const workers: Promise[] = []; + const flightCap = 4; + const laneCount = state.lanes.length; + for (const lane of state.lanes) { + const q = new BoundedAsyncQueue(flightCap); + queues.set(lane.laneId, q); + workers.push(this.runLaneWorker(state, lane.laneId, q)); + } + + if (state.input.size === undefined) { + throw new TransferProtocolError( + 'round-robin resume requires known-size input (random access)', + ); + } + + let maxAcked = -1; + for (const last of lastSeqByLane.values()) { + if (last > maxAcked) maxAcked = last; + } + const skipChunks = (maxAcked + 1) * laneCount; + let cursor = skipChunks * state.chunkSize; + cursor = Math.min(cursor, state.input.size); + let chunkIndex = skipChunks; + + try { + while (cursor < state.input.size) { + if (state.aborted) break; + const end = Math.min(cursor + state.chunkSize, state.input.size); + const buf = await state.input.read(cursor, end); + cursor = end; + const isEof = cursor >= state.input.size; + const laneId = chunkIndex % laneCount; + state.sender.appendOverall(buf); + // Determine isLast: if EOF, this is the last chunk for this lane. + // Other lanes that received fewer chunks need an empty isLast. + const isLast = isEof; // simplification: we mark the very last chunk + await queues.get(laneId)!.push({ plaintext: buf, isLast }); + chunkIndex++; + } + // Close queues; any lane that didn't get an isLast chunk needs one. + // Compute per-lane sent counts and emit empty isLast if needed. + const totalChunksSent = chunkIndex; + const lastLaneId = (totalChunksSent - 1 + laneCount) % laneCount; + void lastLaneId; + for (const lane of state.lanes) { + // Count: chunks for this lane = ceil((total - laneId) / L) if laneId < total mod L else floor + const sentForLane = + Math.floor((totalChunksSent - lane.laneId - 1) / laneCount) + 1; + const expectedAcked = lastSeqByLane.get(lane.laneId)! + 1; + if (sentForLane <= expectedAcked) { + // This lane never got a new chunk; push empty isLast=true. + await queues.get(lane.laneId)!.push({ plaintext: new Uint8Array(0), isLast: true }); + } + queues.get(lane.laneId)!.close(); + } + } catch (err) { + for (const q of queues.values()) q.abort(err); + throw err; + } + await Promise.all(workers); + } + + /** + * Receiver-side: accept a chunk envelope. Called by the transport handler + * (HTTP route or memory-pipe). Returns the ACK that the transport relays + * back to the sender. + */ + async receiveChunk( + from: string, + streamId: string, + laneId: number, + seq: number | bigint, + bytes: Uint8Array, + ): Promise { + if (inspectEnvelopeType(bytes) !== 'stream-chunk') { + throw new TransferProtocolError('not a stream-chunk envelope'); + } + const state = this.incoming.get(streamId); + if (state === undefined) { + throw new TransferProtocolError(`no incoming transfer for streamId=${streamId}`); + } + if (state.peerAddress !== from) { + throw new TransferProtocolError(`peer mismatch: ${from} vs ${state.peerAddress}`); + } + const env = decodeStreamChunk(bytes); + if (env.laneId !== laneId) { + throw new TransferProtocolError(`laneId mismatch: header ${env.laneId} vs route ${laneId}`); + } + const envSeq = typeof env.seq === 'bigint' ? env.seq : BigInt(env.seq); + const expected = typeof seq === 'bigint' ? seq : BigInt(seq); + if (envSeq !== expected) { + throw new TransferProtocolError(`seq mismatch: header ${envSeq} vs route ${expected}`); + } + return state.handleChunk(bytes); + } + + /** Receiver-side: respond to a resume-state lookup. */ + async getResumeState( + from: string, + streamId: string, + ): Promise { + const state = this.incoming.get(streamId); + if (state === undefined || state.peerAddress !== from) return null; + return { + streamId, + lanes: state.lanesProgressForResume(), + }; + } + + /** Tear down all subscriptions. */ + destroy(): void { + this.unsubscribeControl(); + for (const o of this.outgoing.values()) void o.failTransfer(new TransferAbortError('engine destroyed')); + for (const i of this.incoming.values()) void i.abort('engine destroyed'); + this.outgoing.clear(); + this.incoming.clear(); + this.incomingHandlers.clear(); + } + + // ─── Internals ───────────────────────────────────────────── + + private async handleControlMessage( + from: string, + raw: StreamControlMessage | unknown, + ): Promise { + let msg: StreamControlMessage; + if (typeof raw === 'string') { + msg = parseStreamControl(raw); + } else if (isStreamControlMessage(raw)) { + msg = raw; + } else { + return; // Ignore non-stream control messages. + } + + switch (msg.kind) { + case 'shade.stream-init/v1': + await this.handleIncomingInit(from, msg); + return; + case 'shade.stream-finish/v1': + await this.handleIncomingFinish(from, msg); + return; + case 'shade.stream-abort/v1': + await this.handleIncomingAbort(from, msg); + return; + case 'shade.stream-resume-request/v1': + case 'shade.stream-resume-state/v1': + // Resume protocol wires up in M-Stream-6. Ignore in v0.2.0 baseline. + return; + } + } + + private async handleIncomingInit( + from: string, + msg: import('@shade/streams').StreamInitMessage, + ): Promise { + const streamIdBytes = streamIdFromString(msg.streamId); + const streamSecret = base64UrlToBytes(msg.streamSecret); + if (this.incoming.has(msg.streamId)) { + throw new TransferProtocolError(`duplicate stream-init for streamId=${msg.streamId}`); + } + const receiver = await MultiLaneReceiver.create({ + crypto: this.deps.crypto, + ...(this.deps.subtle !== undefined ? { subtle: this.deps.subtle } : {}), + streamId: streamIdBytes, + streamSecret, + lanes: msg.lanes, + }); + const state = new IncomingState( + msg.streamId, + from, + receiver, + msg.metadata, + msg.lanes, + ); + this.incoming.set(msg.streamId, state); + if (this.persister !== null) { + await this.persister.saveIncoming(state, streamSecret); + } + + const incoming: IncomingTransfer = { + streamId: msg.streamId, + from, + metadata: msg.metadata, + lanes: msg.lanes, + accept: async (opts: IncomingTransferAcceptOptions) => { + if (state.accepted) { + throw new TransferProtocolError('already accepted'); + } + state.accepted = true; + state.sink = await resolveOutputSink(opts.output); + state.outputKind = opts.output.kind; + if (opts.onProgress !== undefined) state.onProgress = opts.onProgress; + if (opts.onEvent !== undefined) state.onEvent = opts.onEvent; + state.startedAt = nowMs(); + state.emit({ type: 'start', streamId: msg.streamId }); + return state.handle; + }, + decline: async (reason?: string) => { + state.declined = true; + await state.abort(reason ?? 'receiver-decline'); + }, + }; + + for (const handler of this.incomingHandlers) { + try { + await handler(incoming); + } catch (err) { + console.error('[TransferEngine] incoming handler error:', err); + } + } + } + + private async handleIncomingFinish( + from: string, + msg: StreamFinishMessage, + ): Promise { + const state = this.incoming.get(msg.streamId); + if (state === undefined || state.peerAddress !== from) return; + await state.finalize(msg); + this.incoming.delete(msg.streamId); + } + + private async handleIncomingAbort( + from: string, + msg: import('@shade/streams').StreamAbortMessage, + ): Promise { + const state = this.incoming.get(msg.streamId); + if (state === undefined || state.peerAddress !== from) return; + await state.abort(msg.reason); + this.incoming.delete(msg.streamId); + } + + private async runUpload( + state: OutgoingState, + partitionMode: 'range' | 'round-robin', + ): Promise { + if (partitionMode === 'range') { + await this.runRangeUpload(state); + } else { + await this.runRoundRobinUpload(state); + } + + if (state.aborted) return; + + // Compute fingerprints + send finish. + const laneFps = state.sender.getLaneFingerprints(); + const overall = state.sender.getOverallSha256(); + const finishMsg: StreamFinishMessage = { + kind: 'shade.stream-finish/v1', + streamId: state.streamId, + laneSha256: laneFps.map((f) => ({ + laneId: f.laneId, + sha256: bytesToBase64(f.sha256), + chunkCount: f.chunkCount, + byteCount: f.byteCount, + })), + overallSha256: bytesToBase64(overall), + finishedAt: Date.now(), + }; + await this.deps.controlChannel.send(state.peerAddress, finishMsg); + + const result: TransferResult = { + streamId: state.streamId, + bytesSent: laneFps.reduce((s, l) => s + l.byteCount, 0), + sha256: hex(overall), + durationMs: nowMs() - state.startedAt, + }; + if (this.persister !== null) { + await this.persister.markStatus(state.streamId, 'finished'); + } + state.complete(result); + this.outgoing.delete(state.streamId); + } + + private async runRangeUpload(state: OutgoingState): Promise { + // Single sequential reader; per-lane queue + worker. + const queues = new Map>(); + const workers: Promise[] = []; + const flightCap = 4; + for (const lane of state.lanes) { + const q = new BoundedAsyncQueue(flightCap); + queues.set(lane.laneId, q); + workers.push(this.runLaneWorker(state, lane.laneId, q)); + } + try { + for (const lane of state.lanes) { + if (lane.partition.kind !== 'range') { + throw new ValidationError(`expected range partition for lane ${lane.laneId}`, 'lane'); + } + const slices = chunkRange( + lane.partition.startByte, + lane.partition.endByte, + state.chunkSize, + ); + for (let i = 0; i < slices.length; i++) { + const s = slices[i]!; + const isLast = i === slices.length - 1; + if (state.aborted) break; + const plaintext = await state.input.read(s.start, s.end); + state.sender.appendOverall(plaintext); + await queues.get(lane.laneId)!.push({ plaintext, isLast }); + } + queues.get(lane.laneId)!.close(); + } + } catch (err) { + for (const q of queues.values()) q.abort(err); + throw err; + } + await Promise.all(workers); + } + + private async runRoundRobinUpload(state: OutgoingState): Promise { + const queues = new Map>(); + const workers: Promise[] = []; + const flightCap = 4; + const laneCount = state.lanes.length; + for (const lane of state.lanes) { + const q = new BoundedAsyncQueue(flightCap); + queues.set(lane.laneId, q); + workers.push(this.runLaneWorker(state, lane.laneId, q)); + } + + // Source iterator: random-access inputs get an `input.read(start, end)` + // walk; non-random-access inputs use `input.readNext`. + const readNextChunk: () => Promise = + state.input.randomAccess && state.input.size !== undefined + ? (() => { + let cursor = 0; + const total = state.input.size; + return async () => { + if (cursor >= total) return null; + const end = Math.min(cursor + state.chunkSize, total); + const buf = await state.input.read(cursor, end); + cursor = end; + return buf; + }; + })() + : () => state.input.readNext(state.chunkSize); + + // Read-pump: pull chunks sequentially until EOF. + // We can't mark isLast=true until we know it's truly the last chunk for + // a lane, so we keep a one-chunk lookahead per lane: the most recent + // chunk for each lane sits as a "tail" until either superseded (becomes + // a body chunk) or finalized as isLast at EOF. + interface TailRecord { + chunkIndex: number; + plaintext: Uint8Array; + } + const tailByLane = new Map(); + let chunkIndex = 0; + try { + while (true) { + if (state.aborted) break; + const buf = await readNextChunk(); + if (buf === null) break; + const laneId = chunkIndex % laneCount; + const previousTail = tailByLane.get(laneId); + if (previousTail !== undefined) { + state.sender.appendOverall(previousTail.plaintext); + await queues.get(laneId)!.push({ plaintext: previousTail.plaintext, isLast: false }); + } + tailByLane.set(laneId, { chunkIndex, plaintext: buf }); + chunkIndex++; + } + // At EOF, flush remaining tails in CHUNK-INDEX order (original byte + // order) so appendOverall stays consistent with the receiver. Lanes + // that never saw a chunk get an empty isLast=true to mark completion. + const tailsInOrder = [...tailByLane.entries()] + .map(([laneId, t]) => ({ laneId, chunkIndex: t.chunkIndex, plaintext: t.plaintext })) + .sort((a, b) => a.chunkIndex - b.chunkIndex); + for (const t of tailsInOrder) { + state.sender.appendOverall(t.plaintext); + await queues.get(t.laneId)!.push({ plaintext: t.plaintext, isLast: true }); + } + // Lanes that received nothing — push empty isLast. + for (const lane of state.lanes) { + if (!tailByLane.has(lane.laneId)) { + await queues.get(lane.laneId)!.push({ plaintext: new Uint8Array(0), isLast: true }); + } + } + for (const q of queues.values()) q.close(); + } catch (err) { + for (const q of queues.values()) q.abort(err); + throw err; + } + await Promise.all(workers); + } + + private async runLaneWorker( + state: OutgoingState, + laneId: number, + queue: BoundedAsyncQueue, + ): Promise { + while (true) { + const r = await queue.next(); + if (r.done) return; + if (state.aborted) return; + const { plaintext, isLast } = r.value; + const { bytes, seq } = await state.sender.encryptForLane(laneId, plaintext, isLast); + await this.deps.transport.sendChunk( + state.peerAddress, + state.streamId, + laneId, + seq, + bytes, + ...(state.signal !== undefined ? [{ signal: state.signal }] : []), + ); + state.recordSent(laneId, plaintext.length, seq, isLast); + } + } +} + +// ─── Internal state ────────────────────────────────────────── + +interface ChunkJob { + plaintext: Uint8Array; + isLast: boolean; +} + +class OutgoingState { + startedAt = -1; + aborted = false; + private completedAt = -1; + private resolveDone!: (result: TransferResult) => void; + private rejectDone!: (err: unknown) => void; + private donePromise: Promise; + private events: TransferEvent[] = []; + private eventResolvers: Array<(value: IteratorResult) => void> = []; + private progress: ProgressTracker; + private laneProgress: Map; + + readonly handle: TransferHandle; + + constructor( + public readonly streamId: string, + public readonly peerAddress: string, + public readonly sender: MultiLaneSender, + public readonly metadata: StreamMetadata, + public readonly lanes: LaneInitSpec[], + public readonly input: NormalizedInput, + public readonly chunkSize: number, + public readonly signal: AbortSignal | undefined, + public onProgress: ((p: TransferProgress) => void) | undefined, + public onEvent: ((e: TransferEvent) => void) | undefined, + ) { + this.donePromise = new Promise((resolve, reject) => { + this.resolveDone = resolve; + this.rejectDone = reject; + }); + this.progress = new ProgressTracker(metadata.sizeBytes); + this.laneProgress = new Map( + lanes.map((l) => [ + l.laneId, + { + laneId: l.laneId, + bytesSent: 0, + ...(l.partition.kind === 'range' + ? { bytesTotal: l.partition.endByte - l.partition.startByte } + : {}), + seq: 0, + state: 'idle' as const, + }, + ]), + ); + if (signal !== undefined) { + signal.addEventListener('abort', () => { + void this.failTransfer(new TransferAbortError('signal aborted')); + }); + } + this.handle = { + streamId: this.streamId, + events: this.eventsAsyncIterable(), + pause: async () => { + /* M-Stream-6 territory; placeholder no-op for v0.2.0 baseline. */ + }, + resume: async () => { + /* M-Stream-6 territory. */ + }, + abort: async (reason) => { + await this.failTransfer(new TransferAbortError(reason)); + }, + done: () => this.donePromise, + }; + } + + recordSent(laneId: number, bytes: number, seq: number, isLast: boolean): void { + const lane = this.laneProgress.get(laneId); + if (lane !== undefined) { + lane.bytesSent += bytes; + lane.seq = seq + 1; + lane.state = isLast ? 'done' : 'sending'; + if (isLast) this.emit({ type: 'lane-done', laneId }); + } + this.progress.add(bytes); + const sample = this.progress.sample(); + const tp: TransferProgress = { + streamId: this.streamId, + bytesSent: this.progress.totalProcessed, + bytesPerSecond: sample.bytesPerSecond, + lanes: [...this.laneProgress.values()], + ...(this.metadata.sizeBytes !== undefined ? { bytesTotal: this.metadata.sizeBytes } : {}), + ...(sample.etaSeconds !== undefined ? { etaSeconds: sample.etaSeconds } : {}), + ...(sample.percent !== undefined ? { percent: sample.percent } : {}), + }; + this.onProgress?.(tp); + this.emit({ type: 'progress', progress: tp }); + } + + emit(event: TransferEvent): void { + this.onEvent?.(event); + if (this.eventResolvers.length > 0) { + this.eventResolvers.shift()!({ value: event, done: false }); + } else { + this.events.push(event); + } + } + + complete(result: TransferResult): void { + this.completedAt = nowMs(); + this.emit({ type: 'complete', streamId: result.streamId, sha256: result.sha256, durationMs: result.durationMs }); + this.flushEventEnd(); + this.resolveDone(result); + void this.input.close(); + this.sender.destroy(); + } + + async failTransfer(err: unknown): Promise { + if (this.aborted || this.completedAt > 0) return; + this.aborted = true; + this.emit({ type: 'error', error: err }); + this.flushEventEnd(); + try { + await this.input.close(); + } catch { + /* swallow */ + } + this.sender.destroy(); + this.rejectDone(err); + } + + private flushEventEnd(): void { + for (const r of this.eventResolvers) r({ value: undefined as never, done: true }); + this.eventResolvers = []; + } + + private eventsAsyncIterable(): AsyncIterable { + const self = this; + return { + [Symbol.asyncIterator](): AsyncIterator { + return { + next(): Promise> { + if (self.events.length > 0) { + return Promise.resolve({ value: self.events.shift()!, done: false }); + } + if (self.aborted || self.completedAt > 0) { + return Promise.resolve({ value: undefined as never, done: true }); + } + return new Promise((resolve) => self.eventResolvers.push(resolve)); + }, + }; + }, + }; + } +} + +class IncomingState { + startedAt = -1; + accepted = false; + declined = false; + sink: OutputSink | null = null; + outputKind: 'pipe' | 'callback' | 'buffer' | 'file' | 'fileHandle' | undefined; + onProgress: ((p: TransferProgress) => void) | undefined; + onEvent: ((e: TransferEvent) => void) | undefined; + + private bufferedByLane = new Map>(); + private rangeOrderCursor: Map = new Map(); // laneId → expected seq for sink writes + private currentLaneForSink = 0; // for range mode: which lane's bytes go to sink next + private rrChunkIndex = 0; // for round-robin sink reordering + private rrPending = new Map(); // chunkIndex → plaintext + private rrLaneCount = 0; + private partitionKind: 'range' | 'round-robin'; + + private resolveDone!: (result: TransferResult) => void; + private rejectDone!: (err: unknown) => void; + private donePromise: Promise; + private aborted = false; + private completedAt = -1; + private events: TransferEvent[] = []; + private eventResolvers: Array<(value: IteratorResult) => void> = []; + + private overallHasher = new StreamingSha256(); + private bytesProcessed = 0; + private progress: ProgressTracker; + private laneProgress: Map; + + readonly handle: TransferHandle; + + constructor( + public readonly streamId: string, + public readonly peerAddress: string, + public readonly receiver: MultiLaneReceiver, + public readonly metadata: StreamMetadata, + public readonly lanes: LaneInitSpec[], + ) { + this.partitionKind = lanes[0]!.partition.kind; + this.rrLaneCount = lanes.length; + for (const lane of lanes) this.rangeOrderCursor.set(lane.laneId, 0); + this.donePromise = new Promise((resolve, reject) => { + this.resolveDone = resolve; + this.rejectDone = reject; + }); + this.progress = new ProgressTracker(metadata.sizeBytes); + this.laneProgress = new Map( + lanes.map((l) => [ + l.laneId, + { + laneId: l.laneId, + bytesSent: 0, + ...(l.partition.kind === 'range' + ? { bytesTotal: l.partition.endByte - l.partition.startByte } + : {}), + seq: 0, + state: 'idle' as const, + }, + ]), + ); + this.handle = { + streamId: this.streamId, + events: this.eventsAsyncIterable(), + pause: async () => { /* M-Stream-6 */ }, + resume: async () => { /* M-Stream-6 */ }, + abort: async (reason) => { + await this.abort(reason ?? 'manual'); + }, + done: () => this.donePromise, + }; + } + + async handleChunk(wireBytes: Uint8Array): Promise { + if (this.aborted) { + throw new TransferProtocolError('transfer aborted'); + } + if (!this.accepted) { + // Buffer until accepted? For v0.2.0 simplicity, reject. Consumer must + // accept synchronously in the onIncomingTransfer handler. + throw new TransferProtocolError('chunk arrived before accept()'); + } + const dec = await this.receiver.decryptChunk(wireBytes); + this.recordReceived(dec.laneId, dec.plaintext.length, dec.seq, dec.isLast); + if (this.partitionKind === 'range') { + await this.writeRangeChunk(dec.laneId, dec.seq, dec.plaintext); + } else { + // Round-robin: chunkIndex = lane.partition.lane + seq * laneCount + const lane = this.lanes.find((l) => l.laneId === dec.laneId); + if (lane === undefined || lane.partition.kind !== 'round-robin') { + throw new TransferProtocolError('lane misconfigured'); + } + const chunkIndex = lane.partition.lane + dec.seq * this.rrLaneCount; + await this.writeRoundRobinChunk(chunkIndex, dec.plaintext); + } + return { + lastSeq: dec.seq, + bytesReceived: this.bytesProcessed, + }; + } + + private async writeRangeChunk( + laneId: number, + seq: number, + plaintext: Uint8Array, + ): Promise { + // Per-lane in-order check is done by StreamReceiver. We just need to + // advance writes when this is the currently-flushing lane. + const expected = this.rangeOrderCursor.get(laneId) ?? 0; + if (seq !== expected) { + // Out-of-order within a lane is already rejected by StreamReceiver, + // so this branch is defensive only. + throw new TransferProtocolError(`unexpected seq ${seq} for lane ${laneId}`); + } + this.rangeOrderCursor.set(laneId, seq + 1); + + if (laneId !== this.currentLaneForSink) { + // Buffer for later. + let lb = this.bufferedByLane.get(laneId); + if (lb === undefined) { + lb = new Map(); + this.bufferedByLane.set(laneId, lb); + } + lb.set(seq, plaintext); + return; + } + + // We are the current lane: write directly. + await this.appendToSink(plaintext); + if (this.receiver.getLane(laneId)?.isFinished) { + // Move to next lane and drain its buffer. + this.currentLaneForSink++; + while (this.currentLaneForSink < this.lanes.length) { + const next = this.currentLaneForSink; + const buffered = this.bufferedByLane.get(next); + if (buffered !== undefined) { + // Drain in seq order. + let nextSeq = 0; + while (buffered.has(nextSeq)) { + await this.appendToSink(buffered.get(nextSeq)!); + buffered.delete(nextSeq); + nextSeq++; + } + this.bufferedByLane.delete(next); + } + if (!this.receiver.getLane(next)?.isFinished) break; + this.currentLaneForSink++; + } + } + } + + private async writeRoundRobinChunk(chunkIndex: number, plaintext: Uint8Array): Promise { + if (chunkIndex === this.rrChunkIndex) { + await this.appendToSink(plaintext); + this.rrChunkIndex++; + while (this.rrPending.has(this.rrChunkIndex)) { + await this.appendToSink(this.rrPending.get(this.rrChunkIndex)!); + this.rrPending.delete(this.rrChunkIndex); + this.rrChunkIndex++; + } + } else if (chunkIndex > this.rrChunkIndex) { + this.rrPending.set(chunkIndex, plaintext); + } else { + throw new TransferProtocolError(`backward chunkIndex ${chunkIndex}`); + } + } + + private async appendToSink(plaintext: Uint8Array): Promise { + if (this.sink === null) throw new TransferProtocolError('no sink'); + this.overallHasher.update(plaintext); + this.receiver.appendOverall(plaintext); + await this.sink.write(plaintext); + } + + recordReceived(laneId: number, bytes: number, seq: number, isLast: boolean): void { + const lane = this.laneProgress.get(laneId); + if (lane !== undefined) { + lane.bytesSent += bytes; + lane.seq = seq + 1; + lane.state = isLast ? 'done' : 'sending'; + if (isLast) this.emit({ type: 'lane-done', laneId }); + } + this.bytesProcessed += bytes; + this.progress.add(bytes); + const sample = this.progress.sample(); + const tp: TransferProgress = { + streamId: this.streamId, + bytesSent: this.bytesProcessed, + bytesPerSecond: sample.bytesPerSecond, + lanes: [...this.laneProgress.values()], + ...(this.metadata.sizeBytes !== undefined ? { bytesTotal: this.metadata.sizeBytes } : {}), + ...(sample.etaSeconds !== undefined ? { etaSeconds: sample.etaSeconds } : {}), + ...(sample.percent !== undefined ? { percent: sample.percent } : {}), + }; + this.onProgress?.(tp); + this.emit({ type: 'progress', progress: tp }); + } + + async finalize(msg: StreamFinishMessage): Promise { + if (!this.accepted) return; + if (!this.receiver.allLanesFinished) { + const err = new TransferIntegrityError('finish before all lanes finished'); + await this.failWith(err); + return; + } + // Verify lane fingerprints. + const localFps = this.receiver.getLaneFingerprints(); + for (const expected of msg.laneSha256) { + const local = localFps.find((f) => f.laneId === expected.laneId); + if (local === undefined) { + return this.failWith( + new TransferIntegrityError(`finish references unknown laneId ${expected.laneId}`), + ); + } + if (bytesToBase64(local.sha256) !== expected.sha256) { + return this.failWith( + new TransferIntegrityError(`laneSha256 mismatch on lane ${expected.laneId}`), + ); + } + } + const localOverall = bytesToBase64(this.overallHasher.digest()); + if (localOverall !== msg.overallSha256) { + return this.failWith(new TransferIntegrityError('overallSha256 mismatch')); + } + if (this.sink !== null) await this.sink.finalize(); + const result: TransferResult = { + streamId: this.streamId, + bytesSent: this.bytesProcessed, + sha256: hexFromBase64(msg.overallSha256), + durationMs: nowMs() - this.startedAt, + }; + if (this.outputKind === 'buffer' && this.sink !== null) { + const bytes = this.sink.toBytes(); + if (bytes !== null) (result as TransferResult & { bytes?: Uint8Array }).bytes = bytes; + } + this.completedAt = nowMs(); + this.emit({ type: 'complete', streamId: result.streamId, sha256: result.sha256, durationMs: result.durationMs }); + this.flushEventEnd(); + this.resolveDone(result); + this.receiver.destroy(); + } + + async abort(reason: string): Promise { + if (this.aborted || this.completedAt > 0) return; + this.aborted = true; + if (this.sink !== null) await this.sink.abort(reason); + this.emit({ type: 'abort', reason }); + this.flushEventEnd(); + this.receiver.destroy(); + this.rejectDone(new TransferAbortError(reason)); + } + + lanesProgressForResume(): Array<{ laneId: number; lastSeqAcked: number }> { + const out: Array<{ laneId: number; lastSeqAcked: number }> = []; + for (const lane of this.lanes) { + const r = this.receiver.getLane(lane.laneId); + const next = r ? Number(r.nextExpectedSequence) : 0; + out.push({ laneId: lane.laneId, lastSeqAcked: next - 1 }); + } + return out; + } + + private async failWith(err: unknown): Promise { + if (this.aborted || this.completedAt > 0) return; + this.aborted = true; + this.emit({ type: 'error', error: err }); + this.flushEventEnd(); + if (this.sink !== null) await this.sink.abort('integrity-failure'); + this.receiver.destroy(); + this.rejectDone(err); + } + + emit(event: TransferEvent): void { + this.onEvent?.(event); + if (this.eventResolvers.length > 0) { + this.eventResolvers.shift()!({ value: event, done: false }); + } else { + this.events.push(event); + } + } + + private flushEventEnd(): void { + for (const r of this.eventResolvers) r({ value: undefined as never, done: true }); + this.eventResolvers = []; + } + + private eventsAsyncIterable(): AsyncIterable { + const self = this; + return { + [Symbol.asyncIterator](): AsyncIterator { + return { + next(): Promise> { + if (self.events.length > 0) { + return Promise.resolve({ value: self.events.shift()!, done: false }); + } + if (self.aborted || self.completedAt > 0) { + return Promise.resolve({ value: undefined as never, done: true }); + } + return new Promise((resolve) => self.eventResolvers.push(resolve)); + }, + }; + }, + }; + } +} + +// ─── Helpers ───────────────────────────────────────────────── + +function computeEffectiveLaneCount( + requested: number, + inputSize: number | undefined, + chunkSize: number, +): number { + if (!Number.isInteger(requested) || requested < 1) { + throw new ValidationError(`lanes must be a positive integer`, 'lanes'); + } + const cap = Math.min(64, requested); + if (inputSize !== undefined && inputSize < 2 * chunkSize) return 1; + return cap; +} + +function resolvePartition( + mode: TransferOptions['partition'], + input: NormalizedInput, +): 'range' | 'round-robin' { + if (mode === 'range') return 'range'; + if (mode === 'round-robin') return 'round-robin'; + // 'auto' default + return input.randomAccess && input.size !== undefined ? 'range' : 'round-robin'; +} + +function bytesToBase64(bytes: Uint8Array): string { + let bin = ''; + for (let i = 0; i < bytes.length; i++) bin += String.fromCharCode(bytes[i]!); + return btoa(bin); +} + +function bytesToBase64Url(bytes: Uint8Array): string { + return bytesToBase64(bytes).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); +} + +function base64UrlToBytes(s: string): Uint8Array { + const padded = s.replace(/-/g, '+').replace(/_/g, '/'); + const pad = padded.length % 4 === 0 ? '' : '='.repeat(4 - (padded.length % 4)); + const bin = atob(padded + pad); + const out = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); + return out; +} + +function bytesFromBase64Url(s: string): Uint8Array { + return base64UrlToBytes(s); +} + +function hex(bytes: Uint8Array): string { + return Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join(''); +} + +function hexFromBase64(b64: string): string { + const bin = atob(b64); + let out = ''; + for (let i = 0; i < bin.length; i++) { + out += bin.charCodeAt(i).toString(16).padStart(2, '0'); + } + return out; +} + +function nowMs(): number { + return typeof performance !== 'undefined' ? performance.now() : Date.now(); +} + +// ─── Persister ─────────────────────────────────────────────── + +class Persister { + /** Per-stream lock so saves never interleave. */ + private chains = new Map>(); + /** Random per-process key when no deviceKey is supplied (in-memory only). */ + private fallbackKey: Uint8Array | null = null; + + constructor( + private readonly crypto: CryptoProvider, + private readonly store: ResumeStore, + private readonly deviceKey: Uint8Array | undefined, + ) {} + + private getKey(): Uint8Array { + if (this.deviceKey !== undefined) return this.deviceKey; + if (this.fallbackKey === null) this.fallbackKey = this.crypto.randomBytes(32); + return this.fallbackKey; + } + + private serialize(streamId: string, work: () => Promise): Promise { + const previous = this.chains.get(streamId) ?? Promise.resolve(); + const next = previous.catch(() => {}).then(work); + this.chains.set(streamId, next); + return next; + } + + async saveOutgoing(state: OutgoingState, streamSecret: Uint8Array): Promise { + return this.serialize(state.streamId, async () => { + const wrapped = await wrapStreamSecret(this.crypto, this.getKey(), streamSecret, state.streamId); + const persisted: PersistedStreamState = { + streamId: state.streamId, + direction: 'send', + peerAddress: state.peerAddress, + status: 'active', + metadataJson: JSON.stringify(state.metadata), + partitionJson: JSON.stringify(state.lanes), + laneStateJson: JSON.stringify(snapshotOutgoingLanes(state)), + ioDescriptorJson: JSON.stringify({ kind: 'consumer-supplied' }), + secretEnc: wrapped.ciphertext, + secretNonce: wrapped.nonce, + createdAt: Date.now(), + updatedAt: Date.now(), + }; + await this.store.save(persisted); + }); + } + + async saveIncoming(state: IncomingState, streamSecret: Uint8Array): Promise { + return this.serialize(state.streamId, async () => { + const wrapped = await wrapStreamSecret(this.crypto, this.getKey(), streamSecret, state.streamId); + const persisted: PersistedStreamState = { + streamId: state.streamId, + direction: 'receive', + peerAddress: state.peerAddress, + status: 'active', + metadataJson: JSON.stringify(state.metadata), + partitionJson: JSON.stringify(state.lanes), + laneStateJson: JSON.stringify(snapshotIncomingLanes(state)), + ioDescriptorJson: JSON.stringify({ kind: 'consumer-supplied' }), + secretEnc: wrapped.ciphertext, + secretNonce: wrapped.nonce, + createdAt: Date.now(), + updatedAt: Date.now(), + }; + await this.store.save(persisted); + }); + } + + async markStatus(streamId: string, status: 'finished' | 'aborted'): Promise { + return this.serialize(streamId, async () => { + const existing = await this.store.get(streamId); + if (existing === null) return; + existing.status = status; + existing.updatedAt = Date.now(); + await this.store.save(existing); + }); + } + + async load(streamId: string): Promise<{ + record: PersistedStreamState; + streamSecret: Uint8Array; + } | null> { + const record = await this.store.get(streamId); + if (record === null) return null; + const streamSecret = await unwrapStreamSecret( + this.crypto, + this.getKey(), + record.secretEnc, + record.secretNonce, + record.streamId, + ); + return { record, streamSecret }; + } +} + +function snapshotOutgoingLanes( + state: OutgoingState, +): Array<{ laneId: number; nextSeq: number; bytesSent: number }> { + const out: Array<{ laneId: number; nextSeq: number; bytesSent: number }> = []; + for (const lane of state.lanes) { + const lp = state['laneProgress'].get(lane.laneId); + out.push({ + laneId: lane.laneId, + nextSeq: lp?.seq ?? 0, + bytesSent: lp?.bytesSent ?? 0, + }); + } + return out; +} + +function snapshotIncomingLanes( + state: IncomingState, +): Array<{ laneId: number; expectedSeq: number; bytesReceived: number }> { + const out: Array<{ laneId: number; expectedSeq: number; bytesReceived: number }> = []; + for (const lane of state.lanes) { + const r = state.receiver.getLane(lane.laneId); + out.push({ + laneId: lane.laneId, + expectedSeq: r ? Number(r.nextExpectedSequence) : 0, + bytesReceived: r ? Number(r.bytesReceived) : 0, + }); + } + return out; +} diff --git a/packages/shade-transfer/src/errors.ts b/packages/shade-transfer/src/errors.ts new file mode 100644 index 0000000..172a882 --- /dev/null +++ b/packages/shade-transfer/src/errors.ts @@ -0,0 +1,53 @@ +import { ShadeError } from '@shade/core'; + +export class TransferError extends ShadeError { + constructor(code: string, message: string) { + super(code, message); + this.name = 'TransferError'; + } +} + +export class TransferAbortError extends TransferError { + constructor(message = 'Transfer aborted') { + super('SHADE_TRANSFER_ABORT', message); + this.name = 'TransferAbortError'; + } +} + +export class TransferIntegrityError extends TransferError { + constructor(message: string) { + super('SHADE_TRANSFER_INTEGRITY', message); + this.name = 'TransferIntegrityError'; + } +} + +export class TransferProtocolError extends TransferError { + constructor(message: string) { + super('SHADE_TRANSFER_PROTOCOL', message); + this.name = 'TransferProtocolError'; + } +} + +export class TransferOfflineError extends TransferError { + constructor(peer: string, message?: string) { + super( + 'SHADE_TRANSFER_OFFLINE', + message ?? `Peer ${peer} is offline. Queue/relay support deferred to a later version.`, + ); + this.name = 'TransferOfflineError'; + } +} + +export class TransferResumeError extends TransferError { + constructor(message: string) { + super('SHADE_TRANSFER_RESUME', message); + this.name = 'TransferResumeError'; + } +} + +export class TransferTransportError extends TransferError { + constructor(message: string, public readonly statusCode?: number) { + super('SHADE_TRANSFER_TRANSPORT', message); + this.name = 'TransferTransportError'; + } +} diff --git a/packages/shade-transfer/src/index.ts b/packages/shade-transfer/src/index.ts new file mode 100644 index 0000000..8f6cc57 --- /dev/null +++ b/packages/shade-transfer/src/index.ts @@ -0,0 +1,29 @@ +export * from './errors.js'; +export * from './types.js'; +export * from './retry.js'; +export * from './progress.js'; +export * from './transport/transport.js'; +export * from './transport/memory.js'; +export * from './transport/http-transport.js'; +export * from './transport/ws-transport.js'; +export * from './engine.js'; +export { + createTransferRoutes, + PermissiveAuthenticator, +} from './receiver/server-handler.js'; +export type { + TransferRouteOptions, + TransferRouteAuthenticator, +} from './receiver/server-handler.js'; +export { resolveOutputSink } from './receiver/output.js'; +export type { OutputSink } from './receiver/output.js'; +export { normalizeInput } from './sender/input.js'; +export type { NormalizedInput } from './sender/input.js'; +export { + MemoryResumeStore, + StorageBackedResumeStore, + deriveDeviceKey, + wrapStreamSecret, + unwrapStreamSecret, +} from './persistence/resume.js'; +export type { ResumeStore } from './persistence/resume.js'; diff --git a/packages/shade-transfer/src/persistence/resume.ts b/packages/shade-transfer/src/persistence/resume.ts new file mode 100644 index 0000000..30a752a --- /dev/null +++ b/packages/shade-transfer/src/persistence/resume.ts @@ -0,0 +1,136 @@ +/** + * Resume-store contract for `@shade/transfer`. + * + * The engine persists in-flight transfer state through this interface so a + * crashed sender or receiver can pick up where it left off. v0.2.0 ships + * three implementations: + * + * - `MemoryResumeStore` — never persists; suitable for ephemeral + * transfers where resume is not needed. + * - `StorageBackedResumeStore` — wraps a Shade `StorageProvider`, with + * `deviceKey`-based AES-GCM encryption of the streamSecret at rest. + * - `IndexedDbResumeStore` — browser-side; built on top of the same + * `PersistedStreamState` shape. + */ + +import type { CryptoProvider, PersistedStreamState, StorageProvider } from '@shade/core'; + +export interface ResumeStore { + /** Persist or update a stream's resume record. Idempotent on streamId. */ + save(state: PersistedStreamState): Promise; + /** Look up a stream's record by streamId. Returns null when absent. */ + get(streamId: string): Promise; + /** Remove a stream's record (e.g. on completion). */ + remove(streamId: string): Promise; + /** List all non-final records (status `'active' | 'paused'`). */ + listActive(direction?: 'send' | 'receive'): Promise; +} + +/** In-memory store; useful for tests and ephemeral transfers. */ +export class MemoryResumeStore implements ResumeStore { + private map = new Map(); + + async save(state: PersistedStreamState): Promise { + this.map.set(state.streamId, { ...state }); + } + async get(streamId: string): Promise { + const s = this.map.get(streamId); + return s ? { ...s } : null; + } + async remove(streamId: string): Promise { + this.map.delete(streamId); + } + async listActive(direction?: 'send' | 'receive'): Promise { + const out: PersistedStreamState[] = []; + for (const s of this.map.values()) { + if (s.status !== 'active' && s.status !== 'paused') continue; + if (direction !== undefined && s.direction !== direction) continue; + out.push({ ...s }); + } + out.sort((a, b) => b.updatedAt - a.updatedAt); + return out; + } +} + +/** + * Storage-backed resume store. Delegates persistence to a `StorageProvider` + * that implements the optional stream-state methods. Throws clearly if the + * provider does not. + */ +export class StorageBackedResumeStore implements ResumeStore { + constructor(private readonly storage: StorageProvider) { + if ( + storage.saveStreamState === undefined || + storage.getStreamState === undefined || + storage.removeStreamState === undefined || + storage.listActiveStreamStates === undefined + ) { + throw new Error( + 'StorageBackedResumeStore requires a StorageProvider with stream-state methods', + ); + } + } + async save(state: PersistedStreamState): Promise { + await this.storage.saveStreamState!(state); + } + async get(streamId: string): Promise { + return this.storage.getStreamState!(streamId); + } + async remove(streamId: string): Promise { + await this.storage.removeStreamState!(streamId); + } + async listActive(direction?: 'send' | 'receive'): Promise { + return this.storage.listActiveStreamStates!(direction); + } +} + +/** + * Derive a device-specific encryption key from the local identity signing + * key. Used to wrap the streamSecret before persisting it. + * + * deviceKey = HKDF( + * ikm = signingPrivateKey, + * salt = "shade-stream-resume/v1/device", + * info = "device-encryption-key", + * length = 32, + * ) + * + * Identity rotation invalidates all existing device-keyed records (resume + * stops working until a fresh transfer is initiated). This is intentional: + * a stolen pre-rotation DB cannot decrypt a post-rotation in-flight stream. + */ +export async function deriveDeviceKey( + crypto: CryptoProvider, + signingPrivateKey: Uint8Array, +): Promise { + const enc = new TextEncoder(); + return crypto.hkdf( + signingPrivateKey, + enc.encode('shade-stream-resume/v1/device'), + enc.encode('device-encryption-key'), + 32, + ); +} + +/** Encrypt a streamSecret under a deviceKey. Returns ciphertext + nonce. */ +export async function wrapStreamSecret( + crypto: CryptoProvider, + deviceKey: Uint8Array, + streamSecret: Uint8Array, + streamId: string, +): Promise<{ ciphertext: Uint8Array; nonce: Uint8Array }> { + const aad = new TextEncoder().encode(`resume:${streamId}`); + return crypto.aesGcmEncrypt(deviceKey, streamSecret, aad); +} + +/** Decrypt a streamSecret under a deviceKey. */ +export async function unwrapStreamSecret( + crypto: CryptoProvider, + deviceKey: Uint8Array, + ciphertext: Uint8Array, + nonce: Uint8Array, + streamId: string, +): Promise { + const aad = new TextEncoder().encode(`resume:${streamId}`); + return crypto.aesGcmDecrypt(deviceKey, ciphertext, nonce, aad); +} diff --git a/packages/shade-transfer/src/progress.ts b/packages/shade-transfer/src/progress.ts new file mode 100644 index 0000000..2a9f456 --- /dev/null +++ b/packages/shade-transfer/src/progress.ts @@ -0,0 +1,93 @@ +/** + * Throughput tracking for an in-flight transfer. + * + * Smooths byte-rate via exponential moving average over the last `windowMs` + * window, recomputed on every progress sample. Returns `bytesPerSecond` and + * `etaSeconds` when total size is known. + */ +export interface ProgressSample { + bytesPerSecond: number; + etaSeconds: number | undefined; + percent: number | undefined; +} + +export class ProgressTracker { + private bytesProcessed = 0; + private startedAt = -1; + private lastSampleAt = -1; + private lastBytes = 0; + /** EMA of byte rate (bytes per second). */ + private rateEma = 0; + /** Smoothing factor in [0, 1]. Higher = more responsive, less smooth. */ + private readonly alpha: number; + + constructor( + private readonly bytesTotal: number | undefined, + /** Window over which EMA decay equals 1 - 1/e (~63 %). */ + windowMs = 5000, + /** Sample period — sampling more often than this returns the cached EMA. */ + private readonly sampleEveryMs = 250, + ) { + // Convert window to per-sample alpha: alpha = 1 - exp(-dt/window) + // Approximate over `sampleEveryMs` per tick. + this.alpha = 1 - Math.exp(-this.sampleEveryMs / windowMs); + } + + /** Record bytes processed since last call. */ + add(bytes: number): void { + if (this.startedAt < 0) { + this.startedAt = nowMs(); + this.lastSampleAt = this.startedAt; + } + this.bytesProcessed += bytes; + } + + /** Total bytes accounted-for so far. */ + get totalProcessed(): number { + return this.bytesProcessed; + } + + /** Total elapsed wall-clock since first byte. */ + get elapsedMs(): number { + if (this.startedAt < 0) return 0; + return nowMs() - this.startedAt; + } + + /** Compute a smoothed sample. Cheap to call repeatedly. */ + sample(): ProgressSample { + const now = nowMs(); + if (this.startedAt < 0) { + return { bytesPerSecond: 0, etaSeconds: undefined, percent: this.percent() }; + } + const dt = now - this.lastSampleAt; + if (dt >= this.sampleEveryMs) { + const deltaBytes = this.bytesProcessed - this.lastBytes; + const instantRate = dt > 0 ? (deltaBytes * 1000) / dt : 0; + this.rateEma = + this.rateEma === 0 ? instantRate : this.alpha * instantRate + (1 - this.alpha) * this.rateEma; + this.lastSampleAt = now; + this.lastBytes = this.bytesProcessed; + } + + const bps = Math.max(0, this.rateEma); + let etaSeconds: number | undefined; + if (this.bytesTotal !== undefined && bps > 0) { + const remaining = Math.max(0, this.bytesTotal - this.bytesProcessed); + etaSeconds = remaining / bps; + } else if (this.bytesTotal !== undefined && this.bytesProcessed >= this.bytesTotal) { + etaSeconds = 0; + } + + return { bytesPerSecond: bps, etaSeconds, percent: this.percent() }; + } + + private percent(): number | undefined { + if (this.bytesTotal === undefined) return undefined; + if (this.bytesTotal === 0) return 1; + return Math.min(1, this.bytesProcessed / this.bytesTotal); + } +} + +function nowMs(): number { + return typeof performance !== 'undefined' ? performance.now() : Date.now(); +} diff --git a/packages/shade-transfer/src/receiver/output.ts b/packages/shade-transfer/src/receiver/output.ts new file mode 100644 index 0000000..5d6598a --- /dev/null +++ b/packages/shade-transfer/src/receiver/output.ts @@ -0,0 +1,166 @@ +import type { TransferOutput } from '../types.js'; + +/** + * Generic chunk sink. The orchestrator writes plaintext chunks IN ORIGINAL + * BYTE ORDER, then calls `finalize()`. `toBytes()` is only valid for + * `'buffer'` sinks. + */ +export interface OutputSink { + write(chunk: Uint8Array): Promise; + finalize(): Promise; + abort(reason?: string): Promise; + toBytes(): Uint8Array | null; +} + +/** + * Adapter from a `TransferOutput` (consumer-supplied) to an `OutputSink`. + * + * The `'file'` sink is Bun-specific (uses `Bun.file(path).writer()`) — for + * Node use a `'pipe'` sink with an `fs.createWriteStream`. + */ +export async function resolveOutputSink(out: TransferOutput): Promise { + switch (out.kind) { + case 'pipe': + return createPipeSink(out.pipeTo); + case 'callback': + return createCallbackSink(out.onChunk); + case 'buffer': + return createBufferSink(); + case 'file': + return createBunFileSink(out.path); + case 'fileHandle': + return createFileHandleSink(out.handle); + } +} + +function createPipeSink(stream: WritableStream): OutputSink { + const writer = stream.getWriter(); + return { + async write(chunk) { + await writer.write(chunk); + }, + async finalize() { + try { + await writer.close(); + } finally { + writer.releaseLock?.(); + } + }, + async abort(reason) { + try { + await writer.abort(reason); + } catch { + /* swallow */ + } finally { + writer.releaseLock?.(); + } + }, + toBytes() { + return null; + }, + }; +} + +function createCallbackSink( + onChunk: (chunk: Uint8Array) => void | Promise, +): OutputSink { + return { + async write(chunk) { + await onChunk(chunk); + }, + async finalize() { + /* no-op */ + }, + async abort() { + /* no-op */ + }, + toBytes() { + return null; + }, + }; +} + +function createBufferSink(): OutputSink { + const parts: Uint8Array[] = []; + let totalLen = 0; + return { + async write(chunk) { + parts.push(chunk); + totalLen += chunk.length; + }, + async finalize() { + /* no-op */ + }, + async abort() { + parts.length = 0; + totalLen = 0; + }, + toBytes() { + const out = new Uint8Array(totalLen); + let off = 0; + for (const p of parts) { + out.set(p, off); + off += p.length; + } + return out; + }, + }; +} + +async function createBunFileSink(path: string): Promise { + // Lazy import so non-Bun runtimes don't fail at import time. + const bunGlobal = (globalThis as unknown as { Bun?: { file: (p: string) => { writer: () => unknown } } }) + .Bun; + if (bunGlobal === undefined) { + throw new Error("'file' output sink requires Bun runtime"); + } + const writer = bunGlobal.file(path).writer() as { + write: (chunk: Uint8Array) => number | Promise; + end: () => Promise | number; + flush: () => Promise | number; + }; + return { + async write(chunk) { + await Promise.resolve(writer.write(chunk)); + }, + async finalize() { + await Promise.resolve(writer.flush()); + await Promise.resolve(writer.end()); + }, + async abort() { + try { + await Promise.resolve(writer.end()); + } catch { + /* swallow */ + } + }, + toBytes() { + return null; + }, + }; +} + +async function createFileHandleSink(handle: { + createWritable(): Promise; +}): Promise { + const writable = (await handle.createWritable()) as { + write: (chunk: Uint8Array) => Promise; + close: () => Promise; + abort?: (reason?: unknown) => Promise; + }; + return { + async write(chunk) { + await writable.write(chunk); + }, + async finalize() { + await writable.close(); + }, + async abort(reason) { + if (writable.abort) await writable.abort(reason); + else await writable.close(); + }, + toBytes() { + return null; + }, + }; +} diff --git a/packages/shade-transfer/src/receiver/server-handler.ts b/packages/shade-transfer/src/receiver/server-handler.ts new file mode 100644 index 0000000..eac6a43 --- /dev/null +++ b/packages/shade-transfer/src/receiver/server-handler.ts @@ -0,0 +1,162 @@ +import type { Hono, Context } from 'hono'; +import type { TransferEngine } from '../engine.js'; +import { TransferProtocolError } from '../errors.js'; + +/** + * Auth contract for incoming chunk POSTs / control GETs. Verifies the + * signature attached by the client's `TransferAuthenticator` and returns + * the resolved sender address. + */ +export interface TransferRouteAuthenticator { + verifyChunk(args: { + request: Request; + streamId: string; + laneId: number; + seq: bigint; + bodyHash: Uint8Array; + }): Promise<{ senderAddress: string }>; + + verifyControl(args: { + request: Request; + streamId: string; + method: string; + path: string; + }): Promise<{ senderAddress: string }>; +} + +export interface TransferRouteOptions { + /** Maximum chunk body size in bytes. Default 16 MiB. */ + maxChunkBytes?: number; + /** Server-side authenticator. Defaults to a permissive one for trusted/local nets. */ + authenticator?: TransferRouteAuthenticator; +} + +/** + * Permissive authenticator: extracts sender address from the + * `X-Shade-Sender-Address` header without verification. Suitable ONLY for + * trusted/local network testing; the SDK's M-Stream-5 integration replaces + * this with an Ed25519-verifying implementation. + */ +export const PermissiveAuthenticator: TransferRouteAuthenticator = { + async verifyChunk({ request }) { + const senderAddress = request.headers.get('X-Shade-Sender-Address'); + if (senderAddress === null || senderAddress === '') { + throw new TransferProtocolError('Missing X-Shade-Sender-Address header'); + } + return { senderAddress }; + }, + async verifyControl({ request }) { + const senderAddress = request.headers.get('X-Shade-Sender-Address'); + if (senderAddress === null || senderAddress === '') { + throw new TransferProtocolError('Missing X-Shade-Sender-Address header'); + } + return { senderAddress }; + }, +}; + +const DEFAULT_MAX_CHUNK_BYTES = 16 * 1024 * 1024 + 1024; + +/** + * Mount the transfer-receive routes on a Hono router. Returns the same + * Hono instance for fluent composition. The consumer mounts under any + * base path (e.g. `app.route('/shade', createTransferRoutes(engine))`). + * + * Hono is a peer-dep so non-server consumers can omit it. + */ +export async function createTransferRoutes( + engine: TransferEngine, + options: TransferRouteOptions = {}, +): Promise { + const { Hono: HonoCtor } = (await import('hono')) as { Hono: new () => Hono }; + const app = new HonoCtor(); + const auth = options.authenticator ?? PermissiveAuthenticator; + const maxBytes = options.maxChunkBytes ?? DEFAULT_MAX_CHUNK_BYTES; + + app.get('/v1/transfer/health', (c) => c.json({ ok: true })); + + app.post('/v1/transfer/:streamId/chunk', async (c) => { + const streamId = c.req.param('streamId'); + const laneIdRaw = c.req.header('X-Shade-Lane-Id'); + const seqRaw = c.req.header('X-Shade-Seq'); + if (laneIdRaw === undefined || seqRaw === undefined) { + return c.json({ error: 'missing X-Shade-Lane-Id or X-Shade-Seq' }, 400); + } + const laneId = Number(laneIdRaw); + const seq = BigInt(seqRaw); + if (!Number.isInteger(laneId) || laneId < 0) { + return c.json({ error: 'invalid lane id' }, 400); + } + + const contentLength = c.req.header('content-length'); + if (contentLength !== undefined && Number(contentLength) > maxBytes) { + return c.json({ error: `chunk exceeds maxChunkBytes (${maxBytes})` }, 413); + } + const ab = await c.req.arrayBuffer(); + if (ab.byteLength > maxBytes) { + return c.json({ error: `chunk exceeds maxChunkBytes (${maxBytes})` }, 413); + } + const body = new Uint8Array(ab); + const bodyHash = new Uint8Array( + await globalThis.crypto.subtle.digest('SHA-256', ab), + ); + + let senderAddress: string; + try { + const result = await auth.verifyChunk({ + request: c.req.raw, + streamId, + laneId, + seq, + bodyHash, + }); + senderAddress = result.senderAddress; + } catch (err) { + return errorResponse(c, err); + } + + try { + const ack = await engine.receiveChunk(senderAddress, streamId, laneId, seq, body); + return c.json(ack); + } catch (err) { + return errorResponse(c, err); + } + }); + + app.get('/v1/transfer/:streamId/state', async (c) => { + const streamId = c.req.param('streamId'); + let senderAddress: string; + try { + const result = await auth.verifyControl({ + request: c.req.raw, + streamId, + method: 'GET', + path: c.req.path, + }); + senderAddress = result.senderAddress; + } catch (err) { + return errorResponse(c, err); + } + + const state = await engine.getResumeState(senderAddress, streamId); + if (state === null) return c.json({ error: 'no state' }, 404); + return c.json(state); + }); + + return app; +} + +function errorResponse(c: Context, err: unknown): Response { + const message = err instanceof Error ? err.message : String(err); + const code = + err instanceof Error && (err as unknown as { code?: unknown }).code !== undefined + ? String((err as unknown as { code: unknown }).code) + : 'UNKNOWN'; + let status = 500; + if (code === 'SHADE_TRANSFER_PROTOCOL') status = 400; + if (code === 'SHADE_VALIDATION') status = 400; + if (code === 'SHADE_UNAUTHORIZED') status = 401; + if (code === 'SHADE_INVALID_SIGNATURE') status = 401; + if (code === 'SHADE_STREAM_REPLAY') status = 409; + if (code === 'SHADE_STREAM_OUT_OF_ORDER') status = 409; + return c.json({ error: message, code }, status as 400 | 401 | 409 | 500); +} diff --git a/packages/shade-transfer/src/retry.ts b/packages/shade-transfer/src/retry.ts new file mode 100644 index 0000000..31b0d08 --- /dev/null +++ b/packages/shade-transfer/src/retry.ts @@ -0,0 +1,105 @@ +import { TransferAbortError, TransferTransportError } from './errors.js'; + +export interface RetryPolicy { + maxAttempts: number; + /** Base delay in ms; doubled per attempt with jitter. */ + baseDelayMs: number; + /** Hard cap on a single delay step. */ + maxDelayMs: number; + /** Jitter factor in [0, 1); 0 = deterministic, 0.5 = ±25 % spread. */ + jitter: number; +} + +export const DEFAULT_RETRY: RetryPolicy = { + maxAttempts: 5, + baseDelayMs: 250, + maxDelayMs: 30_000, + jitter: 0.25, +}; + +export interface RetryContext { + attempt: number; + lastError: unknown; + willRetryInMs: number; +} + +/** + * Run an idempotent operation with exponential backoff. Aborts immediately + * via the supplied signal. `onAttempt` runs BEFORE each retry sleep. + * + * The operation is treated as retryable unless it throws an + * `AbortError`/`TransferAbortError` or a non-network error wrapped in + * `TransferTransportError` with a 4xx status code (those are + * deterministic and retrying won't help). + */ +export async function withRetry( + op: (attempt: number) => Promise, + options?: { + policy?: RetryPolicy; + signal?: AbortSignal; + onAttempt?: (ctx: RetryContext) => void; + isRetryable?: (err: unknown) => boolean; + }, +): Promise { + const policy = options?.policy ?? DEFAULT_RETRY; + const signal = options?.signal; + const onAttempt = options?.onAttempt; + const isRetryable = options?.isRetryable ?? defaultIsRetryable; + + let lastError: unknown; + for (let attempt = 1; attempt <= policy.maxAttempts; attempt++) { + if (signal?.aborted) throw new TransferAbortError('Aborted before attempt'); + try { + return await op(attempt); + } catch (err) { + lastError = err; + if (signal?.aborted) throw new TransferAbortError('Aborted during attempt'); + if (!isRetryable(err) || attempt === policy.maxAttempts) throw err; + const delay = computeDelay(attempt, policy); + onAttempt?.({ attempt, lastError: err, willRetryInMs: delay }); + await sleep(delay, signal); + } + } + throw lastError ?? new Error('withRetry: unreachable'); +} + +function defaultIsRetryable(err: unknown): boolean { + if (err instanceof TransferAbortError) return false; + if (err instanceof TransferTransportError) { + if (err.statusCode === undefined) return true; // network-level failure + if (err.statusCode >= 500) return true; + if (err.statusCode === 408 || err.statusCode === 429) return true; + return false; + } + // Network errors / DOMException['AbortError'] + if (typeof err === 'object' && err !== null) { + const name = (err as { name?: unknown }).name; + if (name === 'AbortError') return false; + if (name === 'TypeError') return true; // fetch network failure + } + return true; +} + +function computeDelay(attempt: number, policy: RetryPolicy): number { + const base = Math.min(policy.maxDelayMs, policy.baseDelayMs * 2 ** (attempt - 1)); + const half = base * policy.jitter * 0.5; + return Math.max(0, base - half + Math.random() * half * 2); +} + +function sleep(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(new TransferAbortError('Aborted')); + return; + } + const timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort); + resolve(); + }, ms); + function onAbort() { + clearTimeout(timer); + reject(new TransferAbortError('Aborted')); + } + signal?.addEventListener('abort', onAbort, { once: true }); + }); +} diff --git a/packages/shade-transfer/src/sender/input.ts b/packages/shade-transfer/src/sender/input.ts new file mode 100644 index 0000000..02579ec --- /dev/null +++ b/packages/shade-transfer/src/sender/input.ts @@ -0,0 +1,124 @@ +import { ValidationError } from '@shade/core'; +import type { TransferInput } from '../types.js'; + +export interface NormalizedInput { + /** Total plaintext bytes when known. `undefined` for indeterminate streams. */ + size: number | undefined; + name?: string; + contentType?: string; + /** Read bytes [start, end). End is exclusive. Must be supported when `size` is known. */ + read(start: number, end: number): Promise; + /** Sequential reader for unknown-size inputs. Returns `null` at end-of-stream. */ + readNext(maxBytes: number): Promise; + /** Whether the source supports random `read(start, end)` access. */ + readonly randomAccess: boolean; + /** Free any underlying resources (no-op when nothing to release). */ + close(): Promise; +} + +/** + * Normalize any of the supported `TransferInput` shapes into a uniform + * reader. Determines size eagerly when possible. + */ +export async function normalizeInput(input: TransferInput): Promise { + if (input instanceof Uint8Array) return fromUint8Array(input); + if (typeof Blob !== 'undefined' && input instanceof Blob) return fromBlob(input); + if (isReadableStream(input)) return fromReadableStream(input); + throw new ValidationError('Unsupported TransferInput', 'input'); +} + +function fromUint8Array(buf: Uint8Array): NormalizedInput { + return { + size: buf.length, + randomAccess: true, + async read(start, end) { + return buf.slice(start, end); + }, + async readNext() { + throw new Error('readNext is not supported on random-access inputs'); + }, + async close() { + /* nothing to release */ + }, + }; +} + +function fromBlob(blob: Blob): NormalizedInput { + // Blob.slice / Blob.arrayBuffer are not in `lib: ["ES2022"]`. We cast + // through `unknown` to avoid pulling in the full DOM lib. + const blobLike = blob as unknown as { + size: number; + slice(start: number, end: number): { arrayBuffer(): Promise }; + arrayBuffer(): Promise; + }; + const out: NormalizedInput = { + size: blobLike.size, + randomAccess: true, + async read(start, end) { + const slice = blobLike.slice(start, end); + const ab = await slice.arrayBuffer(); + return new Uint8Array(ab); + }, + async readNext() { + throw new Error('readNext is not supported on random-access inputs'); + }, + async close() { + /* nothing to release */ + }, + }; + // File extends Blob with name/type + const f = blob as Blob & { name?: string; type?: string }; + if (typeof f.name === 'string') out.name = f.name; + if (typeof f.type === 'string' && f.type.length > 0) out.contentType = f.type; + return out; +} + +function fromReadableStream(stream: ReadableStream): NormalizedInput { + const reader = stream.getReader(); + let pending: Uint8Array = new Uint8Array(0); + let done = false; + + return { + size: undefined, + randomAccess: false, + async read() { + throw new Error('Random access not supported for ReadableStream input'); + }, + async readNext(maxBytes) { + while (pending.length < maxBytes && !done) { + const r = await reader.read(); + if (r.done) { + done = true; + break; + } + pending = concat(pending, r.value as Uint8Array); + } + if (pending.length === 0 && done) return null; + const out = pending.subarray(0, Math.min(maxBytes, pending.length)); + pending = pending.subarray(out.length); + return out; + }, + async close() { + try { + await reader.cancel(); + } catch { + /* already closed */ + } + }, + }; +} + +function isReadableStream(x: unknown): x is ReadableStream { + return ( + typeof x === 'object' && + x !== null && + typeof (x as { getReader?: unknown }).getReader === 'function' + ); +} + +function concat(a: Uint8Array, b: Uint8Array): Uint8Array { + const out = new Uint8Array(a.length + b.length); + out.set(a, 0); + out.set(b, a.length); + return out as Uint8Array; +} diff --git a/packages/shade-transfer/src/sender/lane-queue.ts b/packages/shade-transfer/src/sender/lane-queue.ts new file mode 100644 index 0000000..7317883 --- /dev/null +++ b/packages/shade-transfer/src/sender/lane-queue.ts @@ -0,0 +1,88 @@ +/** + * Bounded async FIFO. The producer awaits when the queue is full; the + * consumer awaits when the queue is empty. Used by the upload orchestrator + * to pace per-lane chunk dispatch without unbounded memory growth. + */ +export class BoundedAsyncQueue { + private items: T[] = []; + private waiters: Array<(value: IteratorResult) => void> = []; + private spaceWaiters: Array<() => void> = []; + private closed = false; + private aborted = false; + private abortReason: unknown = null; + + constructor(private readonly capacity: number) { + if (capacity < 1) throw new Error(`capacity must be >= 1`); + } + + /** Push an item. Awaits if the queue is at capacity. */ + async push(item: T): Promise { + if (this.aborted) throw this.abortReason; + if (this.closed) throw new Error('queue closed'); + while (this.items.length >= this.capacity) { + await new Promise((resolve) => this.spaceWaiters.push(resolve)); + if (this.aborted) throw this.abortReason; + if (this.closed) throw new Error('queue closed mid-push'); + } + this.items.push(item); + const waiter = this.waiters.shift(); + if (waiter !== undefined) { + const next = this.items.shift()!; + waiter({ value: next, done: false }); + this.notifySpace(); + } + } + + /** Returns next item or `{done: true}` when closed and drained. */ + async next(): Promise> { + if (this.aborted) throw this.abortReason; + if (this.items.length > 0) { + const value = this.items.shift()!; + this.notifySpace(); + return { value, done: false }; + } + if (this.closed) return { value: undefined as never, done: true }; + return new Promise>((resolve) => { + this.waiters.push(resolve); + }); + } + + /** Mark the queue as closed; subsequent `next()` calls drain remaining items. */ + close(): void { + if (this.closed) return; + this.closed = true; + for (const waiter of this.waiters) { + waiter({ value: undefined as never, done: true }); + } + this.waiters = []; + } + + abort(reason: unknown): void { + if (this.aborted) return; + this.aborted = true; + this.abortReason = reason; + this.closed = true; + for (const w of this.spaceWaiters) w(); + this.spaceWaiters = []; + for (const w of this.waiters) { + // Reject pending consumers via a thrown abort; consumers handle. + // We can't reject here because the next() returns IteratorResult, not a rejected promise. + // So instead, re-resolve as `done`; consumers check `aborted` separately. + w({ value: undefined as never, done: true }); + } + this.waiters = []; + } + + get size(): number { + return this.items.length; + } + + get isClosed(): boolean { + return this.closed; + } + + private notifySpace(): void { + const waiter = this.spaceWaiters.shift(); + if (waiter !== undefined) waiter(); + } +} diff --git a/packages/shade-transfer/src/transport/http-transport.ts b/packages/shade-transfer/src/transport/http-transport.ts new file mode 100644 index 0000000..bc3594a --- /dev/null +++ b/packages/shade-transfer/src/transport/http-transport.ts @@ -0,0 +1,169 @@ +import { TransferAbortError, TransferTransportError } from '../errors.js'; +import { withRetry, type RetryPolicy } from '../retry.js'; +import type { + ChunkAck, + ChunkSendOptions, + ITransferTransport, + TransferResumeState, +} from './transport.js'; + +/** + * Resolves the base URL for a given peer address. + * + * In M-Stream-5 the SDK wires this to the prekey-server's directory or to a + * peer-supplied `transfer.baseUrl` field on the prekey bundle. For + * standalone use, the caller can pass a static map. + */ +export type PeerBaseUrlResolver = (peerAddress: string) => Promise; + +/** + * Authentication hook. Adds outgoing request headers and produces a + * "request signature" that the receiver verifies. The default is a no-op + * authenticator suitable for trusted/local networks; for production use the + * SDK provides an Ed25519-signing implementation in M-Stream-5. + */ +export interface TransferAuthenticator { + signChunk(args: { + streamId: string; + laneId: number; + seq: bigint; + bodyHash: Uint8Array; + }): Promise>; + signControl(args: { streamId: string; method: string; path: string }): Promise< + Record + >; +} + +export const NoopAuthenticator: TransferAuthenticator = { + async signChunk() { + return {}; + }, + async signControl() { + return {}; + }, +}; + +export interface ShadeTransferHttpTransportOptions { + /** Resolves the peer's HTTP base URL (e.g., `https://server.example.com/shade`). */ + resolveBaseUrl: PeerBaseUrlResolver; + /** Optional authenticator (defaults to no-op). */ + authenticator?: TransferAuthenticator; + /** Override `fetch` (e.g., for tests). Defaults to `globalThis.fetch`. */ + fetch?: typeof fetch; + /** Retry policy for chunk POSTs. */ + retryPolicy?: RetryPolicy; +} + +/** + * HTTP-based chunk transport. POSTs each `0x11` envelope to + * `/v1/transfer/:streamId/chunk` and parses the JSON ACK response. + * + * Mounting on the receiver side: `createTransferRoutes(engine, options)`. + */ +export class ShadeTransferHttpTransport implements ITransferTransport { + private readonly fetchFn: typeof fetch; + private readonly auth: TransferAuthenticator; + + constructor(private readonly options: ShadeTransferHttpTransportOptions) { + this.fetchFn = options.fetch ?? globalThis.fetch; + this.auth = options.authenticator ?? NoopAuthenticator; + } + + async probe(peerAddress: string): Promise { + const base = await this.options.resolveBaseUrl(peerAddress); + const url = `${stripTrailingSlash(base)}/v1/transfer/health`; + let res: Response; + try { + res = await this.fetchFn(url, { method: 'GET' }); + } catch (err) { + throw new TransferTransportError(`probe failed: ${(err as Error).message}`); + } + if (!res.ok) { + throw new TransferTransportError(`probe failed: HTTP ${res.status}`, res.status); + } + } + + async sendChunk( + peerAddress: string, + streamId: string, + laneId: number, + seq: number | bigint, + bytes: Uint8Array, + options?: ChunkSendOptions, + ): Promise { + const base = await this.options.resolveBaseUrl(peerAddress); + const url = `${stripTrailingSlash(base)}/v1/transfer/${encodeURIComponent(streamId)}/chunk`; + const seqBig = typeof seq === 'bigint' ? seq : BigInt(seq); + const bodyHash = await sha256(bytes); + const headers: Record = { + 'Content-Type': 'application/octet-stream', + 'X-Shade-Lane-Id': String(laneId), + 'X-Shade-Seq': seqBig.toString(), + }; + Object.assign( + headers, + await this.auth.signChunk({ streamId, laneId, seq: seqBig, bodyHash }), + ); + + const policy = this.options.retryPolicy; + return withRetry( + async () => { + let res: Response; + try { + res = await this.fetchFn(url, { + method: 'POST', + headers, + // BodyInit isn't in lib:["ES2022"]; cast through unknown. + body: bytes as unknown as never, + ...(options?.signal !== undefined ? { signal: options.signal } : {}), + }); + } catch (err) { + if ((err as { name?: string }).name === 'AbortError') { + throw new TransferAbortError('signal aborted'); + } + throw new TransferTransportError(`sendChunk failed: ${(err as Error).message}`); + } + if (!res.ok) { + const text = await res.text().catch(() => ''); + throw new TransferTransportError( + `sendChunk failed: HTTP ${res.status} ${text}`, + res.status, + ); + } + const body = (await res.json()) as { lastSeq: number; bytesReceived?: number }; + return { + lastSeq: body.lastSeq, + ...(body.bytesReceived !== undefined ? { bytesReceived: body.bytesReceived } : {}), + }; + }, + { + ...(policy !== undefined ? { policy } : {}), + ...(options?.signal !== undefined ? { signal: options.signal } : {}), + }, + ); + } + + async fetchResumeState( + peerAddress: string, + streamId: string, + ): Promise { + const base = await this.options.resolveBaseUrl(peerAddress); + const url = `${stripTrailingSlash(base)}/v1/transfer/${encodeURIComponent(streamId)}/state`; + const headers = await this.auth.signControl({ streamId, method: 'GET', path: url }); + const res = await this.fetchFn(url, { method: 'GET', headers }); + if (res.status === 404) return null; + if (!res.ok) { + throw new TransferTransportError(`fetchResumeState failed: HTTP ${res.status}`, res.status); + } + return (await res.json()) as TransferResumeState; + } +} + +function stripTrailingSlash(s: string): string { + return s.endsWith('/') ? s.slice(0, -1) : s; +} + +async function sha256(data: Uint8Array): Promise { + const buf = await globalThis.crypto.subtle.digest('SHA-256', data as unknown as ArrayBuffer); + return new Uint8Array(buf); +} diff --git a/packages/shade-transfer/src/transport/memory.ts b/packages/shade-transfer/src/transport/memory.ts new file mode 100644 index 0000000..8d64aac --- /dev/null +++ b/packages/shade-transfer/src/transport/memory.ts @@ -0,0 +1,163 @@ +import { TransferTransportError } from '../errors.js'; +import type { + ChunkAck, + ChunkSendOptions, + IControlChannel, + ITransferTransport, + TransferResumeState, +} from './transport.js'; +import type { StreamControlMessage } from '@shade/streams'; + +/** + * In-process control channel useful for tests and same-process upload/ + * download (e.g., a CLI that uploads to its own embedded server). + * + * Two endpoints are paired via {@link MemoryControlChannel.linked}: a + * `send()` on one delivers to the other's subscribers. + */ +export class MemoryControlChannel implements IControlChannel { + private peer: MemoryControlChannel | null = null; + private myAddress: string; + private handlers = new Set< + (from: string, message: StreamControlMessage) => void | Promise + >(); + + private constructor(address: string) { + this.myAddress = address; + } + + static linked(addressA: string, addressB: string): { + a: MemoryControlChannel; + b: MemoryControlChannel; + } { + const a = new MemoryControlChannel(addressA); + const b = new MemoryControlChannel(addressB); + a.peer = b; + b.peer = a; + return { a, b }; + } + + async send(peerAddress: string, message: StreamControlMessage): Promise { + if (this.peer === null) { + throw new TransferTransportError('MemoryControlChannel: not linked'); + } + if (peerAddress !== this.peer.myAddress) { + throw new TransferTransportError( + `MemoryControlChannel: peer mismatch (expected ${this.peer.myAddress}, got ${peerAddress})`, + ); + } + const target = this.peer; + const from = this.myAddress; + // Awaiting handler completion is critical: callers (the engine) rely on + // stream-init being processed BEFORE chunk POSTs go out. Real transports + // (HTTP/WS) await delivery similarly. + for (const handler of [...target.handlers]) { + try { + await handler(from, message); + } catch (err) { + console.error('[MemoryControlChannel] handler error:', err); + throw err; + } + } + } + + onMessage( + handler: (from: string, message: StreamControlMessage) => void | Promise, + ): () => void { + this.handlers.add(handler); + return () => this.handlers.delete(handler); + } +} + +/** + * In-process chunk transport. Routes 0x11 envelopes directly to a paired + * receiver-engine via a registered hook. Used in tests and same-process + * usage to avoid an HTTP roundtrip. + */ +export class MemoryTransferTransport implements ITransferTransport { + private peer: MemoryTransferTransport | null = null; + private chunkHandler: + | ((from: string, streamId: string, laneId: number, seq: bigint, bytes: Uint8Array) => Promise) + | null = null; + private resumeProvider: + | ((from: string, streamId: string) => Promise) + | null = null; + private myAddress: string; + + private constructor(address: string) { + this.myAddress = address; + } + + static linked(addressA: string, addressB: string): { + a: MemoryTransferTransport; + b: MemoryTransferTransport; + } { + const a = new MemoryTransferTransport(addressA); + const b = new MemoryTransferTransport(addressB); + a.peer = b; + b.peer = a; + return { a, b }; + } + + /** Receiver-side: register the handler invoked when a chunk arrives. */ + setChunkHandler( + handler: ( + from: string, + streamId: string, + laneId: number, + seq: bigint, + bytes: Uint8Array, + ) => Promise, + ): void { + this.chunkHandler = handler; + } + + /** Receiver-side: register the resume-state lookup. */ + setResumeProvider( + provider: (from: string, streamId: string) => Promise, + ): void { + this.resumeProvider = provider; + } + + async probe(peerAddress: string): Promise { + if (this.peer === null || peerAddress !== this.peer.myAddress) { + throw new TransferTransportError(`Memory peer ${peerAddress} not reachable`); + } + } + + async sendChunk( + peerAddress: string, + streamId: string, + laneId: number, + seq: number | bigint, + bytes: Uint8Array, + _options?: ChunkSendOptions, + ): Promise { + if (this.peer === null) throw new TransferTransportError('Not linked'); + if (peerAddress !== this.peer.myAddress) { + throw new TransferTransportError(`Peer mismatch: ${peerAddress} vs ${this.peer.myAddress}`); + } + if (this.peer.chunkHandler === null) { + throw new TransferTransportError('Peer has no chunk handler'); + } + return this.peer.chunkHandler( + this.myAddress, + streamId, + laneId, + typeof seq === 'bigint' ? seq : BigInt(seq), + bytes, + ); + } + + async fetchResumeState( + peerAddress: string, + streamId: string, + ): Promise { + if (this.peer === null) throw new TransferTransportError('Not linked'); + if (peerAddress !== this.peer.myAddress) { + throw new TransferTransportError(`Peer mismatch: ${peerAddress} vs ${this.peer.myAddress}`); + } + if (this.peer.resumeProvider === null) return null; + return this.peer.resumeProvider(this.myAddress, streamId); + } +} diff --git a/packages/shade-transfer/src/transport/transport.ts b/packages/shade-transfer/src/transport/transport.ts new file mode 100644 index 0000000..fc40f17 --- /dev/null +++ b/packages/shade-transfer/src/transport/transport.ts @@ -0,0 +1,117 @@ +/** + * Transport contracts for @shade/transfer. + * + * The library splits its wire concerns in two: + * + * - {@link IControlChannel} — carries the FOUR control messages + * (stream-init, stream-finish, stream-abort, stream-resume-*) as + * Double-Ratchet-encrypted plaintext over the application's existing + * Shade transport. The `@shade/sdk` integration (M-Stream-5) provides a + * `ShadeControlChannel` that wraps `Shade.send`/`Shade.receive`. + * + * - {@link ITransferTransport} — carries the encrypted CHUNK envelopes + * (wire type 0x11) directly between sender and receiver. This is where + * parallel-lane throughput happens; the default implementation is HTTP + * POST to a `transferRoute()`-mounted server, with optional WS upgrade. + * + * Splitting along this seam lets resume/probe/health concerns live in the + * chunk transport without leaking into the SDK's per-message ratchet API, + * and lets test code swap either side independently. + */ + +import type { + StreamControlMessage, + StreamInitMessage, +} from '@shade/streams'; + +/** Outcome of a successful chunk POST. */ +export interface ChunkAck { + /** Last sequence number the receiver has confirmed for this lane. */ + lastSeq: number; + /** Receiver-side observed bytes-received. Useful for sanity checking. */ + bytesReceived?: number; +} + +export interface ChunkSendOptions { + /** Caller-side request abort signal. */ + signal?: AbortSignal; +} + +/** + * Chunk transport contract. + * + * Implementations: + * - `ShadeTransferHttpTransport` — POST per chunk, signed via Ed25519. + * - `ShadeTransferWsTransport` — WebSocket framing (M-Stream-7). + * - `MemoryTransferTransport` — in-process pipe for tests. + */ +export interface ITransferTransport { + /** Probe peer reachability. Throws on unreachable. */ + probe(peerAddress: string): Promise; + + /** + * Ship a single encoded `0x11` envelope. Returns when the receiver has + * accepted it (and updated its persisted state, if any). Idempotent on + * (streamId, laneId, seq) — replays return the same `ChunkAck`. + */ + sendChunk( + peerAddress: string, + streamId: string, + laneId: number, + seq: number | bigint, + bytes: Uint8Array, + options?: ChunkSendOptions, + ): Promise; + + /** + * Ask the peer for its resume state for a given streamId. Returns + * `null` when the peer has no record (sender then restarts from seq 0 + * across all lanes — chunks are deterministic). + */ + fetchResumeState( + peerAddress: string, + streamId: string, + ): Promise; +} + +/** + * Receiver-reported resume state. `lastSeqAcked = -1` means no chunk for + * that lane has been received. + */ +export interface TransferResumeState { + streamId: string; + lanes: Array<{ laneId: number; lastSeqAcked: number }>; +} + +/** + * Control-channel contract. + * + * Implementations layered on top of `Shade.send`/`Shade.receive` (M-Stream-5 + * `ShadeControlChannel`) or in-memory channels (tests). + */ +export interface IControlChannel { + /** Send a plaintext control message to a peer. Returns when delivered. */ + send(peerAddress: string, message: StreamControlMessage): Promise; + + /** + * Subscribe to incoming control messages for THIS endpoint. The handler + * runs once per incoming message; multiple subscriptions are allowed. + * Returns an unsubscribe function. + */ + onMessage( + handler: ( + from: string, + message: StreamControlMessage, + ) => void | Promise, + ): () => void; +} + +/** + * Sender-side handle returned by `IControlChannel.send` for `stream-init`. + * + * Empty in v0.2.0 (the orchestrator tracks state directly), kept as a + * forward-compatible extension point. + */ +export interface InitDispatch { + init: StreamInitMessage; +} diff --git a/packages/shade-transfer/src/transport/ws-transport.ts b/packages/shade-transfer/src/transport/ws-transport.ts new file mode 100644 index 0000000..ca933ff --- /dev/null +++ b/packages/shade-transfer/src/transport/ws-transport.ts @@ -0,0 +1,331 @@ +import { TransferAbortError, TransferTransportError } from '../errors.js'; +import type { + ChunkAck, + ChunkSendOptions, + ITransferTransport, + TransferResumeState, +} from './transport.js'; +import type { TransferAuthenticator } from './http-transport.js'; +import { NoopAuthenticator } from './http-transport.js'; + +/** + * WebSocket-based chunk transport (opt-in). + * + * One connection per peer, multiplexed across all in-flight streams. Each + * outgoing chunk is sent as a binary frame; the server replies with a JSON + * ACK keyed by an internal request id. Failed connections trigger + * `TransferTransportError`; the wrapping `FallbackTransferTransport` swaps + * to HTTP. + * + * Wire format (client→server): + * 1 byte type (0x01 = chunk, 0x02 = resume-state-query) + * 16 bytes requestId (opaque) + * payload (chunk: streamId(16) + laneId(4 BE) + seq(8 BE) + envelopeBytes; + * resume-query: streamId(16)) + * + * Wire format (server→client): + * 1 byte type (0x81 = chunk-ack, 0x82 = resume-state, 0xFE = error) + * 16 bytes requestId (echoes the request) + * payload (chunk-ack: u32 lastSeq + u32 bytesReceived; + * resume-state: JSON bytes; + * error: JSON bytes) + */ +export interface ShadeTransferWsTransportOptions { + resolveWsUrl: (peerAddress: string) => Promise; + authenticator?: TransferAuthenticator; + /** WebSocket constructor override (browsers vs node). */ + WebSocketCtor?: typeof WebSocket; + /** Connect timeout in ms. */ + connectTimeoutMs?: number; +} + +const TYPE_CHUNK = 0x01; +const TYPE_RESUME_QUERY = 0x02; +const TYPE_CHUNK_ACK = 0x81; +const TYPE_RESUME_STATE = 0x82; +const TYPE_ERROR = 0xfe; + +export class ShadeTransferWsTransport implements ITransferTransport { + private readonly auth: TransferAuthenticator; + private readonly WebSocketCtor: typeof WebSocket; + private readonly connectTimeoutMs: number; + private readonly connections = new Map>(); + + constructor(private readonly options: ShadeTransferWsTransportOptions) { + this.auth = options.authenticator ?? NoopAuthenticator; + const ctor = + options.WebSocketCtor ?? + ((globalThis as unknown as { WebSocket?: typeof WebSocket }).WebSocket); + if (ctor === undefined) { + throw new TransferTransportError('WebSocket constructor not available'); + } + this.WebSocketCtor = ctor; + this.connectTimeoutMs = options.connectTimeoutMs ?? 5000; + } + + async probe(peerAddress: string): Promise { + // Probe by opening a connection; close immediately on success. + const ws = await this.connect(peerAddress); + void ws; + } + + async sendChunk( + peerAddress: string, + streamId: string, + laneId: number, + seq: number | bigint, + bytes: Uint8Array, + options?: ChunkSendOptions, + ): Promise { + const ws = await this.connect(peerAddress); + const requestId = randomBytes(16); + const seqBig = typeof seq === 'bigint' ? seq : BigInt(seq); + + const sidBytes = base64UrlToBytes(streamId); + if (sidBytes.length !== 16) { + throw new TransferTransportError(`streamId must decode to 16 bytes`); + } + const header = new Uint8Array(1 + 16 + 16 + 4 + 8); + header[0] = TYPE_CHUNK; + header.set(requestId, 1); + header.set(sidBytes, 17); + new DataView(header.buffer).setUint32(33, laneId, false); + new DataView(header.buffer).setBigUint64(37, seqBig, false); + const frame = new Uint8Array(header.length + bytes.length); + frame.set(header, 0); + frame.set(bytes, header.length); + + // Sign chunk for auth — included in WS-level metadata frame would be + // ideal, but for v0.2.0 we just call the authenticator and discard, + // matching the HTTP path's contract. Real auth lands in 0.3. + void (await this.auth.signChunk({ + streamId, + laneId, + seq: seqBig, + bodyHash: await sha256(bytes), + })); + + const response = await this.request(ws, requestId, frame, options?.signal); + const view = new DataView(response); + const type = view.getUint8(0); + if (type === TYPE_ERROR) { + const text = new TextDecoder().decode(new Uint8Array(response, 17)); + throw new TransferTransportError(`WS sendChunk error: ${text}`); + } + if (type !== TYPE_CHUNK_ACK) { + throw new TransferTransportError(`unexpected WS response type 0x${type.toString(16)}`); + } + const lastSeq = view.getUint32(17, false); + const bytesReceived = view.getUint32(21, false); + return { lastSeq, bytesReceived }; + } + + async fetchResumeState( + peerAddress: string, + streamId: string, + ): Promise { + const ws = await this.connect(peerAddress); + const requestId = randomBytes(16); + const sidBytes = base64UrlToBytes(streamId); + const frame = new Uint8Array(1 + 16 + 16); + frame[0] = TYPE_RESUME_QUERY; + frame.set(requestId, 1); + frame.set(sidBytes, 17); + + const response = await this.request(ws, requestId, frame); + const view = new DataView(response); + const type = view.getUint8(0); + if (type === TYPE_ERROR) { + const text = new TextDecoder().decode(new Uint8Array(response, 17)); + if (text.includes('not found')) return null; + throw new TransferTransportError(`WS fetchResumeState error: ${text}`); + } + if (type !== TYPE_RESUME_STATE) { + throw new TransferTransportError(`unexpected WS response type 0x${type.toString(16)}`); + } + const json = new TextDecoder().decode(new Uint8Array(response, 17)); + return JSON.parse(json) as TransferResumeState; + } + + /** Force-close all connections (call from `engine.destroy()`). */ + closeAll(): void { + for (const conn of this.connections.values()) { + void conn.then((ws) => ws.close()).catch(() => {}); + } + this.connections.clear(); + } + + private connect(peerAddress: string): Promise { + let conn = this.connections.get(peerAddress); + if (conn === undefined) { + conn = this.openConnection(peerAddress); + this.connections.set(peerAddress, conn); + conn.catch(() => { + this.connections.delete(peerAddress); + }); + } + return conn; + } + + private async openConnection(peerAddress: string): Promise { + const url = await this.options.resolveWsUrl(peerAddress); + const ws = new this.WebSocketCtor(url); + ws.binaryType = 'arraybuffer'; + + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + ws.close(); + reject(new TransferTransportError(`WS connect timeout to ${url}`)); + }, this.connectTimeoutMs); + + ws.addEventListener('open', () => { + clearTimeout(timer); + resolve(ws); + }); + ws.addEventListener('error', () => { + clearTimeout(timer); + reject(new TransferTransportError(`WS connect failed to ${url}`)); + }); + ws.addEventListener('close', () => { + this.connections.delete(peerAddress); + }); + }); + } + + private async request( + ws: WebSocket, + requestId: Uint8Array, + frame: Uint8Array, + signal?: AbortSignal, + ): Promise { + if (signal?.aborted) throw new TransferAbortError('aborted'); + return new Promise((resolve, reject) => { + const onMsg = (ev: MessageEvent): void => { + if (!(ev.data instanceof ArrayBuffer)) return; + const buf = ev.data; + if (buf.byteLength < 17) return; + const idView = new Uint8Array(buf, 1, 16); + if (!byteEqual(idView, requestId)) return; + cleanup(); + resolve(buf); + }; + const onErr = (): void => { + cleanup(); + reject(new TransferTransportError('WS errored mid-request')); + }; + const onClose = (): void => { + cleanup(); + reject(new TransferTransportError('WS closed mid-request')); + }; + const onAbort = (): void => { + cleanup(); + reject(new TransferAbortError('aborted')); + }; + function cleanup() { + ws.removeEventListener('message', onMsg); + ws.removeEventListener('error', onErr); + ws.removeEventListener('close', onClose); + signal?.removeEventListener('abort', onAbort); + } + ws.addEventListener('message', onMsg); + ws.addEventListener('error', onErr); + ws.addEventListener('close', onClose); + signal?.addEventListener('abort', onAbort, { once: true }); + ws.send(frame as unknown as ArrayBuffer); + }); + } +} + +/** + * Tries the primary transport first, falls back to secondary on + * `TransferTransportError`. Does NOT retry per-chunk between transports — + * the engine's retry layer handles that — but switches sticky after the + * first failure so subsequent calls go to the fallback. + */ +export class FallbackTransferTransport implements ITransferTransport { + private failed = false; + + constructor( + private readonly primary: ITransferTransport, + private readonly fallback: ITransferTransport, + ) {} + + private active(): ITransferTransport { + return this.failed ? this.fallback : this.primary; + } + + async probe(peerAddress: string): Promise { + if (this.failed) return this.fallback.probe(peerAddress); + try { + await this.primary.probe(peerAddress); + } catch (err) { + if (err instanceof TransferTransportError) { + this.failed = true; + return this.fallback.probe(peerAddress); + } + throw err; + } + } + + async sendChunk( + peerAddress: string, + streamId: string, + laneId: number, + seq: number | bigint, + bytes: Uint8Array, + options?: ChunkSendOptions, + ): Promise { + if (this.failed) return this.fallback.sendChunk(peerAddress, streamId, laneId, seq, bytes, options); + try { + return await this.primary.sendChunk(peerAddress, streamId, laneId, seq, bytes, options); + } catch (err) { + if (err instanceof TransferTransportError) { + this.failed = true; + return this.fallback.sendChunk(peerAddress, streamId, laneId, seq, bytes, options); + } + throw err; + } + } + + async fetchResumeState( + peerAddress: string, + streamId: string, + ): Promise { + return this.active().fetchResumeState(peerAddress, streamId); + } + + /** True if the primary failed and we've fallen back. */ + get fellBack(): boolean { + return this.failed; + } +} + +// ─── Helpers ───────────────────────────────────────────────── + +function randomBytes(n: number): Uint8Array { + const b = new Uint8Array(n); + globalThis.crypto.getRandomValues(b); + return b; +} + +async function sha256(data: Uint8Array): Promise { + const buf = await globalThis.crypto.subtle.digest('SHA-256', data as unknown as ArrayBuffer); + return new Uint8Array(buf); +} + +function byteEqual(a: Uint8Array, b: Uint8Array): boolean { + if (a.length !== b.length) return false; + for (let i = 0; i < a.length; i++) { + if (a[i] !== b[i]) return false; + } + return true; +} + +function base64UrlToBytes(s: string): Uint8Array { + const padded = s.replace(/-/g, '+').replace(/_/g, '/'); + const pad = padded.length % 4 === 0 ? '' : '='.repeat(4 - (padded.length % 4)); + const bin = atob(padded + pad); + const out = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); + return out; +} diff --git a/packages/shade-transfer/src/types.ts b/packages/shade-transfer/src/types.ts new file mode 100644 index 0000000..dd15bc2 --- /dev/null +++ b/packages/shade-transfer/src/types.ts @@ -0,0 +1,139 @@ +import type { StreamMetadata, LaneInitSpec } from '@shade/streams'; + +/** + * Browser File System Access API handle. Declared locally as an opaque + * type so this package builds without `lib: ["DOM"]`. Consumers in browsers + * pass a real `FileSystemFileHandle`; the runtime structural shape is + * checked at use-sites. + */ +export interface FileSystemFileHandleLike { + readonly kind: 'file'; + readonly name: string; + createWritable(): Promise; +} + +/** Default per-chunk plaintext size (bytes). */ +export const DEFAULT_CHUNK_SIZE = 1024 * 1024; // 1 MiB + +/** Hard cap on per-chunk plaintext size. */ +export const DEFAULT_MAX_CHUNK_SIZE = 16 * 1024 * 1024; // 16 MiB + +/** Default lane count for parallel transfers. */ +export const DEFAULT_LANE_COUNT = 4; + +/** + * Anything we accept as the byte source for an upload. The orchestrator + * normalizes these (see `sender/input.ts`) into a chunk reader. + */ +export type TransferInput = + | Uint8Array + | Blob + | File + | ReadableStream; + +/** Sink kinds for an incoming transfer. The receiver picks ONE. */ +export type TransferOutput = + | { kind: 'pipe'; pipeTo: WritableStream } + | { kind: 'callback'; onChunk: (chunk: Uint8Array) => void | Promise } + | { kind: 'buffer' } + | { kind: 'file'; path: string } + | { kind: 'fileHandle'; handle: FileSystemFileHandleLike }; + +export type TransferDirection = 'send' | 'receive'; +export type TransferStatus = + | 'pending' + | 'active' + | 'paused' + | 'finished' + | 'aborted' + | 'failed'; + +export interface LaneProgress { + laneId: number; + bytesSent: number; + bytesTotal?: number; + seq: number; + state: 'idle' | 'sending' | 'paused' | 'done' | 'error'; +} + +export interface TransferProgress { + streamId: string; + bytesSent: number; + bytesTotal?: number; + /** EMA-smoothed throughput in bytes/sec. */ + bytesPerSecond: number; + etaSeconds?: number; + /** [0, 1] when bytesTotal is known; undefined otherwise. */ + percent?: number; + lanes: LaneProgress[]; +} + +export type TransferEvent = + | { type: 'start'; streamId: string } + | { type: 'progress'; progress: TransferProgress } + | { type: 'lane-done'; laneId: number } + | { type: 'pause'; reason: 'network' | 'manual' } + | { type: 'resume' } + | { type: 'complete'; streamId: string; sha256: string; durationMs: number } + | { type: 'abort'; reason: string } + | { type: 'error'; error: unknown }; + +export interface TransferResult { + streamId: string; + bytesSent: number; + /** Hex-encoded sha256 over the original plaintext. */ + sha256: string; + durationMs: number; +} + +export interface TransferOptions { + to: string; + input: TransferInput; + metadata?: Partial; + /** Lane count override; capped at 64 internally. */ + lanes?: number; + chunkSize?: number; + maxChunkSize?: number; + partition?: 'auto' | 'range' | 'round-robin'; + signal?: AbortSignal; + onProgress?: (p: TransferProgress) => void; + onEvent?: (e: TransferEvent) => void; +} + +export interface TransferHandle { + readonly streamId: string; + readonly events: AsyncIterable; + pause(): Promise; + resume(): Promise; + abort(reason?: string): Promise; + done(): Promise; +} + +export interface IncomingTransferAcceptOptions { + output: TransferOutput; + onProgress?: (p: TransferProgress) => void; + onEvent?: (e: TransferEvent) => void; +} + +export interface IncomingTransfer { + readonly streamId: string; + readonly from: string; + readonly metadata: StreamMetadata; + readonly lanes: LaneInitSpec[]; + /** Accept and start receiving. Throws if already accepted/declined. */ + accept(options: IncomingTransferAcceptOptions): Promise; + /** Reject; sends abort to sender. */ + decline(reason?: string): Promise; +} + +export interface TransferSummary { + streamId: string; + direction: TransferDirection; + peerAddress: string; + status: TransferStatus; + bytesTotal?: number; + bytesProcessed: number; + createdAt: number; + updatedAt: number; + metadata: StreamMetadata | null; +} diff --git a/packages/shade-transfer/tests/http-roundtrip.test.ts b/packages/shade-transfer/tests/http-roundtrip.test.ts new file mode 100644 index 0000000..33628fd --- /dev/null +++ b/packages/shade-transfer/tests/http-roundtrip.test.ts @@ -0,0 +1,204 @@ +import { describe, test, expect, afterAll } from 'bun:test'; +import { SubtleCryptoProvider } from '@shade/crypto-web'; +import { sha256Once } from '@shade/streams'; +import { + TransferEngine, + MemoryControlChannel, + ShadeTransferHttpTransport, + createTransferRoutes, + type TransferHandle, + type TransferResult, +} from '../src/index.js'; + +const crypto = new SubtleCryptoProvider(); + +function hex(bytes: Uint8Array): string { + return Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join(''); +} + +interface IntegrationServers { + baseUrl: string; + senderEngine: TransferEngine; + receiverEngine: TransferEngine; + serverHandle: { stop: () => void }; +} + +const cleanups: Array<() => void> = []; + +afterAll(() => { + for (const c of cleanups) c(); +}); + +async function setup(): Promise { + const { a: ctrlA, b: ctrlB } = MemoryControlChannel.linked('alice', 'bob'); + + const receiverEngine = new TransferEngine({ + crypto, + controlChannel: ctrlB, + transport: { + // Receiver-side transport is unused for outgoing operations; we only + // route incoming chunks via the Hono server. + probe: async () => undefined, + sendChunk: async () => { + throw new Error('receiver-side sendChunk should not be called'); + }, + fetchResumeState: async () => null, + }, + myAddress: 'bob', + }); + + // Spin up a Hono+Bun server that routes chunks into receiverEngine. + const app = await createTransferRoutes(receiverEngine); + // Bun's `Bun.serve` accepts a Hono app's `fetch` handler. + const bunGlobal = (globalThis as unknown as { Bun?: { serve: (opts: unknown) => { url: URL; stop: () => void } } }).Bun; + if (bunGlobal === undefined) throw new Error('Bun runtime required for this test'); + const server = bunGlobal.serve({ + port: 0, + fetch: (req: Request) => app.fetch(req), + }); + const baseUrl = server.url.toString().replace(/\/$/, ''); + + const senderEngine = new TransferEngine({ + crypto, + controlChannel: ctrlA, + transport: new ShadeTransferHttpTransport({ + resolveBaseUrl: async () => baseUrl, + authenticator: { + signChunk: async () => ({ 'X-Shade-Sender-Address': 'alice' }), + signControl: async () => ({ 'X-Shade-Sender-Address': 'alice' }), + }, + }), + myAddress: 'alice', + }); + + cleanups.push(() => { + server.stop(); + senderEngine.destroy(); + receiverEngine.destroy(); + }); + + return { baseUrl, senderEngine, receiverEngine, serverHandle: server }; +} + +async function uploadRoundtrip( + servers: IntegrationServers, + input: Uint8Array, + opts?: { lanes?: number; chunkSize?: number; partition?: 'auto' | 'range' | 'round-robin' }, +): Promise<{ senderResult: TransferResult; received: Uint8Array }> { + let resolveRecv!: (h: TransferHandle) => void; + const recvHandlePromise = new Promise((r) => { + resolveRecv = r; + }); + const unsubscribe = servers.receiverEngine.onIncomingTransfer(async (incoming) => { + const h = await incoming.accept({ output: { kind: 'buffer' } }); + resolveRecv(h); + }); + + const handle = await servers.senderEngine.upload({ + to: 'bob', + input, + ...(opts?.lanes !== undefined ? { lanes: opts.lanes } : {}), + ...(opts?.chunkSize !== undefined ? { chunkSize: opts.chunkSize } : {}), + ...(opts?.partition !== undefined ? { partition: opts.partition } : {}), + metadata: { name: 'http-test.bin' }, + }); + const recvHandle = await recvHandlePromise; + const [senderResult, recvResult] = await Promise.all([handle.done(), recvHandle.done()]); + unsubscribe(); + const received = + (recvResult as TransferResult & { bytes?: Uint8Array }).bytes ?? new Uint8Array(); + return { senderResult, received }; +} + +describe('HTTP transport — Bun.serve loopback', () => { + test('100 KiB / 1 lane', async () => { + const servers = await setup(); + const input = crypto.randomBytes(100 * 1024); + const { senderResult, received } = await uploadRoundtrip(servers, input, { + lanes: 1, + chunkSize: 32 * 1024, + }); + expect(received).toEqual(input); + expect(senderResult.sha256).toBe(hex(sha256Once(input))); + expect(senderResult.bytesSent).toBe(input.length); + }); + + test('1 MiB / 4 lanes range', async () => { + const servers = await setup(); + const input = crypto.randomBytes(1024 * 1024); + const { senderResult, received } = await uploadRoundtrip(servers, input, { + lanes: 4, + chunkSize: 64 * 1024, + partition: 'range', + }); + expect(received).toEqual(input); + expect(senderResult.sha256).toBe(hex(sha256Once(input))); + }); + + test('1 MiB / 4 lanes round-robin', async () => { + const servers = await setup(); + const input = crypto.randomBytes(1024 * 1024); + const { senderResult, received } = await uploadRoundtrip(servers, input, { + lanes: 4, + chunkSize: 64 * 1024, + partition: 'round-robin', + }); + expect(received).toEqual(input); + expect(senderResult.sha256).toBe(hex(sha256Once(input))); + }); + + test( + '8 MiB / 4 lanes range (ship-gate proxy for 100 MB)', + async () => { + const servers = await setup(); + const input = crypto.randomBytes(8 * 1024 * 1024); + const { senderResult, received } = await uploadRoundtrip(servers, input, { + lanes: 4, + chunkSize: 256 * 1024, + partition: 'range', + }); + expect(received).toEqual(input); + expect(senderResult.sha256).toBe(hex(sha256Once(input))); + }, + 30_000, + ); + + test('upload with ReadableStream input — round-robin auto-pick', async () => { + const servers = await setup(); + const input = crypto.randomBytes(512 * 1024); + const stream = new ReadableStream({ + start(controller) { + for (let off = 0; off < input.length; off += 32 * 1024) { + controller.enqueue(input.subarray(off, Math.min(off + 32 * 1024, input.length))); + } + controller.close(); + }, + }); + const { senderResult, received } = await uploadRoundtrip(servers, stream as unknown as Uint8Array, { + lanes: 4, + chunkSize: 32 * 1024, + }); + expect(received).toEqual(input); + expect(senderResult.sha256).toBe(hex(sha256Once(input))); + }); + + test('peer offline → TransferOfflineError', async () => { + const { a: ctrlA } = MemoryControlChannel.linked('alice', 'bob'); + const sender = new TransferEngine({ + crypto, + controlChannel: ctrlA, + transport: new ShadeTransferHttpTransport({ + resolveBaseUrl: async () => 'http://127.0.0.1:1', // intentionally unreachable + }), + myAddress: 'alice', + }); + cleanups.push(() => sender.destroy()); + + await expect( + sender.upload({ + to: 'bob', + input: crypto.randomBytes(64), + }), + ).rejects.toThrow(/offline|fetch failed|connection|ECONN|connect/i); + }); +}); diff --git a/packages/shade-transfer/tests/memory-roundtrip.test.ts b/packages/shade-transfer/tests/memory-roundtrip.test.ts new file mode 100644 index 0000000..8520195 --- /dev/null +++ b/packages/shade-transfer/tests/memory-roundtrip.test.ts @@ -0,0 +1,193 @@ +import { describe, test, expect } from 'bun:test'; +import { SubtleCryptoProvider } from '@shade/crypto-web'; +import { sha256Once } from '@shade/streams'; +import { + TransferEngine, + MemoryControlChannel, + MemoryTransferTransport, +} from '../src/index.js'; +import type { IncomingTransfer, TransferResult } from '../src/index.js'; + +const crypto = new SubtleCryptoProvider(); + +function hex(bytes: Uint8Array): string { + return Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join(''); +} + +interface PairedEngines { + sender: TransferEngine; + receiver: TransferEngine; +} + +function makePair(): PairedEngines { + const { a: ctrlA, b: ctrlB } = MemoryControlChannel.linked('alice', 'bob'); + const { a: txA, b: txB } = MemoryTransferTransport.linked('alice', 'bob'); + + const senderEngine = new TransferEngine({ + crypto, + controlChannel: ctrlA, + transport: txA, + myAddress: 'alice', + }); + const receiverEngine = new TransferEngine({ + crypto, + controlChannel: ctrlB, + transport: txB, + myAddress: 'bob', + }); + + // Wire receiver-side transport to route chunks into receiver-engine. + txB.setChunkHandler(async (from, streamId, laneId, seq, bytes) => + receiverEngine.receiveChunk(from, streamId, laneId, seq, bytes), + ); + txB.setResumeProvider(async (from, streamId) => + receiverEngine.getResumeState(from, streamId), + ); + + return { sender: senderEngine, receiver: receiverEngine }; +} + +async function uploadAndAwait( + pair: PairedEngines, + input: Uint8Array, + opts?: { lanes?: number; chunkSize?: number; partition?: 'auto' | 'range' | 'round-robin' }, +): Promise<{ result: TransferResult; received: Uint8Array }> { + // The handler accepts and PUBLISHES the receive-handle out-of-band so + // it can return promptly (control channel awaits handler completion). + let resolveReceiveHandle!: (h: import('../src/index.js').TransferHandle) => void; + const receiveHandlePromise = new Promise( + (r) => { resolveReceiveHandle = r; }, + ); + const unsubscribe = pair.receiver.onIncomingTransfer(async (incoming: IncomingTransfer) => { + const handle = await incoming.accept({ output: { kind: 'buffer' } }); + resolveReceiveHandle(handle); + }); + + const handle = await pair.sender.upload({ + to: 'bob', + input, + ...(opts?.lanes !== undefined ? { lanes: opts.lanes } : {}), + ...(opts?.chunkSize !== undefined ? { chunkSize: opts.chunkSize } : {}), + ...(opts?.partition !== undefined ? { partition: opts.partition } : {}), + metadata: { name: 'test.bin', contentType: 'application/octet-stream' }, + }); + const recvHandle = await receiveHandlePromise; + const [senderResult, receiverResult] = await Promise.all([handle.done(), recvHandle.done()]); + unsubscribe(); + const bytes = + (receiverResult as TransferResult & { bytes?: Uint8Array }).bytes ?? new Uint8Array(0); + return { result: senderResult, received: bytes }; +} + +describe('TransferEngine (memory loopback)', () => { + test('1 KiB upload — 1 lane (auto-degrade for small file)', async () => { + const pair = makePair(); + const input = crypto.randomBytes(1024); + const { result, received } = await uploadAndAwait(pair, input, { lanes: 4, chunkSize: 256 }); + expect(received).toEqual(input); + expect(result.sha256).toBe(hex(sha256Once(input))); + }); + + test('256 KiB upload — 4 lanes range partition', async () => { + const pair = makePair(); + const input = crypto.randomBytes(256 * 1024); + const { result, received } = await uploadAndAwait(pair, input, { + lanes: 4, + chunkSize: 16 * 1024, + partition: 'range', + }); + expect(received).toEqual(input); + expect(result.sha256).toBe(hex(sha256Once(input))); + }); + + test('1 MiB upload — 4 lanes round-robin partition', async () => { + const pair = makePair(); + const input = crypto.randomBytes(1024 * 1024); + const { result, received } = await uploadAndAwait(pair, input, { + lanes: 4, + chunkSize: 64 * 1024, + partition: 'round-robin', + }); + expect(received).toEqual(input); + expect(result.sha256).toBe(hex(sha256Once(input))); + }); + + test('integrity: same sha256 across lane counts', async () => { + const input = crypto.randomBytes(512 * 1024); + const expected = hex(sha256Once(input)); + for (const lanes of [1, 2, 4, 8]) { + const pair = makePair(); + const { result, received } = await uploadAndAwait(pair, input, { lanes, chunkSize: 8 * 1024 }); + expect(received).toEqual(input); + expect(result.sha256).toBe(expected); + } + }); + + test('upload with ReadableStream input → round-robin partition', async () => { + const pair = makePair(); + const input = crypto.randomBytes(300 * 1024); + const stream = new ReadableStream({ + start(controller) { + for (let off = 0; off < input.length; off += 64 * 1024) { + controller.enqueue(input.subarray(off, Math.min(off + 64 * 1024, input.length))); + } + controller.close(); + }, + }); + let resolveRecv!: (h: import('../src/index.js').TransferHandle) => void; + const recvHandlePromise = new Promise((r) => { + resolveRecv = r; + }); + const unsubscribe = pair.receiver.onIncomingTransfer(async (incoming) => { + const h = await incoming.accept({ output: { kind: 'buffer' } }); + resolveRecv(h); + }); + const handle = await pair.sender.upload({ + to: 'bob', + input: stream, + lanes: 4, + chunkSize: 32 * 1024, + }); + const recvHandle = await recvHandlePromise; + const [, recvResult] = await Promise.all([handle.done(), recvHandle.done()]); + unsubscribe(); + const bytes = (recvResult as TransferResult & { bytes?: Uint8Array }).bytes ?? new Uint8Array(); + expect(bytes).toEqual(input); + }); + + test('progress events fire and end with complete', async () => { + const pair = makePair(); + const input = crypto.randomBytes(64 * 1024); + const senderProgressSamples: number[] = []; + const receiverEvents: string[] = []; + + let resolveRecv!: (h: import('../src/index.js').TransferHandle) => void; + const recvHandlePromise = new Promise((r) => { + resolveRecv = r; + }); + const unsub = pair.receiver.onIncomingTransfer(async (incoming) => { + const h = await incoming.accept({ + output: { kind: 'buffer' }, + onEvent: (e) => { + receiverEvents.push(e.type); + }, + }); + resolveRecv(h); + }); + + const handle = await pair.sender.upload({ + to: 'bob', + input, + lanes: 2, + chunkSize: 8 * 1024, + onProgress: (p) => senderProgressSamples.push(p.bytesSent), + }); + const recvHandle = await recvHandlePromise; + await Promise.all([handle.done(), recvHandle.done()]); + unsub(); + expect(senderProgressSamples.length).toBeGreaterThan(0); + expect(senderProgressSamples[senderProgressSamples.length - 1]).toBe(64 * 1024); + expect(receiverEvents).toContain('start'); + expect(receiverEvents).toContain('complete'); + }); +}); diff --git a/packages/shade-transfer/tests/resume.test.ts b/packages/shade-transfer/tests/resume.test.ts new file mode 100644 index 0000000..76ac562 --- /dev/null +++ b/packages/shade-transfer/tests/resume.test.ts @@ -0,0 +1,152 @@ +import { describe, test, expect } from 'bun:test'; +import { SubtleCryptoProvider } from '@shade/crypto-web'; +import { sha256Once } from '@shade/streams'; +import { + TransferEngine, + MemoryControlChannel, + MemoryResumeStore, + ShadeTransferHttpTransport, + createTransferRoutes, + type TransferHandle, + type TransferResult, +} from '../src/index.js'; + +const crypto = new SubtleCryptoProvider(); + +function hex(b: Uint8Array): string { + return Array.from(b, (x) => x.toString(16).padStart(2, '0')).join(''); +} + +describe('Resume protocol — kill-restart-verify', () => { + test('sender crash mid-transfer → resumeUpload completes the same stream', async () => { + const senderResumeStore = new MemoryResumeStore(); + // Stable deviceKey shared across engine instances — simulates a stable + // identity-derived key. In real use this is `deriveDeviceKey(identity)`. + const deviceKey = crypto.randomBytes(32); + + // Receiver is a single, long-lived engine. Its in-memory IncomingState + // already tracks accepted chunks; we don't need to persist receiver state + // for the in-process scenario. + const { a: ctrlA, b: ctrlB } = MemoryControlChannel.linked('alice', 'bob'); + const receiverEngine = new TransferEngine({ + crypto, + controlChannel: ctrlB, + transport: { + probe: async () => undefined, + sendChunk: async () => { + throw new Error('receiver-side sendChunk should not be called'); + }, + fetchResumeState: async () => null, + }, + myAddress: 'bob', + }); + const receiverApp = await createTransferRoutes(receiverEngine); + const port = 22000 + Math.floor(Math.random() * 500); + const server = Bun.serve({ port, fetch: receiverApp.fetch }); + const baseUrl = `http://localhost:${port}`; + + // Receiver accepts incoming. + let resolveRecv!: (h: TransferHandle) => void; + const recvHandlePromise = new Promise((r) => { + resolveRecv = r; + }); + receiverEngine.onIncomingTransfer(async (incoming) => { + const h = await incoming.accept({ output: { kind: 'buffer' } }); + resolveRecv(h); + }); + + // Sender #1 — this one will "crash" partway through. + const senderEngine1 = new TransferEngine({ + crypto, + controlChannel: ctrlA, + transport: new ShadeTransferHttpTransport({ + resolveBaseUrl: async () => baseUrl, + authenticator: { + signChunk: async () => ({ 'X-Shade-Sender-Address': 'alice' }), + signControl: async () => ({ 'X-Shade-Sender-Address': 'alice' }), + }, + }), + myAddress: 'alice', + resumeStore: senderResumeStore, + deviceKey, + }); + + const input = crypto.randomBytes(256 * 1024); // 256 KiB + + // Start an upload that will be intentionally interrupted. We pause the + // sender by aborting the upload after ~25% bytes. The receiver still + // holds its IncomingState — chunks already accepted stay tracked. + const abort = new AbortController(); + let bytesAtAbort = 0; + const handle1 = await senderEngine1.upload({ + to: 'bob', + input, + lanes: 4, + chunkSize: 8 * 1024, + partition: 'range', + signal: abort.signal, + onProgress: (p) => { + if (p.bytesSent > input.length / 4 && bytesAtAbort === 0) { + bytesAtAbort = p.bytesSent; + abort.abort(); + } + }, + }); + const streamId = handle1.streamId; + + // Wait for the upload to fail (abort). + let firstErrCaught = false; + try { + await handle1.done(); + } catch { + firstErrCaught = true; + } + expect(firstErrCaught).toBe(true); + expect(bytesAtAbort).toBeGreaterThan(0); + + // Tear down the first sender engine — simulates a crashed/restarted client. + senderEngine1.destroy(); + + // Sender #2 — fresh engine, same resume store. + const senderEngine2 = new TransferEngine({ + crypto, + controlChannel: ctrlA, // re-use the link; in real life this would + transport: new ShadeTransferHttpTransport({ + resolveBaseUrl: async () => baseUrl, + authenticator: { + signChunk: async () => ({ 'X-Shade-Sender-Address': 'alice' }), + signControl: async () => ({ 'X-Shade-Sender-Address': 'alice' }), + }, + }), + myAddress: 'alice', + resumeStore: senderResumeStore, + deviceKey, + }); + + // Verify state is still in the resume store. + const persisted = await senderResumeStore.get(streamId); + expect(persisted).not.toBeNull(); + expect(persisted!.status).toBe('active'); + + // Resume the upload with the same input bytes. + const handle2 = await senderEngine2.resumeUpload(streamId, input); + const senderResult = await handle2.done(); + expect(senderResult.streamId).toBe(streamId); + + // Receiver finishes too. + const recvHandle = await recvHandlePromise; + const recvResult = await recvHandle.done(); + const received = + (recvResult as TransferResult & { bytes?: Uint8Array }).bytes ?? new Uint8Array(); + expect(received).toEqual(input); + expect(senderResult.sha256).toBe(hex(sha256Once(input))); + + // Persisted state should now be finished. + const after = await senderResumeStore.get(streamId); + expect(after?.status).toBe('finished'); + + senderEngine2.destroy(); + receiverEngine.destroy(); + server.stop(); + }, 30_000); +}); diff --git a/packages/shade-transfer/tests/ws-fallback.test.ts b/packages/shade-transfer/tests/ws-fallback.test.ts new file mode 100644 index 0000000..3f0f153 --- /dev/null +++ b/packages/shade-transfer/tests/ws-fallback.test.ts @@ -0,0 +1,98 @@ +import { describe, test, expect, afterAll } from 'bun:test'; +import { SubtleCryptoProvider } from '@shade/crypto-web'; +import { sha256Once } from '@shade/streams'; +import { + TransferEngine, + MemoryControlChannel, + ShadeTransferHttpTransport, + ShadeTransferWsTransport, + FallbackTransferTransport, + createTransferRoutes, + type TransferHandle, + type TransferResult, +} from '../src/index.js'; + +const crypto = new SubtleCryptoProvider(); + +function hex(b: Uint8Array): string { + return Array.from(b, (x) => x.toString(16).padStart(2, '0')).join(''); +} + +const cleanups: Array<() => void> = []; +afterAll(() => { + for (const c of cleanups) c(); +}); + +describe('WS opt-in transport with HTTP fallback', () => { + test('WS connect failure → falls back to HTTP transparently', async () => { + const { a: ctrlA, b: ctrlB } = MemoryControlChannel.linked('alice', 'bob'); + const receiverEngine = new TransferEngine({ + crypto, + controlChannel: ctrlB, + transport: { + probe: async () => undefined, + sendChunk: async () => { + throw new Error('not used'); + }, + fetchResumeState: async () => null, + }, + myAddress: 'bob', + }); + const httpApp = await createTransferRoutes(receiverEngine); + const httpPort = 23000 + Math.floor(Math.random() * 500); + const httpServer = Bun.serve({ port: httpPort, fetch: httpApp.fetch }); + const httpBaseUrl = `http://localhost:${httpPort}`; + cleanups.push(() => httpServer.stop()); + + const ws = new ShadeTransferWsTransport({ + // Resolve to a closed port — guaranteed connect failure. + resolveWsUrl: async () => `ws://127.0.0.1:1`, + connectTimeoutMs: 500, + }); + const http = new ShadeTransferHttpTransport({ + resolveBaseUrl: async () => httpBaseUrl, + authenticator: { + signChunk: async () => ({ 'X-Shade-Sender-Address': 'alice' }), + signControl: async () => ({ 'X-Shade-Sender-Address': 'alice' }), + }, + }); + const fallback = new FallbackTransferTransport(ws, http); + + const senderEngine = new TransferEngine({ + crypto, + controlChannel: ctrlA, + transport: fallback, + myAddress: 'alice', + }); + cleanups.push(() => senderEngine.destroy()); + cleanups.push(() => receiverEngine.destroy()); + + let resolveRecv!: (h: TransferHandle) => void; + const recvHandlePromise = new Promise((r) => { + resolveRecv = r; + }); + receiverEngine.onIncomingTransfer(async (incoming) => { + const h = await incoming.accept({ output: { kind: 'buffer' } }); + resolveRecv(h); + }); + + const input = crypto.randomBytes(64 * 1024); + const handle = await senderEngine.upload({ + to: 'bob', + input, + lanes: 2, + chunkSize: 8 * 1024, + }); + const recvHandle = await recvHandlePromise; + const [senderResult, recvResult] = await Promise.all([ + handle.done(), + recvHandle.done(), + ]); + + expect(fallback.fellBack).toBe(true); + const received = + (recvResult as TransferResult & { bytes?: Uint8Array }).bytes ?? new Uint8Array(); + expect(received).toEqual(input); + expect(senderResult.sha256).toBe(hex(sha256Once(input))); + }); +}); diff --git a/packages/shade-transfer/tsconfig.json b/packages/shade-transfer/tsconfig.json new file mode 100644 index 0000000..a086b14 --- /dev/null +++ b/packages/shade-transfer/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src"] +} diff --git a/packages/shade-transport/package.json b/packages/shade-transport/package.json index ab56e3f..a7588b5 100644 --- a/packages/shade-transport/package.json +++ b/packages/shade-transport/package.json @@ -1,6 +1,6 @@ { "name": "@shade/transport", - "version": "0.1.0", + "version": "0.3.0", "type": "module", "main": "src/index.ts", "types": "src/index.ts", diff --git a/packages/shade-transport/src/fetch-transport.ts b/packages/shade-transport/src/fetch-transport.ts index 00a5691..f176cf4 100644 --- a/packages/shade-transport/src/fetch-transport.ts +++ b/packages/shade-transport/src/fetch-transport.ts @@ -1,5 +1,5 @@ import type { PreKeyBundle, OneTimePreKey, CryptoProvider } from '@shade/core'; -import { fromBase64, NetworkError } from '@shade/core'; +import { NetworkError } from '@shade/core'; /** * HTTP transport client for the Shade Prekey Server. @@ -27,7 +27,9 @@ export class ShadeFetchTransport { constructor(options: { baseUrl: string; crypto: CryptoProvider; signingPrivateKey?: Uint8Array }) { this.baseUrl = options.baseUrl; this.crypto = options.crypto; - this.signingPrivateKey = options.signingPrivateKey; + if (options.signingPrivateKey !== undefined) { + this.signingPrivateKey = options.signingPrivateKey; + } } private headers(): Record { @@ -94,8 +96,14 @@ export class ShadeFetchTransport { }); if (!res.ok) throw new NetworkError(`Fetch bundle failed: ${res.status}`, res.status); - const data = await res.json(); - return { + const data = (await res.json()) as { + registrationId?: number; + identitySigningKey: string; + identityDHKey: string; + signedPreKey: { keyId: number; publicKey: string; signature: string }; + oneTimePreKey?: { keyId: number; publicKey: string }; + }; + const bundle: PreKeyBundle = { registrationId: data.registrationId ?? 0, identitySigningKey: fromB64(data.identitySigningKey), identityDHKey: fromB64(data.identityDHKey), @@ -104,13 +112,14 @@ export class ShadeFetchTransport { publicKey: fromB64(data.signedPreKey.publicKey), signature: fromB64(data.signedPreKey.signature), }, - oneTimePreKey: data.oneTimePreKey - ? { - keyId: data.oneTimePreKey.keyId, - publicKey: fromB64(data.oneTimePreKey.publicKey), - } - : undefined, }; + if (data.oneTimePreKey) { + bundle.oneTimePreKey = { + keyId: data.oneTimePreKey.keyId, + publicKey: fromB64(data.oneTimePreKey.publicKey), + }; + } + return bundle; } /** Upload additional one-time prekeys (signed) */ @@ -133,7 +142,7 @@ export class ShadeFetchTransport { body: JSON.stringify(signed), }); if (!res.ok) throw new NetworkError(`Replenish failed: ${res.status}`, res.status); - const data = await res.json(); + const data = (await res.json()) as { remaining: number }; return data.remaining; } @@ -143,7 +152,7 @@ export class ShadeFetchTransport { headers: this.headers(), }); if (!res.ok) throw new NetworkError(`Count failed: ${res.status}`, res.status); - const data = await res.json(); + const data = (await res.json()) as { count: number }; return data.count; } diff --git a/packages/shade-transport/src/ws-adapter.ts b/packages/shade-transport/src/ws-adapter.ts index 5c3ea7b..1de8ba6 100644 --- a/packages/shade-transport/src/ws-adapter.ts +++ b/packages/shade-transport/src/ws-adapter.ts @@ -1,4 +1,4 @@ -import type { ShadeSessionManager, ShadeEnvelope, RatchetMessage } from '@shade/core'; +import type { ShadeSessionManager } from '@shade/core'; import { encodeEnvelope, decodeEnvelope } from '@shade/proto'; /** @@ -40,8 +40,10 @@ export class ShadeWebSocket { const envelope = await this.manager.encrypt(this.peerAddress, plaintext); const bytes = encodeEnvelope(envelope); - // Send as binary - this.ws.send(bytes); + // Send as binary. Cast through BufferSource to satisfy + // Uint8Array vs Uint8Array mismatch under + // strict TS configurations (5.7+). + this.ws.send(bytes as unknown as ArrayBuffer); } /** Register a handler for decrypted incoming messages */ diff --git a/packages/shade-widgets/package.json b/packages/shade-widgets/package.json index 216a10e..b7b083b 100644 --- a/packages/shade-widgets/package.json +++ b/packages/shade-widgets/package.json @@ -1,9 +1,14 @@ { "name": "@shade/widgets", - "version": "0.1.0", + "version": "0.3.0", "type": "module", "main": "src/index.ts", "types": "src/index.ts", + "dependencies": { + "@shade/sdk": "workspace:*", + "@shade/streams": "workspace:*", + "@shade/transfer": "workspace:*" + }, "peerDependencies": { "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" @@ -11,6 +16,7 @@ "devDependencies": { "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", + "happy-dom": "^15.11.7", "react": "^19.2.5", "react-dom": "^19.2.5" } diff --git a/packages/shade-widgets/src/ShadeRuntimeProvider.tsx b/packages/shade-widgets/src/ShadeRuntimeProvider.tsx new file mode 100644 index 0000000..eb0d439 --- /dev/null +++ b/packages/shade-widgets/src/ShadeRuntimeProvider.tsx @@ -0,0 +1,65 @@ +import React, { createContext, useContext, useMemo } from 'react'; +import type { Shade } from '@shade/sdk'; +import { resolveTheme, type ShadeTheme, type ThemeMode } from './theme.js'; + +export interface ShadeRuntimeContextValue { + runtime: Shade; + theme: ShadeTheme; +} + +const ShadeRuntimeContext = createContext(null); + +export interface ShadeRuntimeProviderProps { + /** Initialized `Shade` instance (after `await shade.initialize()`). */ + runtime: Shade; + /** Theme mode: `'dark'` (default), `'light'`, or `'auto'`. */ + themeMode?: ThemeMode; + /** Optional theme overrides applied on top of the resolved base theme. */ + themeOverrides?: Partial; + children: React.ReactNode; +} + +/** + * `ShadeRuntimeProvider` — root for the upload/download widget tree. + * + * Distinct from `ShadeProvider` (which targets the observer dashboard) — + * this one wraps an actual `Shade` runtime so transfer hooks/components + * can call `shade.upload(...)`, `shade.onIncomingTransfer(...)`, etc. + * + * Wrap your application root once; multiple uploaders/downloaders share + * the same runtime instance. + */ +export function ShadeRuntimeProvider({ + runtime, + themeMode = 'dark', + themeOverrides, + children, +}: ShadeRuntimeProviderProps): React.ReactElement { + const value = useMemo( + () => ({ runtime, theme: { ...resolveTheme(themeMode), ...themeOverrides } }), + [runtime, themeMode, themeOverrides], + ); + return ( + {children} + ); +} + +export function useShadeRuntime(): Shade { + const ctx = useContext(ShadeRuntimeContext); + if (ctx === null) { + throw new Error( + 'useShadeRuntime must be used inside ', + ); + } + return ctx.runtime; +} + +export function useShadeRuntimeTheme(): ShadeTheme { + const ctx = useContext(ShadeRuntimeContext); + if (ctx === null) { + throw new Error( + 'useShadeRuntimeTheme must be used inside ', + ); + } + return ctx.theme; +} diff --git a/packages/shade-widgets/src/components/ServerStatus.tsx b/packages/shade-widgets/src/components/ServerStatus.tsx index 27129c6..3854a42 100644 --- a/packages/shade-widgets/src/components/ServerStatus.tsx +++ b/packages/shade-widgets/src/components/ServerStatus.tsx @@ -31,7 +31,7 @@ export function ServerStatus(): React.ReactElement { label="Rate limited" value={stats.totalRateLimited} theme={theme} - color={stats.totalRateLimited > 0 ? theme.danger : undefined} + {...(stats.totalRateLimited > 0 ? { color: theme.danger } : {})} /> )} diff --git a/packages/shade-widgets/src/components/shared.tsx b/packages/shade-widgets/src/components/shared.tsx index 2af20e7..ca13de2 100644 --- a/packages/shade-widgets/src/components/shared.tsx +++ b/packages/shade-widgets/src/components/shared.tsx @@ -1,6 +1,5 @@ import React from 'react'; import { useShadeContext } from '../ShadeProvider.js'; -import type { ShadeTheme } from '../theme.js'; /** * Common widget shell — provides consistent border, padding, header. diff --git a/packages/shade-widgets/src/components/transfer/DropZone.tsx b/packages/shade-widgets/src/components/transfer/DropZone.tsx new file mode 100644 index 0000000..0cc990f --- /dev/null +++ b/packages/shade-widgets/src/components/transfer/DropZone.tsx @@ -0,0 +1,115 @@ +import React, { useCallback, useState } from 'react'; +import { useShadeRuntimeTheme } from '../../ShadeRuntimeProvider.js'; + +export interface DropZoneProps { + /** Called when files are dropped or selected via the click-to-pick action. */ + onFiles: (files: File[]) => void; + /** Allow multiple files (default: true). */ + multiple?: boolean; + /** `` accept filter (e.g. `'.zip,application/zip'`). */ + accept?: string; + className?: string; + style?: React.CSSProperties; + children?: React.ReactNode; +} + +/** + * Drag-and-drop file zone built on native HTML5 DnD APIs. Falls back to a + * native file picker when the user clicks or focuses-and-presses-Enter. + */ +export function DropZone({ + onFiles, + multiple = true, + accept, + className, + style, + children, +}: DropZoneProps): React.ReactElement { + const theme = useShadeRuntimeTheme(); + const [isDragging, setIsDragging] = useState(false); + const inputRef = React.useRef(null); + + const onDragOver = useCallback((e: React.DragEvent) => { + e.preventDefault(); + e.dataTransfer.dropEffect = 'copy'; + setIsDragging(true); + }, []); + + const onDragLeave = useCallback(() => { + setIsDragging(false); + }, []); + + const onDrop = useCallback( + (e: React.DragEvent) => { + e.preventDefault(); + setIsDragging(false); + const files = Array.from(e.dataTransfer.files); + if (files.length > 0) onFiles(files); + }, + [onFiles], + ); + + const onChange = useCallback( + (e: React.ChangeEvent) => { + const files = e.target.files === null ? [] : Array.from(e.target.files); + if (files.length > 0) onFiles(files); + e.target.value = ''; // allow re-picking the same file + }, + [onFiles], + ); + + const onClick = useCallback(() => { + inputRef.current?.click(); + }, []); + + const onKeyDown = useCallback( + (e: React.KeyboardEvent) => { + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault(); + inputRef.current?.click(); + } + }, + [], + ); + + const borderActive = theme.dropZoneBorderActive ?? theme.accent; + + return ( +
+ + {children ?? ( + + {isDragging ? 'Slipp filer for å laste opp' : 'Dra og slipp filer her, eller klikk for å velge'} + + )} +
+ ); +} diff --git a/packages/shade-widgets/src/components/transfer/ETAReadout.tsx b/packages/shade-widgets/src/components/transfer/ETAReadout.tsx new file mode 100644 index 0000000..3ffe6bb --- /dev/null +++ b/packages/shade-widgets/src/components/transfer/ETAReadout.tsx @@ -0,0 +1,42 @@ +import React from 'react'; +import { useShadeRuntimeTheme } from '../../ShadeRuntimeProvider.js'; + +export interface ETAReadoutProps { + /** ETA in seconds. Undefined → unknown. */ + seconds: number | undefined; + className?: string; + style?: React.CSSProperties; +} + +export function ETAReadout({ + seconds, + className, + style, +}: ETAReadoutProps): React.ReactElement { + const theme = useShadeRuntimeTheme(); + return ( + + {formatEta(seconds)} + + ); +} + +export function formatEta(seconds: number | undefined): string { + if (seconds === undefined || !Number.isFinite(seconds)) return 'ETA —'; + if (seconds < 1) return 'ETA <1s'; + if (seconds < 60) return `ETA ${Math.round(seconds)}s`; + const minutes = Math.floor(seconds / 60); + const remSeconds = Math.round(seconds % 60); + if (minutes < 60) return `ETA ${minutes}m ${remSeconds}s`; + const hours = Math.floor(minutes / 60); + const remMinutes = minutes % 60; + return `ETA ${hours}h ${remMinutes}m`; +} diff --git a/packages/shade-widgets/src/components/transfer/LaneIndicator.tsx b/packages/shade-widgets/src/components/transfer/LaneIndicator.tsx new file mode 100644 index 0000000..dba2e15 --- /dev/null +++ b/packages/shade-widgets/src/components/transfer/LaneIndicator.tsx @@ -0,0 +1,50 @@ +import React from 'react'; +import type { LaneProgress } from '@shade/sdk'; +import { useShadeRuntimeTheme } from '../../ShadeRuntimeProvider.js'; + +export interface LaneIndicatorProps { + lanes: LaneProgress[]; + className?: string; + style?: React.CSSProperties; +} + +export function LaneIndicator({ + lanes, + className, + style, +}: LaneIndicatorProps): React.ReactElement { + const theme = useShadeRuntimeTheme(); + const active = theme.laneActive ?? theme.success; + const idle = theme.laneIdle ?? theme.textDim; + const error = theme.laneError ?? theme.danger; + + return ( +
+ {lanes.map((l) => { + let color: string; + if (l.state === 'error') color = error; + else if (l.state === 'sending' || l.state === 'done') color = active; + else color = idle; + return ( + + ); + })} +
+ ); +} diff --git a/packages/shade-widgets/src/components/transfer/ProgressBar.tsx b/packages/shade-widgets/src/components/transfer/ProgressBar.tsx new file mode 100644 index 0000000..e7889f2 --- /dev/null +++ b/packages/shade-widgets/src/components/transfer/ProgressBar.tsx @@ -0,0 +1,51 @@ +import React from 'react'; +import { useShadeRuntimeTheme } from '../../ShadeRuntimeProvider.js'; + +export interface ProgressBarProps { + /** Progress in [0, 1]. Undefined → indeterminate. */ + percent?: number; + height?: number; + className?: string; + style?: React.CSSProperties; +} + +export function ProgressBar({ + percent, + height = 6, + className, + style, +}: ProgressBarProps): React.ReactElement { + const theme = useShadeRuntimeTheme(); + const track = theme.progressTrack ?? theme.border; + const fill = theme.progressFill ?? theme.accent; + const indeterminate = theme.progressFillIndeterminate ?? theme.accentMuted; + const isIndeterminate = percent === undefined; + const widthPct = isIndeterminate ? 100 : Math.max(0, Math.min(1, percent)) * 100; + return ( +
+
+
+ ); +} diff --git a/packages/shade-widgets/src/components/transfer/ShadeDownloader.tsx b/packages/shade-widgets/src/components/transfer/ShadeDownloader.tsx new file mode 100644 index 0000000..b1b5372 --- /dev/null +++ b/packages/shade-widgets/src/components/transfer/ShadeDownloader.tsx @@ -0,0 +1,206 @@ +import React from 'react'; +import type { + IncomingTransfer, + TransferHandle, + TransferOutput, + TransferProgress, +} from '@shade/sdk'; +import { useShadeRuntimeTheme } from '../../ShadeRuntimeProvider.js'; +import { + useShadeDownload, + type DownloadEntry, + type UseShadeDownloadOptions, +} from '../../useShadeDownload.js'; +import { TransferRow } from './TransferRow.js'; + +export interface ShadeDownloaderProps { + /** Auto-accept incoming transfers via this callback (returns the output sink). */ + autoAccept?: UseShadeDownloadOptions['autoAccept']; + /** Render the prompt for a pending (not-yet-accepted) transfer. */ + renderPending?: (args: { + incoming: IncomingTransfer; + accept: (output: TransferOutput) => Promise; + decline: (reason?: string) => Promise; + }) => React.ReactNode; + /** Render-prop for active transfer rows. */ + renderRow?: (args: { + handle: TransferHandle; + progress: TransferProgress | null; + name: string; + done: boolean; + error: unknown; + onDismiss: () => void; + }) => React.ReactNode; + /** Render-prop for the empty state (no pending or active transfers). */ + renderEmpty?: () => React.ReactNode; + className?: string; + style?: React.CSSProperties; +} + +/** + * Composite downloader: lists pending (awaiting accept) and active + * (in-progress) transfers, with bundled defaults for both. Customize via + * `renderPending` / `renderRow`. + */ +export function ShadeDownloader({ + autoAccept, + renderPending, + renderRow, + renderEmpty, + className, + style, +}: ShadeDownloaderProps): React.ReactElement { + const theme = useShadeRuntimeTheme(); + const { pending, active, accept, decline, dismiss } = useShadeDownload( + autoAccept !== undefined ? { autoAccept } : {}, + ); + + return ( +
+ {pending.length === 0 && active.length === 0 ? ( + renderEmpty?.() ?? Ingen overføringer + ) : null} + + {pending.map((entry) => { + const acceptForOutput = (output: TransferOutput): Promise => + accept(entry.incoming.streamId, output); + const declineWith = (reason?: string): Promise => + decline(entry.incoming.streamId, reason); + if (renderPending !== undefined) { + return ( + + {renderPending({ incoming: entry.incoming, accept: acceptForOutput, decline: declineWith })} + + ); + } + return ( + void declineWith('user-decline')} + /> + ); + })} + + {active.map((entry: DownloadEntry) => { + if (entry.handle === null) return null; + const name = entry.incoming.metadata.name ?? entry.incoming.streamId.slice(0, 8); + const dismissEntry = (): void => dismiss(entry.incoming.streamId); + const handle = entry.handle; + if (renderRow !== undefined) { + return ( + + {renderRow({ + handle, + progress: entry.progress, + name, + done: entry.done, + error: entry.error, + onDismiss: dismissEntry, + })} + + ); + } + return ( + + ); + })} +
+ ); +} + +function DefaultPendingRow({ + incoming, + onAccept, + onDecline, +}: { + incoming: IncomingTransfer; + onAccept: (output: TransferOutput) => Promise; + onDecline: () => void; +}): React.ReactElement { + const theme = useShadeRuntimeTheme(); + return ( +
+
+ Innkommende overføring fra {incoming.from} +
+
+ {incoming.metadata.name ?? incoming.streamId} + {incoming.metadata.sizeBytes !== undefined + ? ` · ${formatBytes(incoming.metadata.sizeBytes)}` + : ''} +
+
+ + +
+
+ ); +} + +function formatBytes(b: number): string { + if (b < 1024) return `${b} B`; + if (b < 1024 * 1024) return `${(b / 1024).toFixed(1)} KB`; + if (b < 1024 * 1024 * 1024) return `${(b / (1024 * 1024)).toFixed(1)} MB`; + return `${(b / (1024 * 1024 * 1024)).toFixed(1)} GB`; +} diff --git a/packages/shade-widgets/src/components/transfer/ShadeUploader.tsx b/packages/shade-widgets/src/components/transfer/ShadeUploader.tsx new file mode 100644 index 0000000..ded02e2 --- /dev/null +++ b/packages/shade-widgets/src/components/transfer/ShadeUploader.tsx @@ -0,0 +1,171 @@ +import React from 'react'; +import type { + TransferHandle, + TransferOptions, + TransferProgress, + TransferResult, +} from '@shade/sdk'; +import { useShadeRuntimeTheme } from '../../ShadeRuntimeProvider.js'; +import { useShadeUpload, type UploadEntry } from '../../useShadeUpload.js'; +import { DropZone } from './DropZone.js'; +import { TransferRow } from './TransferRow.js'; + +export interface ShadeUploaderProps { + /** Peer address to upload to. */ + to: string; + multiple?: boolean; + accept?: string; + /** Override the default lane count (default 4 from `@shade/transfer`). */ + lanes?: number; + /** Override per-chunk plaintext size. */ + chunkSize?: number; + /** Extra options forwarded to `Shade.upload`. */ + uploadOptions?: Partial; + /** + * Render-prop for full UI replacement of the per-row entry. Returns whatever + * you want to render in place of the default `TransferRow`. + */ + renderRow?: (args: { + handle: TransferHandle; + progress: TransferProgress | null; + name: string; + done: boolean; + error: unknown; + onDismiss: () => void; + }) => React.ReactNode; + /** Render-prop for the empty state (no uploads in progress). */ + renderEmpty?: () => React.ReactNode; + /** Render-prop for the drop zone. */ + renderDropZone?: (props: { onFiles: (files: File[]) => void }) => React.ReactNode; + onComplete?: (result: TransferResult) => void; + onError?: (error: unknown) => void; + className?: string; + style?: React.CSSProperties; +} + +/** + * Composite uploader: a drop zone + a list of in-flight transfers. + * + * Drop a file (or click to pick), and Shade handles encryption + chunking + + * lanes + retry + integrity verification. Customize via the `renderRow` + * render-prop or use the headless `useShadeUpload` hook directly. + */ +export function ShadeUploader({ + to, + multiple = true, + accept, + lanes, + chunkSize, + uploadOptions, + renderRow, + renderEmpty, + renderDropZone, + onComplete, + onError, + className, + style, +}: ShadeUploaderProps): React.ReactElement { + const theme = useShadeRuntimeTheme(); + const { upload, uploads, dismiss } = useShadeUpload(); + + const handleFiles = React.useCallback( + async (files: File[]) => { + for (const file of files) { + try { + const handle = await upload({ + to, + input: file, + ...(lanes !== undefined ? { lanes } : {}), + ...(chunkSize !== undefined ? { chunkSize } : {}), + ...uploadOptions, + metadata: { + name: file.name, + ...(file.type !== '' ? { contentType: file.type } : {}), + ...uploadOptions?.metadata, + }, + }); + if (onComplete !== undefined) { + void handle + .done() + .then(onComplete) + .catch(() => { + /* error surfaced via onError */ + }); + } + if (onError !== undefined) { + void handle.done().catch(onError); + } + } catch (err) { + onError?.(err); + } + } + }, + [upload, to, lanes, chunkSize, uploadOptions, onComplete, onError], + ); + + return ( +
+ {renderDropZone !== undefined ? ( + renderDropZone({ onFiles: handleFiles }) + ) : ( + + )} + {uploads.length === 0 ? ( + renderEmpty?.() ?? null + ) : ( +
+ {uploads.map((entry: UploadEntry) => { + const name = describeEntry(entry); + const dismissEntry = (): void => dismiss(entry.handle.streamId); + const cancelEntry = (): void => { + void entry.handle.abort('user-cancel'); + }; + if (renderRow !== undefined) { + return ( + + {renderRow({ + handle: entry.handle, + progress: entry.progress, + name, + done: entry.done, + error: entry.error, + onDismiss: dismissEntry, + })} + + ); + } + return ( + + ); + })} +
+ )} +
+ ); +} + +function describeEntry(entry: UploadEntry): string { + // The progress event includes bytesTotal but not the file name; we keep + // a hint by reading the underlying stream metadata when available. + return entry.handle.streamId.slice(0, 8); +} diff --git a/packages/shade-widgets/src/components/transfer/SpeedReadout.tsx b/packages/shade-widgets/src/components/transfer/SpeedReadout.tsx new file mode 100644 index 0000000..db19432 --- /dev/null +++ b/packages/shade-widgets/src/components/transfer/SpeedReadout.tsx @@ -0,0 +1,37 @@ +import React from 'react'; +import { useShadeRuntimeTheme } from '../../ShadeRuntimeProvider.js'; + +export interface SpeedReadoutProps { + bytesPerSecond: number; + className?: string; + style?: React.CSSProperties; +} + +export function SpeedReadout({ + bytesPerSecond, + className, + style, +}: SpeedReadoutProps): React.ReactElement { + const theme = useShadeRuntimeTheme(); + return ( + + {formatBytesPerSecond(bytesPerSecond)} + + ); +} + +export function formatBytesPerSecond(bps: number): string { + if (!Number.isFinite(bps) || bps <= 0) return '— B/s'; + if (bps < 1024) return `${bps.toFixed(0)} B/s`; + if (bps < 1024 * 1024) return `${(bps / 1024).toFixed(1)} KB/s`; + if (bps < 1024 * 1024 * 1024) return `${(bps / (1024 * 1024)).toFixed(2)} MB/s`; + return `${(bps / (1024 * 1024 * 1024)).toFixed(2)} GB/s`; +} diff --git a/packages/shade-widgets/src/components/transfer/TransferRow.tsx b/packages/shade-widgets/src/components/transfer/TransferRow.tsx new file mode 100644 index 0000000..d867706 --- /dev/null +++ b/packages/shade-widgets/src/components/transfer/TransferRow.tsx @@ -0,0 +1,116 @@ +import React from 'react'; +import type { TransferHandle, TransferProgress } from '@shade/sdk'; +import { useShadeRuntimeTheme } from '../../ShadeRuntimeProvider.js'; +import { ProgressBar } from './ProgressBar.js'; +import { SpeedReadout } from './SpeedReadout.js'; +import { ETAReadout } from './ETAReadout.js'; +import { LaneIndicator } from './LaneIndicator.js'; + +export interface TransferRowProps { + handle: TransferHandle; + progress: TransferProgress | null; + /** Optional file name to display. */ + name?: string; + /** Total bytes for this transfer (used to render percent when progress lacks it). */ + bytesTotal?: number; + done?: boolean; + error?: unknown; + onCancel?: () => void; + onDismiss?: () => void; + className?: string; + style?: React.CSSProperties; +} + +/** Default UI for a single transfer. Replace via render-prop on the parent. */ +export function TransferRow({ + handle, + progress, + name, + bytesTotal, + done, + error, + onCancel, + onDismiss, + className, + style, +}: TransferRowProps): React.ReactElement { + const theme = useShadeRuntimeTheme(); + const percent = progress?.percent ?? (bytesTotal !== undefined && progress !== null ? progress.bytesSent / bytesTotal : undefined); + const errorMessage = + error instanceof Error ? error.message : error !== undefined ? String(error) : null; + + return ( +
+
+ + {name ?? handle.streamId} + + {progress !== null ? ( + + ) : null} + {!done && onCancel !== undefined ? ( + + ) : null} + {done && onDismiss !== undefined ? ( + + ) : null} +
+ +
+ + +
+ {errorMessage !== null ? ( +
{errorMessage}
+ ) : null} +
+ ); +} + +function smallButton(fg: string, bg: string): React.CSSProperties { + return { + padding: '2px 8px', + background: bg, + color: fg, + border: `1px solid ${fg}`, + borderRadius: 4, + fontSize: 11, + cursor: 'pointer', + }; +} diff --git a/packages/shade-widgets/src/index.ts b/packages/shade-widgets/src/index.ts index e2f59d6..5319533 100644 --- a/packages/shade-widgets/src/index.ts +++ b/packages/shade-widgets/src/index.ts @@ -16,3 +16,38 @@ export type { ShadeState, UseShadeStateResult } from './useShadeState.js'; export type { ShadeEventEnvelope, UseShadeEventsResult } from './useShadeEvents.js'; export type { ShadeTheme, ThemeMode } from './theme.js'; export type { WidgetCatalogProps, WidgetKey } from './components/WidgetCatalog.js'; + +// ─── Stream-transfer widgets (v0.2.0) ──────────────────── +export { + ShadeRuntimeProvider, + useShadeRuntime, + useShadeRuntimeTheme, +} from './ShadeRuntimeProvider.js'; +export type { + ShadeRuntimeProviderProps, + ShadeRuntimeContextValue, +} from './ShadeRuntimeProvider.js'; +export { useShadeUpload } from './useShadeUpload.js'; +export type { UploadEntry, UseShadeUploadResult } from './useShadeUpload.js'; +export { useShadeDownload } from './useShadeDownload.js'; +export type { + DownloadEntry, + UseShadeDownloadOptions, + UseShadeDownloadResult, +} from './useShadeDownload.js'; +export { ShadeUploader } from './components/transfer/ShadeUploader.js'; +export type { ShadeUploaderProps } from './components/transfer/ShadeUploader.js'; +export { ShadeDownloader } from './components/transfer/ShadeDownloader.js'; +export type { ShadeDownloaderProps } from './components/transfer/ShadeDownloader.js'; +export { DropZone } from './components/transfer/DropZone.js'; +export type { DropZoneProps } from './components/transfer/DropZone.js'; +export { TransferRow } from './components/transfer/TransferRow.js'; +export type { TransferRowProps } from './components/transfer/TransferRow.js'; +export { ProgressBar } from './components/transfer/ProgressBar.js'; +export type { ProgressBarProps } from './components/transfer/ProgressBar.js'; +export { SpeedReadout, formatBytesPerSecond } from './components/transfer/SpeedReadout.js'; +export type { SpeedReadoutProps } from './components/transfer/SpeedReadout.js'; +export { ETAReadout, formatEta } from './components/transfer/ETAReadout.js'; +export type { ETAReadoutProps } from './components/transfer/ETAReadout.js'; +export { LaneIndicator } from './components/transfer/LaneIndicator.js'; +export type { LaneIndicatorProps } from './components/transfer/LaneIndicator.js'; diff --git a/packages/shade-widgets/src/theme.ts b/packages/shade-widgets/src/theme.ts index fe4dcca..308a31e 100644 --- a/packages/shade-widgets/src/theme.ts +++ b/packages/shade-widgets/src/theme.ts @@ -21,6 +21,22 @@ export interface ShadeTheme { fontMono: string; radius: string; shadow: string; + + // ─── Stream-transfer tokens (v0.2.0) ───────────────────── + /** Background of progress-bar track. Default: `border`. */ + progressTrack?: string; + /** Filled portion of the progress bar. Default: `accent`. */ + progressFill?: string; + /** Animated fill for indeterminate progress. Default: `accentMuted`. */ + progressFillIndeterminate?: string; + /** Border of an active drag-and-drop zone. Default: `accent`. */ + dropZoneBorderActive?: string; + /** Lane indicator color when sending. Default: `success`. */ + laneActive?: string; + /** Lane indicator color when idle. Default: `textDim`. */ + laneIdle?: string; + /** Lane indicator color on error. Default: `danger`. */ + laneError?: string; } export const darkTheme: ShadeTheme = { diff --git a/packages/shade-widgets/src/useShadeDownload.ts b/packages/shade-widgets/src/useShadeDownload.ts new file mode 100644 index 0000000..2e85b5f --- /dev/null +++ b/packages/shade-widgets/src/useShadeDownload.ts @@ -0,0 +1,161 @@ +import { useCallback, useEffect, useState } from 'react'; +import type { + IncomingTransfer, + TransferHandle, + TransferOutput, + TransferProgress, +} from '@shade/sdk'; +import { useShadeRuntime } from './ShadeRuntimeProvider.js'; + +export interface DownloadEntry { + incoming: IncomingTransfer; + handle: TransferHandle | null; + progress: TransferProgress | null; + error: unknown; + done: boolean; +} + +export interface UseShadeDownloadOptions { + /** + * If provided, accept transfers automatically with the supplied output + * (called once per incoming). Useful for "auto-save to /uploads" servers. + */ + autoAccept?: (incoming: IncomingTransfer) => Promise; +} + +export interface UseShadeDownloadResult { + /** Pending transfers awaiting accept/decline. */ + pending: DownloadEntry[]; + /** In-flight transfers being received. */ + active: DownloadEntry[]; + /** Manually accept a pending transfer with a chosen output. */ + accept: (streamId: string, output: TransferOutput) => Promise; + /** Manually decline a pending transfer. */ + decline: (streamId: string, reason?: string) => Promise; + dismiss: (streamId: string) => void; +} + +/** + * Headless hook for download tracking. Subscribes to incoming transfers + * and exposes a list of pending and active downloads. + */ +export function useShadeDownload( + options: UseShadeDownloadOptions = {}, +): UseShadeDownloadResult { + const shade = useShadeRuntime(); + const [pending, setPending] = useState([]); + const [active, setActive] = useState([]); + + const trackHandle = useCallback( + (incoming: IncomingTransfer, handle: TransferHandle) => { + const entry: DownloadEntry = { + incoming, + handle, + progress: null, + error: null, + done: false, + }; + setPending((prev) => prev.filter((p) => p.incoming.streamId !== incoming.streamId)); + setActive((prev) => [entry, ...prev]); + + void (async () => { + try { + for await (const ev of handle.events) { + if (ev.type === 'progress') { + setActive((prev) => + prev.map((a) => + a.incoming.streamId === incoming.streamId ? { ...a, progress: ev.progress } : a, + ), + ); + } else if (ev.type === 'complete') { + setActive((prev) => + prev.map((a) => + a.incoming.streamId === incoming.streamId ? { ...a, done: true } : a, + ), + ); + } else if (ev.type === 'error') { + setActive((prev) => + prev.map((a) => + a.incoming.streamId === incoming.streamId + ? { ...a, error: ev.error, done: true } + : a, + ), + ); + } + } + } catch (err) { + setActive((prev) => + prev.map((a) => + a.incoming.streamId === incoming.streamId + ? { ...a, error: err, done: true } + : a, + ), + ); + } + })(); + }, + [], + ); + + useEffect(() => { + let unsubscribe: (() => void) | null = null; + let cancelled = false; + void (async () => { + const cleanup = await shade.onIncomingTransfer(async (incoming) => { + if (options.autoAccept !== undefined) { + const output = await options.autoAccept(incoming); + if (output !== null) { + const handle = await incoming.accept({ output }); + trackHandle(incoming, handle); + return; + } + } + const entry: DownloadEntry = { + incoming, + handle: null, + progress: null, + error: null, + done: false, + }; + setPending((prev) => [entry, ...prev]); + }); + if (cancelled) cleanup(); + else unsubscribe = cleanup; + })(); + return () => { + cancelled = true; + unsubscribe?.(); + }; + }, [shade, options.autoAccept, trackHandle]); + + const accept = useCallback( + async (streamId: string, output: TransferOutput): Promise => { + const entry = pending.find((p) => p.incoming.streamId === streamId); + if (entry === undefined) { + throw new Error(`No pending transfer with streamId=${streamId}`); + } + const handle = await entry.incoming.accept({ output }); + trackHandle(entry.incoming, handle); + return handle; + }, + [pending, trackHandle], + ); + + const decline = useCallback( + async (streamId: string, reason?: string) => { + const entry = pending.find((p) => p.incoming.streamId === streamId); + if (entry !== undefined) { + await entry.incoming.decline(reason); + setPending((prev) => prev.filter((p) => p.incoming.streamId !== streamId)); + } + }, + [pending], + ); + + const dismiss = useCallback((streamId: string) => { + setPending((prev) => prev.filter((p) => p.incoming.streamId !== streamId)); + setActive((prev) => prev.filter((a) => a.incoming.streamId !== streamId)); + }, []); + + return { pending, active, accept, decline, dismiss }; +} diff --git a/packages/shade-widgets/src/useShadeUpload.ts b/packages/shade-widgets/src/useShadeUpload.ts new file mode 100644 index 0000000..db6f9b5 --- /dev/null +++ b/packages/shade-widgets/src/useShadeUpload.ts @@ -0,0 +1,90 @@ +import { useCallback, useEffect, useRef, useState } from 'react'; +import type { TransferHandle, TransferOptions, TransferProgress } from '@shade/sdk'; +import { useShadeRuntime } from './ShadeRuntimeProvider.js'; + +export interface UploadEntry { + handle: TransferHandle; + progress: TransferProgress | null; + error: unknown; + done: boolean; +} + +export interface UseShadeUploadResult { + /** Start a new upload. Resolves once the handle is available. */ + upload: (opts: TransferOptions) => Promise; + /** Currently-tracked uploads (most recent first). */ + uploads: UploadEntry[]; + /** Drop a finished/failed entry from the list. */ + dismiss: (streamId: string) => void; +} + +/** + * Headless hook for upload tracking. Subscribes to progress events on each + * started transfer and surfaces them as React state. Drop a custom UI on + * top, or use the bundled `` component. + */ +export function useShadeUpload(): UseShadeUploadResult { + const shade = useShadeRuntime(); + const [uploads, setUploads] = useState([]); + const mountedRef = useRef(true); + + useEffect(() => { + mountedRef.current = true; + return () => { + mountedRef.current = false; + }; + }, []); + + const upload = useCallback( + async (opts: TransferOptions) => { + const handle = await shade.upload(opts); + const entry: UploadEntry = { handle, progress: null, error: null, done: false }; + setUploads((prev) => [entry, ...prev]); + + void (async () => { + try { + for await (const ev of handle.events) { + if (!mountedRef.current) return; + if (ev.type === 'progress') { + setUploads((prev) => + prev.map((u) => + u.handle.streamId === handle.streamId ? { ...u, progress: ev.progress } : u, + ), + ); + } else if (ev.type === 'complete') { + setUploads((prev) => + prev.map((u) => + u.handle.streamId === handle.streamId ? { ...u, done: true } : u, + ), + ); + } else if (ev.type === 'error') { + setUploads((prev) => + prev.map((u) => + u.handle.streamId === handle.streamId + ? { ...u, error: ev.error, done: true } + : u, + ), + ); + } + } + } catch (err) { + if (!mountedRef.current) return; + setUploads((prev) => + prev.map((u) => + u.handle.streamId === handle.streamId ? { ...u, error: err, done: true } : u, + ), + ); + } + })(); + + return handle; + }, + [shade], + ); + + const dismiss = useCallback((streamId: string) => { + setUploads((prev) => prev.filter((u) => u.handle.streamId !== streamId)); + }, []); + + return { upload, uploads, dismiss }; +} diff --git a/packages/shade-widgets/tests/transfer-formatters.test.ts b/packages/shade-widgets/tests/transfer-formatters.test.ts new file mode 100644 index 0000000..3796616 --- /dev/null +++ b/packages/shade-widgets/tests/transfer-formatters.test.ts @@ -0,0 +1,58 @@ +import { describe, test, expect } from 'bun:test'; +import { formatBytesPerSecond } from '../src/components/transfer/SpeedReadout.js'; +import { formatEta } from '../src/components/transfer/ETAReadout.js'; + +describe('formatBytesPerSecond', () => { + test('renders sub-1KB rates in B/s', () => { + expect(formatBytesPerSecond(1)).toBe('1 B/s'); + expect(formatBytesPerSecond(512)).toBe('512 B/s'); + }); + + test('renders KB/s', () => { + expect(formatBytesPerSecond(1024)).toBe('1.0 KB/s'); + expect(formatBytesPerSecond(1536)).toBe('1.5 KB/s'); + }); + + test('renders MB/s', () => { + expect(formatBytesPerSecond(1024 * 1024)).toBe('1.00 MB/s'); + expect(formatBytesPerSecond(2.5 * 1024 * 1024)).toBe('2.50 MB/s'); + }); + + test('renders GB/s', () => { + expect(formatBytesPerSecond(1024 * 1024 * 1024)).toBe('1.00 GB/s'); + }); + + test('handles zero / non-finite as em-dash', () => { + expect(formatBytesPerSecond(0)).toBe('— B/s'); + expect(formatBytesPerSecond(-1)).toBe('— B/s'); + expect(formatBytesPerSecond(Number.NaN)).toBe('— B/s'); + }); +}); + +describe('formatEta', () => { + test('undefined → em-dash', () => { + expect(formatEta(undefined)).toBe('ETA —'); + }); + + test('NaN/Infinity → em-dash', () => { + expect(formatEta(Number.NaN)).toBe('ETA —'); + expect(formatEta(Number.POSITIVE_INFINITY)).toBe('ETA —'); + }); + + test('sub-second', () => { + expect(formatEta(0.4)).toBe('ETA <1s'); + }); + + test('seconds', () => { + expect(formatEta(45)).toBe('ETA 45s'); + }); + + test('minutes + seconds', () => { + expect(formatEta(90)).toBe('ETA 1m 30s'); + expect(formatEta(125)).toBe('ETA 2m 5s'); + }); + + test('hours + minutes', () => { + expect(formatEta(3600 + 30 * 60)).toBe('ETA 1h 30m'); + }); +}); diff --git a/scripts/Deprecated/publish-all.ts b/scripts/Deprecated/publish-all.ts new file mode 100644 index 0000000..33839d1 --- /dev/null +++ b/scripts/Deprecated/publish-all.ts @@ -0,0 +1,148 @@ +#!/usr/bin/env bun +/** + * Publish all @shade/* packages to the Gitea npm registry. + * + * Expects these env vars: + * GITEA_TOKEN — publish token from Gitea (Settings → Applications) + * GITEA_USER — Gitea username that owns the registry (e.g. "Stian") + * + * Optional: + * DRY_RUN=1 — build tarballs but don't publish + * + * Usage: + * bun run scripts/publish-all.ts + */ +import { readFileSync, writeFileSync, existsSync } from 'fs'; +import { join } from 'path'; +import { $ } from 'bun'; + +const PACKAGES = [ + 'shade-core', + 'shade-crypto-web', + 'shade-proto', + 'shade-storage-sqlite', + 'shade-storage-postgres', + 'shade-server', + 'shade-observer', + 'shade-transport', + 'shade-widgets', + 'shade-sdk', + 'shade-cli', +]; + +const REGISTRY_HOST = 'gt.zyon.no'; +const ROOT = join(import.meta.dir, '..'); + +async function main() { + const token = process.env.GITEA_TOKEN; + const user = process.env.GITEA_USER ?? 'Stian'; + const dryRun = process.env.DRY_RUN === '1'; + + if (!token && !dryRun) { + console.error('GITEA_TOKEN is required (or set DRY_RUN=1)'); + process.exit(1); + } + + const registryUrl = `https://${REGISTRY_HOST}/api/packages/${user}/npm/`; + console.log(`Target registry: ${registryUrl}`); + console.log(`Dry run: ${dryRun ? 'yes' : 'no'}`); + console.log(); + + // Write a temporary .npmrc at the root + const npmrcPath = join(ROOT, '.npmrc.publish'); + const npmrc = [ + `@shade:registry=${registryUrl}`, + dryRun ? '' : `//${REGISTRY_HOST}/api/packages/${user}/npm/:_authToken=${token}`, + ].filter(Boolean).join('\n'); + writeFileSync(npmrcPath, npmrc); + + // Build a name → version map across all workspace packages so we can rewrite + // `workspace:*` (and friends) into concrete `^` specifiers before + // publishing. Without this, the registry stores the literal `workspace:*` + // string in published package.json, which then fails to resolve in any + // consumer (e.g. Dispatch) outside the Shade monorepo. + const versionByName = new Map(); + for (const pkg of PACKAGES) { + const pkgDir = join(ROOT, 'packages', pkg); + if (!existsSync(join(pkgDir, 'package.json'))) continue; + const json = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf-8')); + versionByName.set(json.name, json.version); + } + + let published = 0; + let skipped = 0; + + for (const pkg of PACKAGES) { + const pkgDir = join(ROOT, 'packages', pkg); + const pkgJsonPath = join(pkgDir, 'package.json'); + if (!existsSync(pkgJsonPath)) { + console.log(`⊘ ${pkg} — package.json not found, skipping`); + skipped++; + continue; + } + + const originalPkgJson = readFileSync(pkgJsonPath, 'utf-8'); + const pkgJson = JSON.parse(originalPkgJson); + console.log(`→ ${pkgJson.name}@${pkgJson.version}`); + + rewriteWorkspaceSpecs(pkgJson, versionByName); + writeFileSync(pkgJsonPath, `${JSON.stringify(pkgJson, null, 2)}\n`); + + try { + if (dryRun) { + await $`cd ${pkgDir} && bun pm pack --dry-run`.quiet(); + } else { + await $`cd ${pkgDir} && npm publish --registry=${registryUrl} --userconfig ${npmrcPath}`.quiet(); + } + published++; + console.log(` ✓ ${dryRun ? 'packed' : 'published'}`); + } catch (err) { + console.error(` ✗ failed: ${(err as Error).message}`); + process.exitCode = 1; + } finally { + // Always restore the original package.json so the workspace stays usable + // for `bun install` after publish, regardless of success or failure. + writeFileSync(pkgJsonPath, originalPkgJson); + } + } + + // Clean up temp npmrc + try { + await $`rm ${npmrcPath}`.quiet(); + } catch {} + + console.log(); + console.log(`Done: ${published} published, ${skipped} skipped`); +} + +/** + * Rewrite `workspace:*` (and `workspace:^`, `workspace:~`, `workspace:`) + * specifiers in dependency sections to concrete `^` specifiers. + * Mutates the passed-in object. + */ +function rewriteWorkspaceSpecs( + pkgJson: Record, + versionByName: Map, +): void { + const sections = ['dependencies', 'peerDependencies', 'optionalDependencies'] as const; + for (const section of sections) { + const deps = pkgJson[section]; + if (!deps || typeof deps !== 'object') continue; + for (const [name, spec] of Object.entries(deps as Record)) { + if (typeof spec !== 'string' || !spec.startsWith('workspace:')) continue; + const version = versionByName.get(name); + if (!version) { + throw new Error( + `No workspace version known for ${name} (referenced from ${pkgJson.name as string}). ` + + `Add it to PACKAGES or remove the workspace dependency.`, + ); + } + (deps as Record)[name] = `^${version}`; + } + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/scripts/publish-all.ts b/scripts/publish-all.ts index 3cf7a05..032d398 100644 --- a/scripts/publish-all.ts +++ b/scripts/publish-all.ts @@ -16,17 +16,23 @@ import { readFileSync, writeFileSync, existsSync } from 'fs'; import { join } from 'path'; import { $ } from 'bun'; +// Order matters: each package only depends on packages above it. Publishing +// in this order means a consumer fetching mid-publish never sees a manifest +// pointing at an unpublished version. const PACKAGES = [ 'shade-core', - 'shade-crypto-web', 'shade-proto', + 'shade-crypto-web', 'shade-storage-sqlite', 'shade-storage-postgres', - 'shade-server', - 'shade-observer', + 'shade-streams', 'shade-transport', - 'shade-widgets', + 'shade-server', + 'shade-transfer', + 'shade-files', + 'shade-observer', 'shade-sdk', + 'shade-widgets', 'shade-cli', ]; @@ -56,20 +62,40 @@ async function main() { ].filter(Boolean).join('\n'); writeFileSync(npmrcPath, npmrc); + // Build a name → version map across all workspace packages so we can rewrite + // `workspace:*` (and friends) into concrete `^` specifiers before + // publishing. Without this, the registry stores the literal `workspace:*` + // string in published package.json, which then fails to resolve in any + // consumer (e.g. Dispatch) outside the Shade monorepo. + const versionByName = new Map(); + for (const pkg of PACKAGES) { + const pkgDir = join(ROOT, 'packages', pkg); + if (!existsSync(join(pkgDir, 'package.json'))) continue; + const json = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf-8')); + versionByName.set(json.name, json.version); + } + let published = 0; let skipped = 0; + let alreadyPublished = 0; + let failed = 0; for (const pkg of PACKAGES) { const pkgDir = join(ROOT, 'packages', pkg); - if (!existsSync(join(pkgDir, 'package.json'))) { + const pkgJsonPath = join(pkgDir, 'package.json'); + if (!existsSync(pkgJsonPath)) { console.log(`⊘ ${pkg} — package.json not found, skipping`); skipped++; continue; } - const pkgJson = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf-8')); + const originalPkgJson = readFileSync(pkgJsonPath, 'utf-8'); + const pkgJson = JSON.parse(originalPkgJson); console.log(`→ ${pkgJson.name}@${pkgJson.version}`); + rewriteWorkspaceSpecs(pkgJson, versionByName); + writeFileSync(pkgJsonPath, `${JSON.stringify(pkgJson, null, 2)}\n`); + try { if (dryRun) { await $`cd ${pkgDir} && bun pm pack --dry-run`.quiet(); @@ -79,8 +105,24 @@ async function main() { published++; console.log(` ✓ ${dryRun ? 'packed' : 'published'}`); } catch (err) { - console.error(` ✗ failed: ${(err as Error).message}`); - process.exitCode = 1; + const out = `${err instanceof Error ? err.message : String(err)} ${ + (err as { stderr?: { toString(): string } }).stderr?.toString() ?? '' + } ${(err as { stdout?: { toString(): string } }).stdout?.toString() ?? ''}`; + // Gitea (and npm) report already-published versions as 409 / EPUBLISHCONFLICT. + // Skip silently rather than failing the whole run — bumping the version + // is the user's explicit decision via `bun run version `. + if (/409|EPUBLISHCONFLICT|already exists|already been published/i.test(out)) { + alreadyPublished++; + console.log(` ⊙ already published — skipping`); + } else { + failed++; + console.error(` ✗ failed: ${(err as Error).message}`); + process.exitCode = 1; + } + } finally { + // Always restore the original package.json so the workspace stays usable + // for `bun install` after publish, regardless of success or failure. + writeFileSync(pkgJsonPath, originalPkgJson); } } @@ -90,7 +132,36 @@ async function main() { } catch {} console.log(); - console.log(`Done: ${published} published, ${skipped} skipped`); + console.log( + `Done: ${published} ${dryRun ? 'packed' : 'published'}, ${alreadyPublished} already published, ${skipped} skipped, ${failed} failed`, + ); +} + +/** + * Rewrite `workspace:*` (and `workspace:^`, `workspace:~`, `workspace:`) + * specifiers in dependency sections to concrete `^` specifiers. + * Mutates the passed-in object. + */ +function rewriteWorkspaceSpecs( + pkgJson: Record, + versionByName: Map, +): void { + const sections = ['dependencies', 'peerDependencies', 'optionalDependencies'] as const; + for (const section of sections) { + const deps = pkgJson[section]; + if (!deps || typeof deps !== 'object') continue; + for (const [name, spec] of Object.entries(deps as Record)) { + if (typeof spec !== 'string' || !spec.startsWith('workspace:')) continue; + const version = versionByName.get(name); + if (!version) { + throw new Error( + `No workspace version known for ${name} (referenced from ${pkgJson.name as string}). ` + + `Add it to PACKAGES or remove the workspace dependency.`, + ); + } + (deps as Record)[name] = `^${version}`; + } + } } main().catch((err) => { diff --git a/test-vectors/wire-format.json b/test-vectors/wire-format.json index cc04c5f..701a86e 100644 --- a/test-vectors/wire-format.json +++ b/test-vectors/wire-format.json @@ -1,7 +1,7 @@ { "vectors": [ { - "description": "Wire format: RatchetMessage encoding", + "description": "Wire format: RatchetMessage encoding (wire VERSION 0x02 — u32 length-prefixed)", "message": { "dhPublicKey": "1111111111111111111111111111111111111111111111111111111111111111", "previousCounter": 42, @@ -9,7 +9,7 @@ "ciphertext": "22222222222222222222222222222222", "nonce": "333333333333333333333333" }, - "encoded": "0102002011111111111111111111111111111111111111111111111111111111111111110000002a00000007001022222222222222222222222222222222000c333333333333333333333333" + "encoded": "02020000002011111111111111111111111111111111111111111111111111111111111111110000002a0000000700000010222222222222222222222222222222220000000c333333333333333333333333" } ] } diff --git a/tsconfig.json b/tsconfig.json index 2b921a4..17c3f56 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,6 +4,11 @@ "module": "ESNext", "moduleResolution": "bundler", "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitOverride": true, + "exactOptionalPropertyTypes": true, + "lib": ["ES2022"], "esModuleInterop": true, "skipLibCheck": true, "declaration": true,