Files
Shade/packages/shade-storage-sqlite/src/sqlite-storage.ts

614 lines
24 KiB
TypeScript
Raw Normal View History

import { Database } from 'bun:sqlite';
import type {
StorageProvider, IdentityKeyPair, SignedPreKey, OneTimePreKey,
SessionState, RetiredIdentity, PersistedStreamState, PeerVerification,
PeerVerificationSource, BroadcastChannelRecord, BroadcastMemberRecord,
} from '@shade/core';
import {
toBase64, fromBase64,
feat(hardening): M-Hard 1-5 — crypto, auth, rate limit, fingerprints, rotation M-Hard 1: Cryptographic Hardening - constantTimeEqual, zeroize, randomUint32 on CryptoProvider - Fix Math.random() → crypto.randomUint32() for registrationId - Zero message keys and chain keys after use in ratchet.ts - Constant-time trust comparison in MemoryStorage + SQLiteStorage - Timing variance test catches early-exit regressions M-Hard 2: Self-Authenticated Prekey Server - Ed25519 signature verification on all write routes - signPayload/verifyPayload with canonical JSON, ±5 min replay window - Address validation (NFKC, alphanumeric + :_-.) - Global ShadeError → HTTP status mapping - ShadeFetchTransport signs requests when signingPrivateKey provided - Anonymous bundle fetches still allowed (read-only) M-Hard 3: Rate Limiting + DoS Protection - Token bucket rate limiter with pluggable store - Per-route limits: register 5/h/IP, fetch 60/min/IP, replenish 10/min/id - 64KB body size limit on POST - Retry-After header on 429 responses M-Hard 4: Auto-replenish + Fingerprints + Session Reset - Safety numbers (12 groups × 5 digits, Signal-style) - ensurePreKeyStock, resetSession, acceptIdentityChange - verifyRemoteIdentity for out-of-band comparison M-Hard 5: Identity Rotation with Grace Period - rotateIdentity archives old identity, generates fresh signed prekey - RetiredIdentity storage with addRetired/getRetired/pruneRetired - 7-day default grace period for decrypting old sessions - pruneExpiredIdentities for cleanup M-Hard 8: Error Hierarchy - New error types: Network, Storage, Validation, Timeout, RateLimit, Configuration, Unauthorized, Replay, IdentityRotation - All errors have stable SHADE_* codes - errorToHttpStatus for consistent HTTP mapping - toJSON() for network serialization 188 tests passing, zero failures. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-10 17:45:34 +02:00
constantTimeEqual,
serializeSessionState, deserializeSessionState,
serializeSignedPreKey, deserializeSignedPreKey,
serializeOneTimePreKey, deserializeOneTimePreKey,
feat(hardening): M-Hard 1-5 — crypto, auth, rate limit, fingerprints, rotation M-Hard 1: Cryptographic Hardening - constantTimeEqual, zeroize, randomUint32 on CryptoProvider - Fix Math.random() → crypto.randomUint32() for registrationId - Zero message keys and chain keys after use in ratchet.ts - Constant-time trust comparison in MemoryStorage + SQLiteStorage - Timing variance test catches early-exit regressions M-Hard 2: Self-Authenticated Prekey Server - Ed25519 signature verification on all write routes - signPayload/verifyPayload with canonical JSON, ±5 min replay window - Address validation (NFKC, alphanumeric + :_-.) - Global ShadeError → HTTP status mapping - ShadeFetchTransport signs requests when signingPrivateKey provided - Anonymous bundle fetches still allowed (read-only) M-Hard 3: Rate Limiting + DoS Protection - Token bucket rate limiter with pluggable store - Per-route limits: register 5/h/IP, fetch 60/min/IP, replenish 10/min/id - 64KB body size limit on POST - Retry-After header on 429 responses M-Hard 4: Auto-replenish + Fingerprints + Session Reset - Safety numbers (12 groups × 5 digits, Signal-style) - ensurePreKeyStock, resetSession, acceptIdentityChange - verifyRemoteIdentity for out-of-band comparison M-Hard 5: Identity Rotation with Grace Period - rotateIdentity archives old identity, generates fresh signed prekey - RetiredIdentity storage with addRetired/getRetired/pruneRetired - 7-day default grace period for decrypting old sessions - pruneExpiredIdentities for cleanup M-Hard 8: Error Hierarchy - New error types: Network, Storage, Validation, Timeout, RateLimit, Configuration, Unauthorized, Replay, IdentityRotation - All errors have stable SHADE_* codes - errorToHttpStatus for consistent HTTP mapping - toJSON() for network serialization 188 tests passing, zero failures. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-10 17:45:34 +02:00
serializeIdentityKeyPair, deserializeIdentityKeyPair,
} from '@shade/core';
/**
* SQLite-backed StorageProvider for Shade client-side key/session storage.
*
* Uses bun:sqlite (built-in, zero deps). Stores private keys — for trusted environments only.
* WAL mode enabled for crash safety. Auto-creates tables on first use.
*
* Docker usage:
* Volume mount /data, set SHADE_DB_PATH=/data/shade-client.db
*/
export class SQLiteStorage implements StorageProvider {
private db: Database;
// Prepared statements
private stmts!: {
getIdentity: ReturnType<Database['prepare']>;
saveIdentity: ReturnType<Database['prepare']>;
getConfig: ReturnType<Database['prepare']>;
saveConfig: ReturnType<Database['prepare']>;
getSignedPreKey: ReturnType<Database['prepare']>;
saveSignedPreKey: ReturnType<Database['prepare']>;
removeSignedPreKey: ReturnType<Database['prepare']>;
getOneTimePreKey: ReturnType<Database['prepare']>;
saveOneTimePreKey: ReturnType<Database['prepare']>;
removeOneTimePreKey: ReturnType<Database['prepare']>;
countOneTimePreKeys: ReturnType<Database['prepare']>;
getSession: ReturnType<Database['prepare']>;
saveSession: ReturnType<Database['prepare']>;
removeSession: ReturnType<Database['prepare']>;
fix(session): remember where aliasSession moved a session aliasSession knew that two labels name the same peer, then threw that knowledge away. The binding lived only in the caller's memory, so a restart lost it — and the peer could not repair it from its side. First contact forces the receiver to label a session by the only sender hint a relay surfaces, an 8-byte signing-key fingerprint (`fp:<hex>`). Once the peer announces its canonical address, aliasSession moves the session there. But the peer keeps sending under `fp:<hex>`, because its transport derives the same label from the same hint every time. After a restart the session sat under the canonical address, inbound frames resolved to `fp:<hex>`, and nothing matched. The peer held a valid session so it never re-ran X3DH: the failure was permanent, and only a manual re-link cleared it. Observed in Prism as `No session for address: fp:579c3b335d66e2c0` on every receive for three days, with a phone whose every RPC timed out. StorageProvider gains saveSessionAlias / getSessionAlias / removeSessionAliasesFor, optional so third-party implementations keep compiling, and implemented across all seven backends. Lookups resolve through resolveLabel(), which runs BEFORE the peer mutex — locking the alias while mutating the canonical session would let an aliased and a canonical caller ratchet the same state concurrently. A live session under a label always wins over an alias, and prekey envelopes never resolve: both keep a re-link establishing a fresh session instead of being redirected into the stale one. Aliases are dropped in resetSession and acceptIdentityChange, and memoized so the hot path costs no extra read. The sdk.test.ts case that asserted a dead fp-label encoded the old behaviour; it now pins the new contract. Verified: 1166 tests pass (from 1160). With alias persistence disabled as a negative control, 5 of the 6 new tests fail, including both restart cases. Also drops `baseUrl` from the consumer-strict tsconfig — removed in TS 6.0, and it was failing the typecheck that gates publishing. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-13 19:36:31 +02:00
getSessionAlias: ReturnType<Database['prepare']>;
saveSessionAlias: ReturnType<Database['prepare']>;
removeAliasesFor: ReturnType<Database['prepare']>;
getTrust: ReturnType<Database['prepare']>;
saveTrust: ReturnType<Database['prepare']>;
feat(hardening): M-Hard 1-5 — crypto, auth, rate limit, fingerprints, rotation M-Hard 1: Cryptographic Hardening - constantTimeEqual, zeroize, randomUint32 on CryptoProvider - Fix Math.random() → crypto.randomUint32() for registrationId - Zero message keys and chain keys after use in ratchet.ts - Constant-time trust comparison in MemoryStorage + SQLiteStorage - Timing variance test catches early-exit regressions M-Hard 2: Self-Authenticated Prekey Server - Ed25519 signature verification on all write routes - signPayload/verifyPayload with canonical JSON, ±5 min replay window - Address validation (NFKC, alphanumeric + :_-.) - Global ShadeError → HTTP status mapping - ShadeFetchTransport signs requests when signingPrivateKey provided - Anonymous bundle fetches still allowed (read-only) M-Hard 3: Rate Limiting + DoS Protection - Token bucket rate limiter with pluggable store - Per-route limits: register 5/h/IP, fetch 60/min/IP, replenish 10/min/id - 64KB body size limit on POST - Retry-After header on 429 responses M-Hard 4: Auto-replenish + Fingerprints + Session Reset - Safety numbers (12 groups × 5 digits, Signal-style) - ensurePreKeyStock, resetSession, acceptIdentityChange - verifyRemoteIdentity for out-of-band comparison M-Hard 5: Identity Rotation with Grace Period - rotateIdentity archives old identity, generates fresh signed prekey - RetiredIdentity storage with addRetired/getRetired/pruneRetired - 7-day default grace period for decrypting old sessions - pruneExpiredIdentities for cleanup M-Hard 8: Error Hierarchy - New error types: Network, Storage, Validation, Timeout, RateLimit, Configuration, Unauthorized, Replay, IdentityRotation - All errors have stable SHADE_* codes - errorToHttpStatus for consistent HTTP mapping - toJSON() for network serialization 188 tests passing, zero failures. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-10 17:45:34 +02:00
addRetired: ReturnType<Database['prepare']>;
listRetired: ReturnType<Database['prepare']>;
pruneRetired: ReturnType<Database['prepare']>;
feat(files): @shade/files 0.3.0 — E2EE filesystem RPC primitive M-Files-1..6 land the full files-RPC layer + everything 0.3.0 needs to ship. Apps keep their own UI; this layer ships the typed RPC, the streams bridge for content I/O, and production hooks (rate limit, retention, fingerprint gate, metrics). @shade/files (NEW) - Standard ops: list/stat/mkdir/delete/move/read/write/getThumbnail with Zod-validated wire schemas + clean user-handler types. - Custom ops: typed via TypeScript declaration merging on CustomOpsMap + per-op Zod schemas; client.custom('app.foo', {...}) is fully typed. - Content I/O: inline (≤ 256 KiB plaintext) base64-in-RPC; streams (> 256 KiB) ride @shade/transfer via userMetadata.shadeFilesWriteId / shadeFilesReadStreamId correlation. Server-side TransformStream bridges accept inbound transfers immediately (engine rejects chunks that arrive before accept) and park the readable for the matching RPC. - Directory ops: walk(path, opts) async-iterable depth-first walker; uploadDirectory()/downloadDirectory() with bounded concurrency pool (default 4, cap 16), aggregated progress, abort. - Production hooks (callback-based, vendor-neutral): rate-limit (op + byte), idempotency cache (LRU + TTL + in-flight de-dupe), path policy (traversal + percent-decode hardening), fingerprint gate (required/optional/reject), pluggable Ed25519 sig verification with ±5 min replay window, onMetric sink (standard names). - React hooks (subpath @shade/files/react): ShadeFilesProvider, useShadeFiles, useFileList, useFileTransfer/Upload/Download. - Shade.files.serve(handler) + Shade.files.client(peer) high-level entrypoint in @shade/sdk; lazy + memoized; one handler per Shade. Wire format bump - @shade/proto wire VERSION 0x01 → 0x02. Length prefixes changed from u16 to u32. The previous u16 silently truncated payloads above 64 KiB — a hard correctness ceiling that blocked inline file ops up to 256 KiB. Wire-incompatible with 0.2.x peers; new sessions only. Cross-platform Kotlin port (android/shade-android) updated to match; test-vectors/wire-format.json regenerated. Concurrency safety - ShadeSessionManager.encrypt/.decrypt now run under per-peer mutex. Concurrent decryptions of the same peer raced ratchet state (manifested as sporadic "Failed to decrypt — wrong key or tampered data" under load — surfaced once concurrent uploadDirectory pumped many writes in flight). Encrypt was already serialized via Shade.send's encryptChains; decrypt is now serialized at the manager layer too. @shade/streams extension - StreamMetadata.userMetadata?: Record<string, string> for application-level key/value pairs that round-trip verbatim through stream-init plaintext. Used by @shade/files for write/read correlation; available to any consumer. @shade/sdk extension - Shade.files getter (lazy + memoized). - BackgroundHooks.onPruneFiles + periodic timer (default 5 min) + BackgroundTasks.setHook(name, fn) for runtime hook registration. Bundles in-flight 0.2.0 work - packages/shade-streams/, packages/shade-transfer/, related shade-sdk streams-bridge + shade-widgets transfer hooks were uncommitted prior to this session. Including them keeps the workspace consistent at 0.3.0 since @shade/files depends on them. Tests - 74 new tests in @shade/files (572 → 646 workspace pass; 0 fail; 3× stable). Coverage spans unit (inline-threshold + concurrency), integration (read-write inline + streams up to 1 MiB, walk + upload/download directory, custom-op, metrics, SDK namespace end-to-end), and security (tampered-envelope sig verification, replay window, fingerprint gate, rate-limit + quota). Release artifacts - All packages bumped to 0.3.0 via scripts/bump-version.ts. - scripts/publish-all.ts PACKAGES updated with shade-files in topological order (after shade-transfer, before shade-sdk). - bun run publish:dry clean (14 packed, 0 failed). - examples/08-files-browser/ — three-process CLI demo (prekey + Bob server + Alice CLI) covering list/stat/mkdir/delete/upload/download. - docs/files.md — full API + design doc. - CHANGELOG.md 0.3.0 entry. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-02 14:00:01 +02:00
saveStreamState: ReturnType<Database['prepare']>;
getStreamState: ReturnType<Database['prepare']>;
removeStreamState: ReturnType<Database['prepare']>;
listActiveStreamStates: ReturnType<Database['prepare']>;
listActiveStreamStatesByDirection: ReturnType<Database['prepare']>;
pruneStreamStates: ReturnType<Database['prepare']>;
savePeerVerification: ReturnType<Database['prepare']>;
getPeerVerification: ReturnType<Database['prepare']>;
removePeerVerification: ReturnType<Database['prepare']>;
getPeerIdentityVersion: ReturnType<Database['prepare']>;
upsertPeerIdentityVersion: ReturnType<Database['prepare']>;
saveBroadcastChannel: ReturnType<Database['prepare']>;
getBroadcastChannel: ReturnType<Database['prepare']>;
listBroadcastChannels: ReturnType<Database['prepare']>;
removeBroadcastChannel: ReturnType<Database['prepare']>;
removeBroadcastChannelMembers: ReturnType<Database['prepare']>;
saveBroadcastMember: ReturnType<Database['prepare']>;
getBroadcastMembers: ReturnType<Database['prepare']>;
removeBroadcastMember: ReturnType<Database['prepare']>;
};
constructor(dbPath?: string) {
const path = dbPath ?? process.env.SHADE_DB_PATH ?? '/data/shade-client.db';
this.db = new Database(path, { create: true });
this.db.exec('PRAGMA journal_mode=WAL');
this.ensureTables();
this.prepareStatements();
}
private ensureTables() {
this.db.exec(`
CREATE TABLE IF NOT EXISTS identity (
id INTEGER PRIMARY KEY CHECK (id = 1),
signing_public_key TEXT NOT NULL,
signing_private_key TEXT NOT NULL,
dh_public_key TEXT NOT NULL,
dh_private_key TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS config (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS signed_prekeys (
key_id INTEGER PRIMARY KEY,
data_json TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS one_time_prekeys (
key_id INTEGER PRIMARY KEY,
data_json TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS sessions (
address TEXT PRIMARY KEY,
state_json TEXT NOT NULL
);
fix(session): remember where aliasSession moved a session aliasSession knew that two labels name the same peer, then threw that knowledge away. The binding lived only in the caller's memory, so a restart lost it — and the peer could not repair it from its side. First contact forces the receiver to label a session by the only sender hint a relay surfaces, an 8-byte signing-key fingerprint (`fp:<hex>`). Once the peer announces its canonical address, aliasSession moves the session there. But the peer keeps sending under `fp:<hex>`, because its transport derives the same label from the same hint every time. After a restart the session sat under the canonical address, inbound frames resolved to `fp:<hex>`, and nothing matched. The peer held a valid session so it never re-ran X3DH: the failure was permanent, and only a manual re-link cleared it. Observed in Prism as `No session for address: fp:579c3b335d66e2c0` on every receive for three days, with a phone whose every RPC timed out. StorageProvider gains saveSessionAlias / getSessionAlias / removeSessionAliasesFor, optional so third-party implementations keep compiling, and implemented across all seven backends. Lookups resolve through resolveLabel(), which runs BEFORE the peer mutex — locking the alias while mutating the canonical session would let an aliased and a canonical caller ratchet the same state concurrently. A live session under a label always wins over an alias, and prekey envelopes never resolve: both keep a re-link establishing a fresh session instead of being redirected into the stale one. Aliases are dropped in resetSession and acceptIdentityChange, and memoized so the hot path costs no extra read. The sdk.test.ts case that asserted a dead fp-label encoded the old behaviour; it now pins the new contract. Verified: 1166 tests pass (from 1160). With alias persistence disabled as a negative control, 5 of the 6 new tests fail, including both restart cases. Also drops `baseUrl` from the consumer-strict tsconfig — removed in TS 6.0, and it was failing the typecheck that gates publishing. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-13 19:36:31 +02:00
CREATE TABLE IF NOT EXISTS session_aliases (
alias TEXT PRIMARY KEY,
canonical TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_session_aliases_canonical
ON session_aliases(canonical);
CREATE TABLE IF NOT EXISTS trusted_identities (
address TEXT PRIMARY KEY,
identity_key TEXT NOT NULL
);
feat(hardening): M-Hard 1-5 — crypto, auth, rate limit, fingerprints, rotation M-Hard 1: Cryptographic Hardening - constantTimeEqual, zeroize, randomUint32 on CryptoProvider - Fix Math.random() → crypto.randomUint32() for registrationId - Zero message keys and chain keys after use in ratchet.ts - Constant-time trust comparison in MemoryStorage + SQLiteStorage - Timing variance test catches early-exit regressions M-Hard 2: Self-Authenticated Prekey Server - Ed25519 signature verification on all write routes - signPayload/verifyPayload with canonical JSON, ±5 min replay window - Address validation (NFKC, alphanumeric + :_-.) - Global ShadeError → HTTP status mapping - ShadeFetchTransport signs requests when signingPrivateKey provided - Anonymous bundle fetches still allowed (read-only) M-Hard 3: Rate Limiting + DoS Protection - Token bucket rate limiter with pluggable store - Per-route limits: register 5/h/IP, fetch 60/min/IP, replenish 10/min/id - 64KB body size limit on POST - Retry-After header on 429 responses M-Hard 4: Auto-replenish + Fingerprints + Session Reset - Safety numbers (12 groups × 5 digits, Signal-style) - ensurePreKeyStock, resetSession, acceptIdentityChange - verifyRemoteIdentity for out-of-band comparison M-Hard 5: Identity Rotation with Grace Period - rotateIdentity archives old identity, generates fresh signed prekey - RetiredIdentity storage with addRetired/getRetired/pruneRetired - 7-day default grace period for decrypting old sessions - pruneExpiredIdentities for cleanup M-Hard 8: Error Hierarchy - New error types: Network, Storage, Validation, Timeout, RateLimit, Configuration, Unauthorized, Replay, IdentityRotation - All errors have stable SHADE_* codes - errorToHttpStatus for consistent HTTP mapping - toJSON() for network serialization 188 tests passing, zero failures. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-10 17:45:34 +02:00
CREATE TABLE IF NOT EXISTS retired_identities (
id INTEGER PRIMARY KEY AUTOINCREMENT,
data_json TEXT NOT NULL,
retired_at INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_retired_at ON retired_identities(retired_at);
feat(files): @shade/files 0.3.0 — E2EE filesystem RPC primitive M-Files-1..6 land the full files-RPC layer + everything 0.3.0 needs to ship. Apps keep their own UI; this layer ships the typed RPC, the streams bridge for content I/O, and production hooks (rate limit, retention, fingerprint gate, metrics). @shade/files (NEW) - Standard ops: list/stat/mkdir/delete/move/read/write/getThumbnail with Zod-validated wire schemas + clean user-handler types. - Custom ops: typed via TypeScript declaration merging on CustomOpsMap + per-op Zod schemas; client.custom('app.foo', {...}) is fully typed. - Content I/O: inline (≤ 256 KiB plaintext) base64-in-RPC; streams (> 256 KiB) ride @shade/transfer via userMetadata.shadeFilesWriteId / shadeFilesReadStreamId correlation. Server-side TransformStream bridges accept inbound transfers immediately (engine rejects chunks that arrive before accept) and park the readable for the matching RPC. - Directory ops: walk(path, opts) async-iterable depth-first walker; uploadDirectory()/downloadDirectory() with bounded concurrency pool (default 4, cap 16), aggregated progress, abort. - Production hooks (callback-based, vendor-neutral): rate-limit (op + byte), idempotency cache (LRU + TTL + in-flight de-dupe), path policy (traversal + percent-decode hardening), fingerprint gate (required/optional/reject), pluggable Ed25519 sig verification with ±5 min replay window, onMetric sink (standard names). - React hooks (subpath @shade/files/react): ShadeFilesProvider, useShadeFiles, useFileList, useFileTransfer/Upload/Download. - Shade.files.serve(handler) + Shade.files.client(peer) high-level entrypoint in @shade/sdk; lazy + memoized; one handler per Shade. Wire format bump - @shade/proto wire VERSION 0x01 → 0x02. Length prefixes changed from u16 to u32. The previous u16 silently truncated payloads above 64 KiB — a hard correctness ceiling that blocked inline file ops up to 256 KiB. Wire-incompatible with 0.2.x peers; new sessions only. Cross-platform Kotlin port (android/shade-android) updated to match; test-vectors/wire-format.json regenerated. Concurrency safety - ShadeSessionManager.encrypt/.decrypt now run under per-peer mutex. Concurrent decryptions of the same peer raced ratchet state (manifested as sporadic "Failed to decrypt — wrong key or tampered data" under load — surfaced once concurrent uploadDirectory pumped many writes in flight). Encrypt was already serialized via Shade.send's encryptChains; decrypt is now serialized at the manager layer too. @shade/streams extension - StreamMetadata.userMetadata?: Record<string, string> for application-level key/value pairs that round-trip verbatim through stream-init plaintext. Used by @shade/files for write/read correlation; available to any consumer. @shade/sdk extension - Shade.files getter (lazy + memoized). - BackgroundHooks.onPruneFiles + periodic timer (default 5 min) + BackgroundTasks.setHook(name, fn) for runtime hook registration. Bundles in-flight 0.2.0 work - packages/shade-streams/, packages/shade-transfer/, related shade-sdk streams-bridge + shade-widgets transfer hooks were uncommitted prior to this session. Including them keeps the workspace consistent at 0.3.0 since @shade/files depends on them. Tests - 74 new tests in @shade/files (572 → 646 workspace pass; 0 fail; 3× stable). Coverage spans unit (inline-threshold + concurrency), integration (read-write inline + streams up to 1 MiB, walk + upload/download directory, custom-op, metrics, SDK namespace end-to-end), and security (tampered-envelope sig verification, replay window, fingerprint gate, rate-limit + quota). Release artifacts - All packages bumped to 0.3.0 via scripts/bump-version.ts. - scripts/publish-all.ts PACKAGES updated with shade-files in topological order (after shade-transfer, before shade-sdk). - bun run publish:dry clean (14 packed, 0 failed). - examples/08-files-browser/ — three-process CLI demo (prekey + Bob server + Alice CLI) covering list/stat/mkdir/delete/upload/download. - docs/files.md — full API + design doc. - CHANGELOG.md 0.3.0 entry. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-02 14:00:01 +02:00
CREATE TABLE IF NOT EXISTS stream_state (
stream_id TEXT PRIMARY KEY,
direction TEXT NOT NULL,
peer_address TEXT NOT NULL,
status TEXT NOT NULL,
metadata_json TEXT NOT NULL,
partition_json TEXT NOT NULL,
lane_state_json TEXT NOT NULL,
io_descriptor_json TEXT NOT NULL,
secret_enc BLOB NOT NULL,
secret_nonce BLOB NOT NULL,
overall_hash_state TEXT,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_stream_state_peer ON stream_state(peer_address);
CREATE INDEX IF NOT EXISTS idx_stream_state_updated ON stream_state(updated_at);
CREATE INDEX IF NOT EXISTS idx_stream_state_status ON stream_state(status, direction);
CREATE TABLE IF NOT EXISTS peer_verifications (
peer_address TEXT PRIMARY KEY,
fingerprint TEXT NOT NULL,
verified_at INTEGER NOT NULL,
verified_by TEXT NOT NULL,
identity_version INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS peer_identity_versions (
peer_address TEXT PRIMARY KEY,
version INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS broadcast_channels (
channel_id TEXT PRIMARY KEY,
owner_role TEXT NOT NULL,
owner_address TEXT NOT NULL,
label TEXT,
generation INTEGER NOT NULL,
chain_key BLOB NOT NULL,
iteration INTEGER NOT NULL,
signing_public_key BLOB NOT NULL,
signing_private_key BLOB,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS broadcast_members (
channel_id TEXT NOT NULL,
peer_address TEXT NOT NULL,
joined_at INTEGER NOT NULL,
removed_at INTEGER,
PRIMARY KEY (channel_id, peer_address)
);
CREATE INDEX IF NOT EXISTS idx_broadcast_members_channel ON broadcast_members(channel_id);
`);
}
private prepareStatements() {
this.stmts = {
getIdentity: this.db.prepare('SELECT * FROM identity WHERE id = 1'),
saveIdentity: this.db.prepare('INSERT OR REPLACE INTO identity (id, signing_public_key, signing_private_key, dh_public_key, dh_private_key) VALUES (1, ?, ?, ?, ?)'),
getConfig: this.db.prepare('SELECT value FROM config WHERE key = ?'),
saveConfig: this.db.prepare('INSERT OR REPLACE INTO config (key, value) VALUES (?, ?)'),
getSignedPreKey: this.db.prepare('SELECT data_json FROM signed_prekeys WHERE key_id = ?'),
saveSignedPreKey: this.db.prepare('INSERT OR REPLACE INTO signed_prekeys (key_id, data_json) VALUES (?, ?)'),
removeSignedPreKey: this.db.prepare('DELETE FROM signed_prekeys WHERE key_id = ?'),
getOneTimePreKey: this.db.prepare('SELECT data_json FROM one_time_prekeys WHERE key_id = ?'),
saveOneTimePreKey: this.db.prepare('INSERT OR REPLACE INTO one_time_prekeys (key_id, data_json) VALUES (?, ?)'),
removeOneTimePreKey: this.db.prepare('DELETE FROM one_time_prekeys WHERE key_id = ?'),
countOneTimePreKeys: this.db.prepare('SELECT COUNT(*) as count FROM one_time_prekeys'),
getSession: this.db.prepare('SELECT state_json FROM sessions WHERE address = ?'),
saveSession: this.db.prepare('INSERT OR REPLACE INTO sessions (address, state_json) VALUES (?, ?)'),
removeSession: this.db.prepare('DELETE FROM sessions WHERE address = ?'),
fix(session): remember where aliasSession moved a session aliasSession knew that two labels name the same peer, then threw that knowledge away. The binding lived only in the caller's memory, so a restart lost it — and the peer could not repair it from its side. First contact forces the receiver to label a session by the only sender hint a relay surfaces, an 8-byte signing-key fingerprint (`fp:<hex>`). Once the peer announces its canonical address, aliasSession moves the session there. But the peer keeps sending under `fp:<hex>`, because its transport derives the same label from the same hint every time. After a restart the session sat under the canonical address, inbound frames resolved to `fp:<hex>`, and nothing matched. The peer held a valid session so it never re-ran X3DH: the failure was permanent, and only a manual re-link cleared it. Observed in Prism as `No session for address: fp:579c3b335d66e2c0` on every receive for three days, with a phone whose every RPC timed out. StorageProvider gains saveSessionAlias / getSessionAlias / removeSessionAliasesFor, optional so third-party implementations keep compiling, and implemented across all seven backends. Lookups resolve through resolveLabel(), which runs BEFORE the peer mutex — locking the alias while mutating the canonical session would let an aliased and a canonical caller ratchet the same state concurrently. A live session under a label always wins over an alias, and prekey envelopes never resolve: both keep a re-link establishing a fresh session instead of being redirected into the stale one. Aliases are dropped in resetSession and acceptIdentityChange, and memoized so the hot path costs no extra read. The sdk.test.ts case that asserted a dead fp-label encoded the old behaviour; it now pins the new contract. Verified: 1166 tests pass (from 1160). With alias persistence disabled as a negative control, 5 of the 6 new tests fail, including both restart cases. Also drops `baseUrl` from the consumer-strict tsconfig — removed in TS 6.0, and it was failing the typecheck that gates publishing. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-13 19:36:31 +02:00
getSessionAlias: this.db.prepare('SELECT canonical FROM session_aliases WHERE alias = ?'),
saveSessionAlias: this.db.prepare('INSERT OR REPLACE INTO session_aliases (alias, canonical) VALUES (?, ?)'),
removeAliasesFor: this.db.prepare('DELETE FROM session_aliases WHERE canonical = ?'),
getTrust: this.db.prepare('SELECT identity_key FROM trusted_identities WHERE address = ?'),
saveTrust: this.db.prepare('INSERT OR REPLACE INTO trusted_identities (address, identity_key) VALUES (?, ?)'),
feat(hardening): M-Hard 1-5 — crypto, auth, rate limit, fingerprints, rotation M-Hard 1: Cryptographic Hardening - constantTimeEqual, zeroize, randomUint32 on CryptoProvider - Fix Math.random() → crypto.randomUint32() for registrationId - Zero message keys and chain keys after use in ratchet.ts - Constant-time trust comparison in MemoryStorage + SQLiteStorage - Timing variance test catches early-exit regressions M-Hard 2: Self-Authenticated Prekey Server - Ed25519 signature verification on all write routes - signPayload/verifyPayload with canonical JSON, ±5 min replay window - Address validation (NFKC, alphanumeric + :_-.) - Global ShadeError → HTTP status mapping - ShadeFetchTransport signs requests when signingPrivateKey provided - Anonymous bundle fetches still allowed (read-only) M-Hard 3: Rate Limiting + DoS Protection - Token bucket rate limiter with pluggable store - Per-route limits: register 5/h/IP, fetch 60/min/IP, replenish 10/min/id - 64KB body size limit on POST - Retry-After header on 429 responses M-Hard 4: Auto-replenish + Fingerprints + Session Reset - Safety numbers (12 groups × 5 digits, Signal-style) - ensurePreKeyStock, resetSession, acceptIdentityChange - verifyRemoteIdentity for out-of-band comparison M-Hard 5: Identity Rotation with Grace Period - rotateIdentity archives old identity, generates fresh signed prekey - RetiredIdentity storage with addRetired/getRetired/pruneRetired - 7-day default grace period for decrypting old sessions - pruneExpiredIdentities for cleanup M-Hard 8: Error Hierarchy - New error types: Network, Storage, Validation, Timeout, RateLimit, Configuration, Unauthorized, Replay, IdentityRotation - All errors have stable SHADE_* codes - errorToHttpStatus for consistent HTTP mapping - toJSON() for network serialization 188 tests passing, zero failures. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-10 17:45:34 +02:00
addRetired: this.db.prepare('INSERT INTO retired_identities (data_json, retired_at) VALUES (?, ?)'),
listRetired: this.db.prepare('SELECT data_json, retired_at FROM retired_identities ORDER BY retired_at DESC'),
pruneRetired: this.db.prepare('DELETE FROM retired_identities WHERE retired_at < ?'),
feat(files): @shade/files 0.3.0 — E2EE filesystem RPC primitive M-Files-1..6 land the full files-RPC layer + everything 0.3.0 needs to ship. Apps keep their own UI; this layer ships the typed RPC, the streams bridge for content I/O, and production hooks (rate limit, retention, fingerprint gate, metrics). @shade/files (NEW) - Standard ops: list/stat/mkdir/delete/move/read/write/getThumbnail with Zod-validated wire schemas + clean user-handler types. - Custom ops: typed via TypeScript declaration merging on CustomOpsMap + per-op Zod schemas; client.custom('app.foo', {...}) is fully typed. - Content I/O: inline (≤ 256 KiB plaintext) base64-in-RPC; streams (> 256 KiB) ride @shade/transfer via userMetadata.shadeFilesWriteId / shadeFilesReadStreamId correlation. Server-side TransformStream bridges accept inbound transfers immediately (engine rejects chunks that arrive before accept) and park the readable for the matching RPC. - Directory ops: walk(path, opts) async-iterable depth-first walker; uploadDirectory()/downloadDirectory() with bounded concurrency pool (default 4, cap 16), aggregated progress, abort. - Production hooks (callback-based, vendor-neutral): rate-limit (op + byte), idempotency cache (LRU + TTL + in-flight de-dupe), path policy (traversal + percent-decode hardening), fingerprint gate (required/optional/reject), pluggable Ed25519 sig verification with ±5 min replay window, onMetric sink (standard names). - React hooks (subpath @shade/files/react): ShadeFilesProvider, useShadeFiles, useFileList, useFileTransfer/Upload/Download. - Shade.files.serve(handler) + Shade.files.client(peer) high-level entrypoint in @shade/sdk; lazy + memoized; one handler per Shade. Wire format bump - @shade/proto wire VERSION 0x01 → 0x02. Length prefixes changed from u16 to u32. The previous u16 silently truncated payloads above 64 KiB — a hard correctness ceiling that blocked inline file ops up to 256 KiB. Wire-incompatible with 0.2.x peers; new sessions only. Cross-platform Kotlin port (android/shade-android) updated to match; test-vectors/wire-format.json regenerated. Concurrency safety - ShadeSessionManager.encrypt/.decrypt now run under per-peer mutex. Concurrent decryptions of the same peer raced ratchet state (manifested as sporadic "Failed to decrypt — wrong key or tampered data" under load — surfaced once concurrent uploadDirectory pumped many writes in flight). Encrypt was already serialized via Shade.send's encryptChains; decrypt is now serialized at the manager layer too. @shade/streams extension - StreamMetadata.userMetadata?: Record<string, string> for application-level key/value pairs that round-trip verbatim through stream-init plaintext. Used by @shade/files for write/read correlation; available to any consumer. @shade/sdk extension - Shade.files getter (lazy + memoized). - BackgroundHooks.onPruneFiles + periodic timer (default 5 min) + BackgroundTasks.setHook(name, fn) for runtime hook registration. Bundles in-flight 0.2.0 work - packages/shade-streams/, packages/shade-transfer/, related shade-sdk streams-bridge + shade-widgets transfer hooks were uncommitted prior to this session. Including them keeps the workspace consistent at 0.3.0 since @shade/files depends on them. Tests - 74 new tests in @shade/files (572 → 646 workspace pass; 0 fail; 3× stable). Coverage spans unit (inline-threshold + concurrency), integration (read-write inline + streams up to 1 MiB, walk + upload/download directory, custom-op, metrics, SDK namespace end-to-end), and security (tampered-envelope sig verification, replay window, fingerprint gate, rate-limit + quota). Release artifacts - All packages bumped to 0.3.0 via scripts/bump-version.ts. - scripts/publish-all.ts PACKAGES updated with shade-files in topological order (after shade-transfer, before shade-sdk). - bun run publish:dry clean (14 packed, 0 failed). - examples/08-files-browser/ — three-process CLI demo (prekey + Bob server + Alice CLI) covering list/stat/mkdir/delete/upload/download. - docs/files.md — full API + design doc. - CHANGELOG.md 0.3.0 entry. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-02 14:00:01 +02:00
saveStreamState: this.db.prepare(
`INSERT OR REPLACE INTO stream_state (
stream_id, direction, peer_address, status,
metadata_json, partition_json, lane_state_json, io_descriptor_json,
secret_enc, secret_nonce, overall_hash_state, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
),
getStreamState: this.db.prepare('SELECT * FROM stream_state WHERE stream_id = ?'),
removeStreamState: this.db.prepare('DELETE FROM stream_state WHERE stream_id = ?'),
listActiveStreamStates: this.db.prepare(
"SELECT * FROM stream_state WHERE status IN ('active', 'paused') ORDER BY updated_at DESC",
),
listActiveStreamStatesByDirection: this.db.prepare(
"SELECT * FROM stream_state WHERE status IN ('active', 'paused') AND direction = ? ORDER BY updated_at DESC",
),
pruneStreamStates: this.db.prepare(
"DELETE FROM stream_state WHERE status IN ('finished', 'aborted') AND updated_at < ?",
),
savePeerVerification: this.db.prepare(
`INSERT OR REPLACE INTO peer_verifications
(peer_address, fingerprint, verified_at, verified_by, identity_version)
VALUES (?, ?, ?, ?, ?)`,
),
getPeerVerification: this.db.prepare(
'SELECT peer_address, fingerprint, verified_at, verified_by, identity_version FROM peer_verifications WHERE peer_address = ?',
),
removePeerVerification: this.db.prepare('DELETE FROM peer_verifications WHERE peer_address = ?'),
getPeerIdentityVersion: this.db.prepare('SELECT version FROM peer_identity_versions WHERE peer_address = ?'),
upsertPeerIdentityVersion: this.db.prepare(
`INSERT INTO peer_identity_versions (peer_address, version) VALUES (?, ?)
ON CONFLICT(peer_address) DO UPDATE SET version = excluded.version`,
),
saveBroadcastChannel: this.db.prepare(
`INSERT OR REPLACE INTO broadcast_channels
(channel_id, owner_role, owner_address, label, generation,
chain_key, iteration, signing_public_key, signing_private_key,
created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
),
getBroadcastChannel: this.db.prepare(
'SELECT * FROM broadcast_channels WHERE channel_id = ?',
),
listBroadcastChannels: this.db.prepare(
'SELECT * FROM broadcast_channels ORDER BY created_at ASC',
),
removeBroadcastChannel: this.db.prepare(
'DELETE FROM broadcast_channels WHERE channel_id = ?',
),
removeBroadcastChannelMembers: this.db.prepare(
'DELETE FROM broadcast_members WHERE channel_id = ?',
),
saveBroadcastMember: this.db.prepare(
`INSERT OR REPLACE INTO broadcast_members
(channel_id, peer_address, joined_at, removed_at)
VALUES (?, ?, ?, ?)`,
),
getBroadcastMembers: this.db.prepare(
'SELECT channel_id, peer_address, joined_at, removed_at FROM broadcast_members WHERE channel_id = ? ORDER BY joined_at ASC',
),
removeBroadcastMember: this.db.prepare(
'DELETE FROM broadcast_members WHERE channel_id = ? AND peer_address = ?',
),
};
}
close() {
this.db.close();
}
// ─── Identity ──────────────────────────────────────────────
async getIdentityKeyPair(): Promise<IdentityKeyPair | null> {
const row = this.stmts.getIdentity.get() as any;
if (!row) return null;
return {
signingPublicKey: fromBase64(row.signing_public_key),
signingPrivateKey: fromBase64(row.signing_private_key),
dhPublicKey: fromBase64(row.dh_public_key),
dhPrivateKey: fromBase64(row.dh_private_key),
};
}
async saveIdentityKeyPair(kp: IdentityKeyPair): Promise<void> {
this.stmts.saveIdentity.run(
toBase64(kp.signingPublicKey),
toBase64(kp.signingPrivateKey),
toBase64(kp.dhPublicKey),
toBase64(kp.dhPrivateKey),
);
}
async getLocalRegistrationId(): Promise<number> {
const row = this.stmts.getConfig.get('registrationId') as any;
return row ? parseInt(row.value, 10) : 0;
}
async saveLocalRegistrationId(id: number): Promise<void> {
this.stmts.saveConfig.run('registrationId', String(id));
}
// ─── Signed PreKeys ───────────────────────────────────────
async getSignedPreKey(keyId: number): Promise<SignedPreKey | null> {
const row = this.stmts.getSignedPreKey.get(keyId) as any;
if (!row) return null;
return deserializeSignedPreKey(row.data_json);
}
async saveSignedPreKey(key: SignedPreKey): Promise<void> {
this.stmts.saveSignedPreKey.run(key.keyId, serializeSignedPreKey(key));
}
async removeSignedPreKey(keyId: number): Promise<void> {
this.stmts.removeSignedPreKey.run(keyId);
}
// ─── One-Time PreKeys ─────────────────────────────────────
async getOneTimePreKey(keyId: number): Promise<OneTimePreKey | null> {
const row = this.stmts.getOneTimePreKey.get(keyId) as any;
if (!row) return null;
return deserializeOneTimePreKey(row.data_json);
}
async saveOneTimePreKey(key: OneTimePreKey): Promise<void> {
this.stmts.saveOneTimePreKey.run(key.keyId, serializeOneTimePreKey(key));
}
async removeOneTimePreKey(keyId: number): Promise<void> {
this.stmts.removeOneTimePreKey.run(keyId);
}
async getOneTimePreKeyCount(): Promise<number> {
const row = this.stmts.countOneTimePreKeys.get() as any;
return row.count;
}
// ─── Sessions ─────────────────────────────────────────────
async getSession(address: string): Promise<SessionState | null> {
const row = this.stmts.getSession.get(address) as any;
if (!row) return null;
return deserializeSessionState(row.state_json);
}
async saveSession(address: string, state: SessionState): Promise<void> {
this.stmts.saveSession.run(address, serializeSessionState(state));
}
async removeSession(address: string): Promise<void> {
this.stmts.removeSession.run(address);
}
fix(session): remember where aliasSession moved a session aliasSession knew that two labels name the same peer, then threw that knowledge away. The binding lived only in the caller's memory, so a restart lost it — and the peer could not repair it from its side. First contact forces the receiver to label a session by the only sender hint a relay surfaces, an 8-byte signing-key fingerprint (`fp:<hex>`). Once the peer announces its canonical address, aliasSession moves the session there. But the peer keeps sending under `fp:<hex>`, because its transport derives the same label from the same hint every time. After a restart the session sat under the canonical address, inbound frames resolved to `fp:<hex>`, and nothing matched. The peer held a valid session so it never re-ran X3DH: the failure was permanent, and only a manual re-link cleared it. Observed in Prism as `No session for address: fp:579c3b335d66e2c0` on every receive for three days, with a phone whose every RPC timed out. StorageProvider gains saveSessionAlias / getSessionAlias / removeSessionAliasesFor, optional so third-party implementations keep compiling, and implemented across all seven backends. Lookups resolve through resolveLabel(), which runs BEFORE the peer mutex — locking the alias while mutating the canonical session would let an aliased and a canonical caller ratchet the same state concurrently. A live session under a label always wins over an alias, and prekey envelopes never resolve: both keep a re-link establishing a fresh session instead of being redirected into the stale one. Aliases are dropped in resetSession and acceptIdentityChange, and memoized so the hot path costs no extra read. The sdk.test.ts case that asserted a dead fp-label encoded the old behaviour; it now pins the new contract. Verified: 1166 tests pass (from 1160). With alias persistence disabled as a negative control, 5 of the 6 new tests fail, including both restart cases. Also drops `baseUrl` from the consumer-strict tsconfig — removed in TS 6.0, and it was failing the typecheck that gates publishing. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-13 19:36:31 +02:00
// ─── Session label aliases ────────────────────────────────
async getSessionAlias(alias: string): Promise<string | null> {
const row = this.stmts.getSessionAlias.get(alias) as { canonical: string } | undefined;
return row?.canonical ?? null;
}
async saveSessionAlias(alias: string, canonical: string): Promise<void> {
this.stmts.saveSessionAlias.run(alias, canonical);
}
async removeSessionAliasesFor(canonical: string): Promise<void> {
this.stmts.removeAliasesFor.run(canonical);
}
// ─── Trust ────────────────────────────────────────────────
async isTrustedIdentity(address: string, identityKey: Uint8Array): Promise<boolean> {
const row = this.stmts.getTrust.get(address) as any;
if (!row) return true; // TOFU
feat(hardening): M-Hard 1-5 — crypto, auth, rate limit, fingerprints, rotation M-Hard 1: Cryptographic Hardening - constantTimeEqual, zeroize, randomUint32 on CryptoProvider - Fix Math.random() → crypto.randomUint32() for registrationId - Zero message keys and chain keys after use in ratchet.ts - Constant-time trust comparison in MemoryStorage + SQLiteStorage - Timing variance test catches early-exit regressions M-Hard 2: Self-Authenticated Prekey Server - Ed25519 signature verification on all write routes - signPayload/verifyPayload with canonical JSON, ±5 min replay window - Address validation (NFKC, alphanumeric + :_-.) - Global ShadeError → HTTP status mapping - ShadeFetchTransport signs requests when signingPrivateKey provided - Anonymous bundle fetches still allowed (read-only) M-Hard 3: Rate Limiting + DoS Protection - Token bucket rate limiter with pluggable store - Per-route limits: register 5/h/IP, fetch 60/min/IP, replenish 10/min/id - 64KB body size limit on POST - Retry-After header on 429 responses M-Hard 4: Auto-replenish + Fingerprints + Session Reset - Safety numbers (12 groups × 5 digits, Signal-style) - ensurePreKeyStock, resetSession, acceptIdentityChange - verifyRemoteIdentity for out-of-band comparison M-Hard 5: Identity Rotation with Grace Period - rotateIdentity archives old identity, generates fresh signed prekey - RetiredIdentity storage with addRetired/getRetired/pruneRetired - 7-day default grace period for decrypting old sessions - pruneExpiredIdentities for cleanup M-Hard 8: Error Hierarchy - New error types: Network, Storage, Validation, Timeout, RateLimit, Configuration, Unauthorized, Replay, IdentityRotation - All errors have stable SHADE_* codes - errorToHttpStatus for consistent HTTP mapping - toJSON() for network serialization 188 tests passing, zero failures. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-10 17:45:34 +02:00
const storedBytes = fromBase64(row.identity_key);
return constantTimeEqual(storedBytes, identityKey);
}
async saveTrustedIdentity(address: string, identityKey: Uint8Array): Promise<void> {
this.stmts.saveTrust.run(address, toBase64(identityKey));
}
feat(hardening): M-Hard 1-5 — crypto, auth, rate limit, fingerprints, rotation M-Hard 1: Cryptographic Hardening - constantTimeEqual, zeroize, randomUint32 on CryptoProvider - Fix Math.random() → crypto.randomUint32() for registrationId - Zero message keys and chain keys after use in ratchet.ts - Constant-time trust comparison in MemoryStorage + SQLiteStorage - Timing variance test catches early-exit regressions M-Hard 2: Self-Authenticated Prekey Server - Ed25519 signature verification on all write routes - signPayload/verifyPayload with canonical JSON, ±5 min replay window - Address validation (NFKC, alphanumeric + :_-.) - Global ShadeError → HTTP status mapping - ShadeFetchTransport signs requests when signingPrivateKey provided - Anonymous bundle fetches still allowed (read-only) M-Hard 3: Rate Limiting + DoS Protection - Token bucket rate limiter with pluggable store - Per-route limits: register 5/h/IP, fetch 60/min/IP, replenish 10/min/id - 64KB body size limit on POST - Retry-After header on 429 responses M-Hard 4: Auto-replenish + Fingerprints + Session Reset - Safety numbers (12 groups × 5 digits, Signal-style) - ensurePreKeyStock, resetSession, acceptIdentityChange - verifyRemoteIdentity for out-of-band comparison M-Hard 5: Identity Rotation with Grace Period - rotateIdentity archives old identity, generates fresh signed prekey - RetiredIdentity storage with addRetired/getRetired/pruneRetired - 7-day default grace period for decrypting old sessions - pruneExpiredIdentities for cleanup M-Hard 8: Error Hierarchy - New error types: Network, Storage, Validation, Timeout, RateLimit, Configuration, Unauthorized, Replay, IdentityRotation - All errors have stable SHADE_* codes - errorToHttpStatus for consistent HTTP mapping - toJSON() for network serialization 188 tests passing, zero failures. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-10 17:45:34 +02:00
// ─── Identity History ─────────────────────────────────────
async addRetiredIdentity(identity: RetiredIdentity): Promise<void> {
this.stmts.addRetired.run(
serializeIdentityKeyPair(identity.keyPair),
identity.retiredAt,
);
}
async getRetiredIdentities(): Promise<RetiredIdentity[]> {
const rows = this.stmts.listRetired.all() as any[];
return rows.map((r) => ({
keyPair: deserializeIdentityKeyPair(r.data_json),
retiredAt: r.retired_at,
}));
}
async pruneRetiredIdentities(olderThan: number): Promise<void> {
this.stmts.pruneRetired.run(olderThan);
}
feat(files): @shade/files 0.3.0 — E2EE filesystem RPC primitive M-Files-1..6 land the full files-RPC layer + everything 0.3.0 needs to ship. Apps keep their own UI; this layer ships the typed RPC, the streams bridge for content I/O, and production hooks (rate limit, retention, fingerprint gate, metrics). @shade/files (NEW) - Standard ops: list/stat/mkdir/delete/move/read/write/getThumbnail with Zod-validated wire schemas + clean user-handler types. - Custom ops: typed via TypeScript declaration merging on CustomOpsMap + per-op Zod schemas; client.custom('app.foo', {...}) is fully typed. - Content I/O: inline (≤ 256 KiB plaintext) base64-in-RPC; streams (> 256 KiB) ride @shade/transfer via userMetadata.shadeFilesWriteId / shadeFilesReadStreamId correlation. Server-side TransformStream bridges accept inbound transfers immediately (engine rejects chunks that arrive before accept) and park the readable for the matching RPC. - Directory ops: walk(path, opts) async-iterable depth-first walker; uploadDirectory()/downloadDirectory() with bounded concurrency pool (default 4, cap 16), aggregated progress, abort. - Production hooks (callback-based, vendor-neutral): rate-limit (op + byte), idempotency cache (LRU + TTL + in-flight de-dupe), path policy (traversal + percent-decode hardening), fingerprint gate (required/optional/reject), pluggable Ed25519 sig verification with ±5 min replay window, onMetric sink (standard names). - React hooks (subpath @shade/files/react): ShadeFilesProvider, useShadeFiles, useFileList, useFileTransfer/Upload/Download. - Shade.files.serve(handler) + Shade.files.client(peer) high-level entrypoint in @shade/sdk; lazy + memoized; one handler per Shade. Wire format bump - @shade/proto wire VERSION 0x01 → 0x02. Length prefixes changed from u16 to u32. The previous u16 silently truncated payloads above 64 KiB — a hard correctness ceiling that blocked inline file ops up to 256 KiB. Wire-incompatible with 0.2.x peers; new sessions only. Cross-platform Kotlin port (android/shade-android) updated to match; test-vectors/wire-format.json regenerated. Concurrency safety - ShadeSessionManager.encrypt/.decrypt now run under per-peer mutex. Concurrent decryptions of the same peer raced ratchet state (manifested as sporadic "Failed to decrypt — wrong key or tampered data" under load — surfaced once concurrent uploadDirectory pumped many writes in flight). Encrypt was already serialized via Shade.send's encryptChains; decrypt is now serialized at the manager layer too. @shade/streams extension - StreamMetadata.userMetadata?: Record<string, string> for application-level key/value pairs that round-trip verbatim through stream-init plaintext. Used by @shade/files for write/read correlation; available to any consumer. @shade/sdk extension - Shade.files getter (lazy + memoized). - BackgroundHooks.onPruneFiles + periodic timer (default 5 min) + BackgroundTasks.setHook(name, fn) for runtime hook registration. Bundles in-flight 0.2.0 work - packages/shade-streams/, packages/shade-transfer/, related shade-sdk streams-bridge + shade-widgets transfer hooks were uncommitted prior to this session. Including them keeps the workspace consistent at 0.3.0 since @shade/files depends on them. Tests - 74 new tests in @shade/files (572 → 646 workspace pass; 0 fail; 3× stable). Coverage spans unit (inline-threshold + concurrency), integration (read-write inline + streams up to 1 MiB, walk + upload/download directory, custom-op, metrics, SDK namespace end-to-end), and security (tampered-envelope sig verification, replay window, fingerprint gate, rate-limit + quota). Release artifacts - All packages bumped to 0.3.0 via scripts/bump-version.ts. - scripts/publish-all.ts PACKAGES updated with shade-files in topological order (after shade-transfer, before shade-sdk). - bun run publish:dry clean (14 packed, 0 failed). - examples/08-files-browser/ — three-process CLI demo (prekey + Bob server + Alice CLI) covering list/stat/mkdir/delete/upload/download. - docs/files.md — full API + design doc. - CHANGELOG.md 0.3.0 entry. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-02 14:00:01 +02:00
// ─── Stream-transfer resume state (v0.2.0) ────────────────
async saveStreamState(state: PersistedStreamState): Promise<void> {
this.stmts.saveStreamState.run(
state.streamId,
state.direction,
state.peerAddress,
state.status,
state.metadataJson,
state.partitionJson,
state.laneStateJson,
state.ioDescriptorJson,
state.secretEnc,
state.secretNonce,
state.overallHashState ?? null,
state.createdAt,
state.updatedAt,
);
}
async getStreamState(streamId: string): Promise<PersistedStreamState | null> {
const row = this.stmts.getStreamState.get(streamId) as any;
if (!row) return null;
return rowToStreamState(row);
}
async removeStreamState(streamId: string): Promise<void> {
this.stmts.removeStreamState.run(streamId);
}
async listActiveStreamStates(direction?: 'send' | 'receive'): Promise<PersistedStreamState[]> {
const rows = (
direction === undefined
? (this.stmts.listActiveStreamStates.all() as any[])
: (this.stmts.listActiveStreamStatesByDirection.all(direction) as any[])
);
return rows.map(rowToStreamState);
}
async pruneStreamStates(olderThan: number): Promise<void> {
this.stmts.pruneStreamStates.run(olderThan);
}
// ─── Peer verifications (V3.3) ────────────────────────────
async savePeerVerification(v: PeerVerification): Promise<void> {
this.stmts.savePeerVerification.run(
v.peerAddress,
v.fingerprint,
v.verifiedAt,
v.verifiedBy,
v.identityVersion,
);
}
async getPeerVerification(address: string): Promise<PeerVerification | null> {
const row = this.stmts.getPeerVerification.get(address) as
| { peer_address: string; fingerprint: string; verified_at: number | bigint; verified_by: string; identity_version: number | bigint }
| undefined;
if (!row) return null;
return {
peerAddress: row.peer_address,
fingerprint: row.fingerprint,
verifiedAt: Number(row.verified_at),
verifiedBy: row.verified_by as PeerVerificationSource,
identityVersion: Number(row.identity_version),
};
}
async removePeerVerification(address: string): Promise<void> {
this.stmts.removePeerVerification.run(address);
}
async getPeerIdentityVersion(address: string): Promise<number> {
const row = this.stmts.getPeerIdentityVersion.get(address) as { version: number | bigint } | undefined;
return row ? Number(row.version) : 1;
}
async bumpPeerIdentityVersion(address: string): Promise<number> {
const current = await this.getPeerIdentityVersion(address);
const next = current + 1;
this.stmts.upsertPeerIdentityVersion.run(address, next);
return next;
}
// ─── Broadcast channels (V4.6) ────────────────────────────
async saveBroadcastChannel(channel: BroadcastChannelRecord): Promise<void> {
this.stmts.saveBroadcastChannel.run(
channel.channelId,
channel.ownerRole,
channel.ownerAddress,
channel.label ?? null,
channel.generation,
channel.chainKey,
channel.iteration,
channel.signingPublicKey,
channel.signingPrivateKey ?? null,
channel.createdAt,
channel.updatedAt,
);
}
async getBroadcastChannel(channelId: string): Promise<BroadcastChannelRecord | null> {
const row = this.stmts.getBroadcastChannel.get(channelId) as BroadcastChannelRow | undefined;
if (!row) return null;
return rowToBroadcastChannel(row);
}
async listBroadcastChannels(): Promise<BroadcastChannelRecord[]> {
const rows = this.stmts.listBroadcastChannels.all() as BroadcastChannelRow[];
return rows.map(rowToBroadcastChannel);
}
async removeBroadcastChannel(channelId: string): Promise<void> {
this.stmts.removeBroadcastChannelMembers.run(channelId);
this.stmts.removeBroadcastChannel.run(channelId);
}
async saveBroadcastMember(member: BroadcastMemberRecord): Promise<void> {
this.stmts.saveBroadcastMember.run(
member.channelId,
member.peerAddress,
member.joinedAt,
member.removedAt,
);
}
async getBroadcastMembers(channelId: string): Promise<BroadcastMemberRecord[]> {
const rows = this.stmts.getBroadcastMembers.all(channelId) as BroadcastMemberRow[];
return rows.map((r) => ({
channelId: r.channel_id,
peerAddress: r.peer_address,
joinedAt: Number(r.joined_at),
removedAt: r.removed_at === null || r.removed_at === undefined ? null : Number(r.removed_at),
}));
}
async removeBroadcastMember(channelId: string, peerAddress: string): Promise<void> {
this.stmts.removeBroadcastMember.run(channelId, peerAddress);
}
}
interface BroadcastChannelRow {
channel_id: string;
owner_role: 'sender' | 'receiver';
owner_address: string;
label: string | null;
generation: number | bigint;
chain_key: Uint8Array | ArrayBuffer;
iteration: number | bigint;
signing_public_key: Uint8Array | ArrayBuffer;
signing_private_key: Uint8Array | ArrayBuffer | null;
created_at: number | bigint;
updated_at: number | bigint;
}
interface BroadcastMemberRow {
channel_id: string;
peer_address: string;
joined_at: number | bigint;
removed_at: number | bigint | null;
}
function rowToBroadcastChannel(r: BroadcastChannelRow): BroadcastChannelRecord {
const out: BroadcastChannelRecord = {
channelId: r.channel_id,
ownerRole: r.owner_role,
ownerAddress: r.owner_address,
generation: Number(r.generation),
chainKey: toBytes(r.chain_key),
iteration: Number(r.iteration),
signingPublicKey: toBytes(r.signing_public_key),
createdAt: Number(r.created_at),
updatedAt: Number(r.updated_at),
};
if (r.label !== null && r.label !== undefined) out.label = r.label;
if (r.signing_private_key !== null && r.signing_private_key !== undefined) {
out.signingPrivateKey = toBytes(r.signing_private_key);
}
return out;
feat(files): @shade/files 0.3.0 — E2EE filesystem RPC primitive M-Files-1..6 land the full files-RPC layer + everything 0.3.0 needs to ship. Apps keep their own UI; this layer ships the typed RPC, the streams bridge for content I/O, and production hooks (rate limit, retention, fingerprint gate, metrics). @shade/files (NEW) - Standard ops: list/stat/mkdir/delete/move/read/write/getThumbnail with Zod-validated wire schemas + clean user-handler types. - Custom ops: typed via TypeScript declaration merging on CustomOpsMap + per-op Zod schemas; client.custom('app.foo', {...}) is fully typed. - Content I/O: inline (≤ 256 KiB plaintext) base64-in-RPC; streams (> 256 KiB) ride @shade/transfer via userMetadata.shadeFilesWriteId / shadeFilesReadStreamId correlation. Server-side TransformStream bridges accept inbound transfers immediately (engine rejects chunks that arrive before accept) and park the readable for the matching RPC. - Directory ops: walk(path, opts) async-iterable depth-first walker; uploadDirectory()/downloadDirectory() with bounded concurrency pool (default 4, cap 16), aggregated progress, abort. - Production hooks (callback-based, vendor-neutral): rate-limit (op + byte), idempotency cache (LRU + TTL + in-flight de-dupe), path policy (traversal + percent-decode hardening), fingerprint gate (required/optional/reject), pluggable Ed25519 sig verification with ±5 min replay window, onMetric sink (standard names). - React hooks (subpath @shade/files/react): ShadeFilesProvider, useShadeFiles, useFileList, useFileTransfer/Upload/Download. - Shade.files.serve(handler) + Shade.files.client(peer) high-level entrypoint in @shade/sdk; lazy + memoized; one handler per Shade. Wire format bump - @shade/proto wire VERSION 0x01 → 0x02. Length prefixes changed from u16 to u32. The previous u16 silently truncated payloads above 64 KiB — a hard correctness ceiling that blocked inline file ops up to 256 KiB. Wire-incompatible with 0.2.x peers; new sessions only. Cross-platform Kotlin port (android/shade-android) updated to match; test-vectors/wire-format.json regenerated. Concurrency safety - ShadeSessionManager.encrypt/.decrypt now run under per-peer mutex. Concurrent decryptions of the same peer raced ratchet state (manifested as sporadic "Failed to decrypt — wrong key or tampered data" under load — surfaced once concurrent uploadDirectory pumped many writes in flight). Encrypt was already serialized via Shade.send's encryptChains; decrypt is now serialized at the manager layer too. @shade/streams extension - StreamMetadata.userMetadata?: Record<string, string> for application-level key/value pairs that round-trip verbatim through stream-init plaintext. Used by @shade/files for write/read correlation; available to any consumer. @shade/sdk extension - Shade.files getter (lazy + memoized). - BackgroundHooks.onPruneFiles + periodic timer (default 5 min) + BackgroundTasks.setHook(name, fn) for runtime hook registration. Bundles in-flight 0.2.0 work - packages/shade-streams/, packages/shade-transfer/, related shade-sdk streams-bridge + shade-widgets transfer hooks were uncommitted prior to this session. Including them keeps the workspace consistent at 0.3.0 since @shade/files depends on them. Tests - 74 new tests in @shade/files (572 → 646 workspace pass; 0 fail; 3× stable). Coverage spans unit (inline-threshold + concurrency), integration (read-write inline + streams up to 1 MiB, walk + upload/download directory, custom-op, metrics, SDK namespace end-to-end), and security (tampered-envelope sig verification, replay window, fingerprint gate, rate-limit + quota). Release artifacts - All packages bumped to 0.3.0 via scripts/bump-version.ts. - scripts/publish-all.ts PACKAGES updated with shade-files in topological order (after shade-transfer, before shade-sdk). - bun run publish:dry clean (14 packed, 0 failed). - examples/08-files-browser/ — three-process CLI demo (prekey + Bob server + Alice CLI) covering list/stat/mkdir/delete/upload/download. - docs/files.md — full API + design doc. - CHANGELOG.md 0.3.0 entry. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-02 14:00:01 +02:00
}
function rowToStreamState(row: any): PersistedStreamState {
const out: PersistedStreamState = {
streamId: row.stream_id,
direction: row.direction,
peerAddress: row.peer_address,
status: row.status,
metadataJson: row.metadata_json,
partitionJson: row.partition_json,
laneStateJson: row.lane_state_json,
ioDescriptorJson: row.io_descriptor_json,
secretEnc: toBytes(row.secret_enc),
secretNonce: toBytes(row.secret_nonce),
createdAt: Number(row.created_at),
updatedAt: Number(row.updated_at),
};
if (row.overall_hash_state !== null && row.overall_hash_state !== undefined) {
out.overallHashState = row.overall_hash_state;
}
return out;
}
function toBytes(value: unknown): Uint8Array {
if (value instanceof Uint8Array) return value;
if (value instanceof ArrayBuffer) return new Uint8Array(value);
if (Array.isArray(value)) return new Uint8Array(value);
if (typeof value === 'string') {
return new Uint8Array(value.split('').map((c) => c.charCodeAt(0)));
}
throw new Error(`Unsupported BLOB representation: ${typeof value}`);
}