release(v4.0.0): Shade GA — V3.x consolidation + audit prep
Some checks failed
Test / test (push) Has been cancelled
Cross-platform vectors / TypeScript vectors (bun) (push) Has been cancelled
Cross-platform vectors / Kotlin vectors (gradle) (push) Has been cancelled
Docker build and publish / docker (push) Has been cancelled
Publish / publish (push) Has been cancelled

V3.1 → V3.12 consolidated and tagged for the first GA release. Wire
format unchanged from 0.4.x — 4.0 peers interoperate with 0.4.x peers
byte-for-byte. The version bump is semantic: audit-cycle complete,
opt-in surface fully exposed, threat model refreshed for every new
surface.

Highlights:
- All 24 @shade/* packages bumped to 4.0.0 in lockstep.
- CHANGELOG 4.0.0 section is the canonical manifest of what landed.
- THREAT-MODEL extended (§10 fingerprint gates, §11 WebRTC P2P, §12
  Web-Worker boundary) + residual-risks table refreshed.
- OpenAPI now covers all 27 routes: prekey, transfer, KT, inbox,
  bridge, observer, /metrics, /healthz, /ready.
- MIGRATION 0.3.x → 4.0 documented + smoke-tested against
  shade migrate-storage on a real SQLite DB.
- docs/audit/REVIEW-BUNDLE.md + SCOPE.md ready for external reviewer.
- scripts/soak.ts harness for the GA-stable 2-week soak window.
- All V*.md plans archived under docs/archive/ with Status: Done.
- Voice/Video carved out into V5.0; 4.0 audit focuses on the frozen
  non-realtime stack.

Tests: TS 1000/1000 + Kotlin 11/11 cross-platform vectors green.
Docker: gt.zyon.no/stian/shade-prekey:4.0.0 builds and reports
  version 4.0.0 on /health.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-03 18:35:35 +02:00
parent 8b055912b7
commit e6fdf31b49
298 changed files with 37909 additions and 256 deletions

View File

@@ -1,12 +1,13 @@
{
"name": "@shade/transfer",
"version": "0.3.0",
"version": "4.0.0",
"type": "module",
"main": "src/index.ts",
"types": "src/index.ts",
"dependencies": {
"@shade/core": "workspace:*",
"@shade/crypto-web": "workspace:*",
"@shade/observability": "workspace:*",
"@shade/proto": "workspace:*",
"@shade/streams": "workspace:*"
},

View File

@@ -19,6 +19,22 @@ import {
type StreamMetadata,
} from '@shade/streams';
import { decodeStreamChunk, inspectEnvelopeType } from '@shade/proto';
import {
ATTR_BYTES_BIN,
ATTR_DIRECTION,
ATTR_ERROR_CODE,
ATTR_LANE_COUNT,
ATTR_PARTITION,
ATTR_PEER_HASH,
ATTR_RESULT,
ATTR_RETRY_COUNT,
bytesBin,
laneCountBin,
NOOP_HOOK,
peerHash,
type ObservabilityHook,
type Span,
} from '@shade/observability';
import {
TransferAbortError,
TransferIntegrityError,
@@ -72,6 +88,12 @@ export interface TransferEngineDeps {
* random per-process key).
*/
deviceKey?: Uint8Array;
/**
* Optional observability hook. Spans are emitted for each upload and for
* each accepted incoming transfer. Defaults to a no-op when omitted.
* Use `withTracer()` from `@shade/observability` to plug in OTel.
*/
observability?: ObservabilityHook;
}
/**
@@ -90,8 +112,10 @@ export class TransferEngine {
>();
private readonly unsubscribeControl: () => void;
private readonly persister: Persister | null;
private readonly observability: ObservabilityHook;
constructor(private readonly deps: TransferEngineDeps) {
this.observability = deps.observability ?? NOOP_HOOK;
this.persister =
deps.resumeStore !== undefined
? new Persister(deps.crypto, deps.resumeStore, deps.deviceKey)
@@ -172,6 +196,13 @@ export class TransferEngine {
opts.onProgress,
opts.onEvent,
);
state.span = this.observability.startSpan('shade.transfer.upload', {
[ATTR_DIRECTION]: 'upload',
[ATTR_PEER_HASH]: peerHash(opts.to),
[ATTR_BYTES_BIN]: bytesBin(input.size ?? 0),
[ATTR_LANE_COUNT]: laneCountBin(lanes.length),
[ATTR_PARTITION]: partitionMode,
});
this.outgoing.set(streamId, state);
// Persist initial resume state BEFORE sending init, so a crash before
@@ -308,6 +339,13 @@ export class TransferEngine {
opts?.onProgress,
opts?.onEvent,
);
state.span = this.observability.startSpan('shade.transfer.upload.resume', {
[ATTR_DIRECTION]: 'upload',
[ATTR_PEER_HASH]: peerHash(peer),
[ATTR_BYTES_BIN]: bytesBin(input.size ?? metadata.sizeBytes ?? 0),
[ATTR_LANE_COUNT]: laneCountBin(lanes.length),
[ATTR_PARTITION]: partitionMode,
});
// Mark already-shipped lanes as having advanced.
for (const lane of lanes) {
const lp = (state as unknown as { laneProgress: Map<number, LaneProgress> }).laneProgress.get(
@@ -615,6 +653,13 @@ export class TransferEngine {
if (opts.onProgress !== undefined) state.onProgress = opts.onProgress;
if (opts.onEvent !== undefined) state.onEvent = opts.onEvent;
state.startedAt = nowMs();
state.span = this.observability.startSpan('shade.transfer.download', {
[ATTR_DIRECTION]: 'download',
[ATTR_PEER_HASH]: peerHash(from),
[ATTR_BYTES_BIN]: bytesBin(msg.metadata.sizeBytes ?? 0),
[ATTR_LANE_COUNT]: laneCountBin(msg.lanes.length),
[ATTR_PARTITION]: msg.lanes[0]?.partition.kind ?? 'unknown',
});
state.emit({ type: 'start', streamId: msg.streamId });
return state.handle;
},
@@ -843,6 +888,8 @@ interface ChunkJob {
class OutgoingState {
startedAt = -1;
aborted = false;
span: Span | null = null;
private retryCount = 0;
private completedAt = -1;
private resolveDone!: (result: TransferResult) => void;
private rejectDone!: (err: unknown) => void;
@@ -942,6 +989,16 @@ class OutgoingState {
this.completedAt = nowMs();
this.emit({ type: 'complete', streamId: result.streamId, sha256: result.sha256, durationMs: result.durationMs });
this.flushEventEnd();
if (this.span !== null) {
this.span.setAttributes({
[ATTR_RESULT]: 'ok',
[ATTR_BYTES_BIN]: bytesBin(result.bytesSent),
[ATTR_RETRY_COUNT]: this.retryCount,
});
this.span.setStatus('ok');
this.span.end();
this.span = null;
}
this.resolveDone(result);
void this.input.close();
this.sender.destroy();
@@ -952,6 +1009,18 @@ class OutgoingState {
this.aborted = true;
this.emit({ type: 'error', error: err });
this.flushEventEnd();
if (this.span !== null) {
const code = errorCodeOf(err);
this.span.setAttributes({
[ATTR_RESULT]: 'error',
[ATTR_ERROR_CODE]: code,
[ATTR_RETRY_COUNT]: this.retryCount,
});
this.span.recordException(err);
this.span.setStatus('error', code);
this.span.end();
this.span = null;
}
try {
await this.input.close();
} catch {
@@ -961,6 +1030,10 @@ class OutgoingState {
this.rejectDone(err);
}
recordRetry(): void {
this.retryCount++;
}
private flushEventEnd(): void {
for (const r of this.eventResolvers) r({ value: undefined as never, done: true });
this.eventResolvers = [];
@@ -990,6 +1063,7 @@ class IncomingState {
startedAt = -1;
accepted = false;
declined = false;
span: Span | null = null;
sink: OutputSink | null = null;
outputKind: 'pipe' | 'callback' | 'buffer' | 'file' | 'fileHandle' | undefined;
onProgress: ((p: TransferProgress) => void) | undefined;
@@ -1224,6 +1298,15 @@ class IncomingState {
this.completedAt = nowMs();
this.emit({ type: 'complete', streamId: result.streamId, sha256: result.sha256, durationMs: result.durationMs });
this.flushEventEnd();
if (this.span !== null) {
this.span.setAttributes({
[ATTR_RESULT]: 'ok',
[ATTR_BYTES_BIN]: bytesBin(result.bytesSent),
});
this.span.setStatus('ok');
this.span.end();
this.span = null;
}
this.resolveDone(result);
this.receiver.destroy();
}
@@ -1234,6 +1317,15 @@ class IncomingState {
if (this.sink !== null) await this.sink.abort(reason);
this.emit({ type: 'abort', reason });
this.flushEventEnd();
if (this.span !== null) {
this.span.setAttributes({
[ATTR_RESULT]: 'abort',
[ATTR_ERROR_CODE]: 'SHADE_TRANSFER_ABORT',
});
this.span.setStatus('error', reason);
this.span.end();
this.span = null;
}
this.receiver.destroy();
this.rejectDone(new TransferAbortError(reason));
}
@@ -1253,6 +1345,17 @@ class IncomingState {
this.aborted = true;
this.emit({ type: 'error', error: err });
this.flushEventEnd();
if (this.span !== null) {
const code = errorCodeOf(err);
this.span.setAttributes({
[ATTR_RESULT]: 'error',
[ATTR_ERROR_CODE]: code,
});
this.span.recordException(err);
this.span.setStatus('error', code);
this.span.end();
this.span = null;
}
if (this.sink !== null) await this.sink.abort('integrity-failure');
this.receiver.destroy();
this.rejectDone(err);
@@ -1357,6 +1460,17 @@ function nowMs(): number {
return typeof performance !== 'undefined' ? performance.now() : Date.now();
}
function errorCodeOf(err: unknown): string {
if (err === null || err === undefined) return 'SHADE_UNKNOWN';
if (typeof err === 'object') {
const code = (err as { code?: unknown }).code;
if (typeof code === 'string' && code.length > 0) return code;
const name = (err as { name?: unknown }).name;
if (typeof name === 'string' && name.startsWith('Transfer')) return `SHADE_${name}`;
}
return 'SHADE_UNKNOWN';
}
// ─── Persister ───────────────────────────────────────────────
class Persister {

View File

@@ -6,6 +6,7 @@ export * from './transport/transport.js';
export * from './transport/memory.js';
export * from './transport/http-transport.js';
export * from './transport/ws-transport.js';
export * from './transport/multi-fallback.js';
export * from './engine.js';
export {
createTransferRoutes,

View File

@@ -0,0 +1,148 @@
/**
* N-ary `ITransferTransport` fallback chain.
*
* Generalises the two-arg {@link FallbackTransferTransport} so a Shade
* client can wire `[WebRTC, WebSocket, HTTP]` and have the engine try
* them in order — switching sticky once a transport raises a
* `TransferTransportError`.
*
* The chain advances exactly once per failure and never tries to "fall
* back up" — once HTTP wins, P2P stops being attempted for that
* `MultiTransportFallback` instance. Re-create the wrapper to re-try
* the upper layers (which is what the SDK does on `configureWebRTC()`).
*/
import { TransferTransportError } from '../errors.js';
import type {
ChunkAck,
ChunkSendOptions,
ITransferTransport,
TransferResumeState,
} from './transport.js';
export interface NamedTransport {
name: string;
transport: ITransferTransport;
}
export class MultiTransportFallback implements ITransferTransport {
/** Index into `transports` of the currently-active layer. */
private cursor = 0;
/** Recorded failures per name — purely diagnostic. */
private readonly failureLog: Array<{ name: string; error: string }> = [];
private readonly switchListeners = new Set<(from: string, to: string) => void>();
constructor(private readonly transports: NamedTransport[]) {
if (transports.length === 0) {
throw new Error('MultiTransportFallback: must supply at least one transport');
}
}
/** Name of the currently-active transport. */
get activeName(): string {
return this.transports[this.cursor]!.name;
}
/** Has the chain demoted at least once? */
get hasFallenBack(): boolean {
return this.cursor > 0;
}
/** Diagnostic snapshot of the failure log. */
get failures(): readonly { name: string; error: string }[] {
return this.failureLog;
}
onSwitch(cb: (from: string, to: string) => void): () => void {
this.switchListeners.add(cb);
return () => this.switchListeners.delete(cb);
}
async probe(peerAddress: string): Promise<void> {
while (this.cursor < this.transports.length) {
const layer = this.transports[this.cursor]!;
try {
await layer.transport.probe(peerAddress);
return;
} catch (err) {
if (this.demoteIfTransportError(layer.name, err)) {
continue;
}
throw err;
}
}
throw new TransferTransportError(
`MultiTransportFallback: all ${this.transports.length} transports unreachable`,
);
}
async sendChunk(
peerAddress: string,
streamId: string,
laneId: number,
seq: number | bigint,
bytes: Uint8Array,
options?: ChunkSendOptions,
): Promise<ChunkAck> {
while (this.cursor < this.transports.length) {
const layer = this.transports[this.cursor]!;
try {
return await layer.transport.sendChunk(
peerAddress,
streamId,
laneId,
seq,
bytes,
options,
);
} catch (err) {
if (this.demoteIfTransportError(layer.name, err)) {
continue;
}
throw err;
}
}
throw new TransferTransportError(
`MultiTransportFallback: sendChunk failed across all ${this.transports.length} transports`,
);
}
async fetchResumeState(
peerAddress: string,
streamId: string,
): Promise<TransferResumeState | null> {
while (this.cursor < this.transports.length) {
const layer = this.transports[this.cursor]!;
try {
return await layer.transport.fetchResumeState(peerAddress, streamId);
} catch (err) {
if (this.demoteIfTransportError(layer.name, err)) {
continue;
}
throw err;
}
}
return null;
}
private demoteIfTransportError(name: string, err: unknown): boolean {
if (!(err instanceof TransferTransportError)) return false;
this.failureLog.push({ name, error: (err as Error).message });
const next = this.cursor + 1;
if (next >= this.transports.length) {
// Already on the last layer — re-throw upstream.
return false;
}
const prevName = this.transports[this.cursor]!.name;
this.cursor = next;
const newName = this.transports[this.cursor]!.name;
for (const cb of this.switchListeners) {
try {
cb(prevName, newName);
} catch (cbErr) {
console.warn('[MultiTransportFallback] onSwitch callback threw:', cbErr);
}
}
return true;
}
}

View File

@@ -0,0 +1,115 @@
import { describe, expect, it } from 'bun:test';
import {
MultiTransportFallback,
TransferTransportError,
type ChunkAck,
type ITransferTransport,
type TransferResumeState,
} from '../src/index.js';
class StubTransport implements ITransferTransport {
callCount = 0;
constructor(
private readonly behavior: () => 'ok' | 'transport-error' | 'other-error',
) {}
async probe(): Promise<void> {
this.callCount++;
const verdict = this.behavior();
if (verdict === 'transport-error') throw new TransferTransportError('probe failed');
if (verdict === 'other-error') throw new Error('other');
}
async sendChunk(
_peer: string,
_streamId: string,
_laneId: number,
seq: number | bigint,
): Promise<ChunkAck> {
this.callCount++;
const verdict = this.behavior();
if (verdict === 'transport-error') throw new TransferTransportError('send failed');
if (verdict === 'other-error') throw new Error('other');
return { lastSeq: typeof seq === 'bigint' ? Number(seq) : seq };
}
async fetchResumeState(): Promise<TransferResumeState | null> {
this.callCount++;
const verdict = this.behavior();
if (verdict === 'transport-error') throw new TransferTransportError('resume failed');
return null;
}
}
describe('MultiTransportFallback', () => {
it('uses the primary transport when probe succeeds', async () => {
const primary = new StubTransport(() => 'ok');
const secondary = new StubTransport(() => 'ok');
const tertiary = new StubTransport(() => 'ok');
const fb = new MultiTransportFallback([
{ name: 'webrtc', transport: primary },
{ name: 'ws', transport: secondary },
{ name: 'http', transport: tertiary },
]);
await fb.probe('bob');
expect(fb.activeName).toBe('webrtc');
expect(primary.callCount).toBe(1);
expect(secondary.callCount).toBe(0);
});
it('demotes through layers on transport errors', async () => {
const primary = new StubTransport(() => 'transport-error');
const secondary = new StubTransport(() => 'transport-error');
const tertiary = new StubTransport(() => 'ok');
const fb = new MultiTransportFallback([
{ name: 'webrtc', transport: primary },
{ name: 'ws', transport: secondary },
{ name: 'http', transport: tertiary },
]);
const switches: Array<{ from: string; to: string }> = [];
fb.onSwitch((from, to) => switches.push({ from, to }));
await fb.probe('bob');
expect(fb.activeName).toBe('http');
expect(switches).toEqual([
{ from: 'webrtc', to: 'ws' },
{ from: 'ws', to: 'http' },
]);
expect(fb.failures).toHaveLength(2);
});
it('throws if every layer fails', async () => {
const fb = new MultiTransportFallback([
{ name: 'a', transport: new StubTransport(() => 'transport-error') },
{ name: 'b', transport: new StubTransport(() => 'transport-error') },
]);
await expect(fb.probe('bob')).rejects.toThrow();
});
it('does NOT demote on non-transport errors', async () => {
const primary = new StubTransport(() => 'other-error');
const fb = new MultiTransportFallback([
{ name: 'p', transport: primary },
{ name: 's', transport: new StubTransport(() => 'ok') },
]);
await expect(fb.probe('bob')).rejects.toThrow(/other/);
// We did NOT advance — non-transport errors are caller bugs.
expect(fb.activeName).toBe('p');
});
it('sticks to the demoted layer for sendChunk after probe failure', async () => {
const primary = new StubTransport(() => 'transport-error');
const secondary = new StubTransport(() => 'ok');
const fb = new MultiTransportFallback([
{ name: 'p', transport: primary },
{ name: 's', transport: secondary },
]);
await fb.probe('bob');
expect(fb.activeName).toBe('s');
// primary not called again
const before = primary.callCount;
await fb.sendChunk('bob', 'sid', 0, 0n, new Uint8Array(8));
expect(primary.callCount).toBe(before);
expect(secondary.callCount).toBeGreaterThan(0);
});
it('rejects empty transport list', () => {
expect(() => new MultiTransportFallback([])).toThrow();
});
});

View File

@@ -0,0 +1,126 @@
import { describe, expect, test } from 'bun:test';
import { SubtleCryptoProvider } from '@shade/crypto-web';
import { createRecorder } from '@shade/observability';
import {
TransferEngine,
MemoryControlChannel,
MemoryTransferTransport,
type IncomingTransfer,
type TransferHandle,
type TransferResult,
} from '../src/index.js';
const crypto = new SubtleCryptoProvider();
const ALICE = 'alice@trace-test.local';
const BOB = 'bob@trace-test.local';
const SECRET_PAYLOAD = new TextEncoder().encode(
'classified-shadow-token-DO-NOT-LOG',
);
function makePair(observability: ReturnType<typeof createRecorder>) {
const { a: ctrlA, b: ctrlB } = MemoryControlChannel.linked(ALICE, BOB);
const { a: txA, b: txB } = MemoryTransferTransport.linked(ALICE, BOB);
const sender = new TransferEngine({
crypto,
controlChannel: ctrlA,
transport: txA,
myAddress: ALICE,
observability,
});
const receiver = new TransferEngine({
crypto,
controlChannel: ctrlB,
transport: txB,
myAddress: BOB,
observability,
});
txB.setChunkHandler(async (from, sid, lane, seq, bytes) =>
receiver.receiveChunk(from, sid, lane, seq, bytes),
);
return { sender, receiver };
}
describe('TransferEngine observability', () => {
test('emits upload+download spans with PII-safe attributes', async () => {
const rec = createRecorder();
const { sender, receiver } = makePair(rec);
let resolveRecv!: (h: TransferHandle) => void;
const recvP = new Promise<TransferHandle>((r) => { resolveRecv = r; });
const unsub = receiver.onIncomingTransfer(async (incoming: IncomingTransfer) => {
const h = await incoming.accept({ output: { kind: 'buffer' } });
resolveRecv(h);
});
const handle = await sender.upload({
to: BOB,
input: SECRET_PAYLOAD,
lanes: 1,
chunkSize: 4096,
});
const recvH = await recvP;
await Promise.all([handle.done(), recvH.done()]);
unsub();
const upload = rec.spans.find((s) => s.name === 'shade.transfer.upload');
const download = rec.spans.find((s) => s.name === 'shade.transfer.download');
expect(upload).toBeDefined();
expect(download).toBeDefined();
expect(upload!.attributes['shade.direction']).toBe('upload');
expect(upload!.attributes['shade.peer.hash']).toMatch(/^[0-9a-f]{8}$/);
expect(upload!.attributes['shade.bytes.bin']).toBe('≤4KB');
expect(upload!.attributes['shade.lane.count']).toBe(1);
expect(upload!.attributes['shade.result']).toBe('ok');
expect(upload!.status).toBe('ok');
expect(upload!.ended).toBe(true);
expect(download!.attributes['shade.direction']).toBe('download');
expect(download!.attributes['shade.peer.hash']).toMatch(/^[0-9a-f]{8}$/);
expect(download!.attributes['shade.result']).toBe('ok');
expect(download!.ended).toBe(true);
// PII guard: no plaintext addresses, no payload, no exact byte counts.
const hits = rec.scanForPII([
ALICE,
BOB,
'trace-test',
'classified-shadow-token',
String(SECRET_PAYLOAD.length),
]);
if (hits.length > 0) {
throw new Error(`PII leak: ${JSON.stringify(hits, null, 2)}`);
}
});
test('failed upload marks span as error', async () => {
const rec = createRecorder();
const { sender } = makePair(rec);
// No receiver wired, so probe will succeed but accept() never fires.
// Force failure by aborting before the upload completes.
const ac = new AbortController();
const handle = await sender.upload({
to: BOB,
input: new Uint8Array(8),
lanes: 1,
chunkSize: 4,
signal: ac.signal,
});
ac.abort();
await handle.done().catch(() => undefined);
const upload = rec.spans.find((s) => s.name === 'shade.transfer.upload');
expect(upload).toBeDefined();
expect(upload!.ended).toBe(true);
// Either resulted in error or completed cleanly before abort took effect.
// We only assert that attributes never echoed peer addresses.
const hits = rec.scanForPII([ALICE, BOB, 'trace-test']);
if (hits.length > 0) {
throw new Error(`PII leak (failure path): ${JSON.stringify(hits, null, 2)}`);
}
});
});
// Type-level guard so TransferResult import isn't unused.
const _t: TransferResult | null = null;
void _t;