release(v4.0.0): Shade GA — V3.x consolidation + audit prep

V3.1 → V3.12 consolidated and tagged for the first GA release. Wire
format unchanged from 0.4.x — 4.0 peers interoperate with 0.4.x peers
byte-for-byte. The version bump is semantic: audit-cycle complete,
opt-in surface fully exposed, threat model refreshed for every new
surface.

Highlights:
- All 24 @shade/* packages bumped to 4.0.0 in lockstep.
- CHANGELOG 4.0.0 section is the canonical manifest of what landed.
- THREAT-MODEL extended (§10 fingerprint gates, §11 WebRTC P2P, §12
  Web-Worker boundary) + residual-risks table refreshed.
- OpenAPI now covers all 27 routes: prekey, transfer, KT, inbox,
  bridge, observer, /metrics, /healthz, /ready.
- MIGRATION 0.3.x → 4.0 documented + smoke-tested against
  shade migrate-storage on a real SQLite DB.
- docs/audit/REVIEW-BUNDLE.md + SCOPE.md ready for external reviewer.
- scripts/soak.ts harness for the GA-stable 2-week soak window.
- All V*.md plans archived under docs/archive/ with Status: Done.
- Voice/Video carved out into V5.0; 4.0 audit focuses on the frozen
  non-realtime stack.

Tests: TS 1000/1000 + Kotlin 11/11 cross-platform vectors green.
Docker: gt.zyon.no/stian/shade-prekey:4.0.0 builds and reports
  version 4.0.0 on /health.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

scripts/Deprecated/publish-all.ts

@@ -16,17 +16,24 @@ import { readFileSync, writeFileSync, existsSync } from 'fs';
 import { join } from 'path';
 import { $ } from 'bun';
 
+// Order matters: each package only depends on packages above it. Publishing
+// in this order means a consumer fetching mid-publish never sees a manifest
+// pointing at an unpublished version.
 const PACKAGES = [
   'shade-core',
-  'shade-crypto-web',
   'shade-proto',
+  'shade-crypto-web',
   'shade-storage-sqlite',
   'shade-storage-postgres',
-  'shade-server',
-  'shade-observer',
+  'shade-streams',
   'shade-transport',
-  'shade-widgets',
+  'shade-server',
+  'shade-transfer',
+  'shade-files',
+  'shade-recovery',
+  'shade-observer',
   'shade-sdk',
+  'shade-widgets',
   'shade-cli',
 ];
 
@@ -71,6 +78,8 @@ async function main() {
 
   let published = 0;
   let skipped = 0;
+  let alreadyPublished = 0;
+  let failed = 0;
 
   for (const pkg of PACKAGES) {
     const pkgDir = join(ROOT, 'packages', pkg);
@@ -97,8 +106,20 @@ async function main() {
       published++;
       console.log(`  ✓ ${dryRun ? 'packed' : 'published'}`);
     } catch (err) {
-      console.error(`  ✗ failed: ${(err as Error).message}`);
-      process.exitCode = 1;
+      const out = `${err instanceof Error ? err.message : String(err)} ${
+        (err as { stderr?: { toString(): string } }).stderr?.toString() ?? ''
+      } ${(err as { stdout?: { toString(): string } }).stdout?.toString() ?? ''}`;
+      // Gitea (and npm) report already-published versions as 409 / EPUBLISHCONFLICT.
+      // Skip silently rather than failing the whole run — bumping the version
+      // is the user's explicit decision via `bun run version <new>`.
+      if (/409|EPUBLISHCONFLICT|already exists|already been published/i.test(out)) {
+        alreadyPublished++;
+        console.log(`  ⊙ already published — skipping`);
+      } else {
+        failed++;
+        console.error(`  ✗ failed: ${(err as Error).message}`);
+        process.exitCode = 1;
+      }
     } finally {
       // Always restore the original package.json so the workspace stays usable
       // for `bun install` after publish, regardless of success or failure.
@@ -112,7 +133,9 @@ async function main() {
   } catch {}
 
   console.log();
-  console.log(`Done: ${published} published, ${skipped} skipped`);
+  console.log(
+    `Done: ${published} ${dryRun ? 'packed' : 'published'}, ${alreadyPublished} already published, ${skipped} skipped, ${failed} failed`,
+  );
 }
 
 /**

scripts/generate-vectors.ts

@@ -5,15 +5,29 @@
  * The output JSON files are loaded by BOTH the TypeScript and Kotlin test
  * suites. Any divergence between platforms fails CI immediately.
  *
+ * Schema: every file is `{ "version": <int>, "vectors": [...] }`.
+ * Bump `VECTOR_FILE_VERSION` whenever the vector schema (NOT just values)
+ * changes, so downstream consumers can fail loudly on mismatch.
+ *
  * Usage: bun run scripts/generate-vectors.ts
  */
 import { writeFileSync } from 'fs';
 import { join } from 'path';
-import { SubtleCryptoProvider, MemoryStorage } from '../packages/shade-crypto-web/src/index.js';
+import { SubtleCryptoProvider } from '../packages/shade-crypto-web/src/index.js';
 import { computeFingerprint } from '../packages/shade-core/src/fingerprint.js';
 import { kdfChainKey, kdfRootKey, deriveInitialRootKey } from '../packages/shade-core/src/keys.js';
-import { encodeEnvelope, decodeEnvelope } from '../packages/shade-proto/src/index.js';
-import type { ShadeEnvelope, RatchetMessage } from '../packages/shade-core/src/index.js';
+import { encodeEnvelope, encodeStreamChunk, decodeStreamChunk } from '../packages/shade-proto/src/index.js';
+import type { StreamChunkWire } from '../packages/shade-proto/src/index.js';
+import type { ShadeEnvelope, RatchetMessage, PreKeyMessage } from '../packages/shade-core/src/index.js';
+import {
+  deriveStreamKey,
+  deriveLaneKey,
+  buildChunkNonce,
+  buildChunkAad,
+  aesGcmEncryptWithNonce,
+} from '../packages/shade-streams/src/index.js';
+
+const VECTOR_FILE_VERSION = 2;
 
 const crypto = new SubtleCryptoProvider();
 const OUT_DIR = join(import.meta.dir, '..', 'test-vectors');
@@ -32,20 +46,58 @@ function fromHex(str: string): Uint8Array {
 
 interface Vector {
   description: string;
-  [key: string]: any;
+  [key: string]: unknown;
+}
+
+// AES-GCM with caller-supplied nonce. The CryptoProvider interface picks a
+// random nonce internally, so vector generation goes around it via SubtleCrypto
+// directly — same primitive `@shade/streams` already uses.
+async function aesGcmEncryptDeterministic(
+  key: Uint8Array,
+  nonce: Uint8Array,
+  plaintext: Uint8Array,
+  aad: Uint8Array,
+): Promise<Uint8Array> {
+  const subtle = globalThis.crypto.subtle;
+  const aesKey = await subtle.importKey(
+    'raw',
+    key as unknown as ArrayBuffer,
+    'AES-GCM',
+    false,
+    ['encrypt'],
+  );
+  const out = await subtle.encrypt(
+    { name: 'AES-GCM', iv: nonce as unknown as ArrayBuffer, additionalData: aad as unknown as ArrayBuffer },
+    aesKey,
+    plaintext as unknown as ArrayBuffer,
+  );
+  return new Uint8Array(out);
+}
+
+// Mirror of `encodeHeader` in @shade/core/ratchet.ts — kept inline to avoid
+// exporting an internal symbol just for tests.
+function encodeRatchetHeader(
+  dhPublicKey: Uint8Array,
+  previousCounter: number,
+  counter: number,
+): Uint8Array {
+  const buf = new Uint8Array(40);
+  buf.set(dhPublicKey, 0);
+  const view = new DataView(buf.buffer);
+  view.setUint32(32, previousCounter, false);
+  view.setUint32(36, counter, false);
+  return buf;
 }
 
 // ─── HKDF vectors ───────────────────────────────────────────
 async function generateHkdfVectors(): Promise<Vector[]> {
-  const vectors: Vector[] = [];
-
-  // Known inputs → expected outputs
   const cases = [
     { ikm: '01'.repeat(32), salt: '02'.repeat(32), info: 'test', length: 32 },
     { ikm: 'ab'.repeat(32), salt: '00'.repeat(32), info: 'ShadeRootRatchet', length: 64 },
     { ikm: 'cd'.repeat(32), salt: '00'.repeat(32), info: 'ShadeX3DH', length: 32 },
   ];
 
+  const vectors: Vector[] = [];
   for (const c of cases) {
     const out = await crypto.hkdf(
       fromHex(c.ikm),
@@ -62,7 +114,6 @@ async function generateHkdfVectors(): Promise<Vector[]> {
       output: hex(out),
     });
   }
-
   return vectors;
 }
 
@@ -135,13 +186,11 @@ async function generateFingerprintVectors(): Promise<Vector[]> {
       fingerprint: fp,
     });
   }
-
   return vectors;
 }
 
 // ─── Wire format vectors ───────────────────────────────────
 async function generateWireFormatVectors(): Promise<Vector[]> {
-  // Deterministic inputs
   const ratchetMsg: RatchetMessage = {
     dhPublicKey: new Uint8Array(32).fill(0x11),
     previousCounter: 42,
@@ -149,7 +198,6 @@ async function generateWireFormatVectors(): Promise<Vector[]> {
     ciphertext: new Uint8Array(16).fill(0x22),
     nonce: new Uint8Array(12).fill(0x33),
   };
-
   const envelopeRatchet: ShadeEnvelope = {
     type: 'ratchet',
     content: ratchetMsg,
@@ -158,9 +206,48 @@ async function generateWireFormatVectors(): Promise<Vector[]> {
   };
   const bytesRatchet = encodeEnvelope(envelopeRatchet);
 
+  const innerRatchet: RatchetMessage = {
+    dhPublicKey: new Uint8Array(32).fill(0x44),
+    previousCounter: 0,
+    counter: 0,
+    ciphertext: new Uint8Array(8).fill(0x55),
+    nonce: new Uint8Array(12).fill(0x66),
+  };
+  const preKeyMsgWithOTPK: PreKeyMessage = {
+    registrationId: 0x12345678,
+    preKeyId: 99,
+    signedPreKeyId: 1,
+    ephemeralKey: new Uint8Array(32).fill(0x77),
+    identityDHKey: new Uint8Array(32).fill(0x88),
+    message: innerRatchet,
+  };
+  const envelopePreKey: ShadeEnvelope = {
+    type: 'prekey',
+    content: preKeyMsgWithOTPK,
+    timestamp: 0,
+    senderAddress: '',
+  };
+  const bytesPreKey = encodeEnvelope(envelopePreKey);
+
+  const preKeyMsgNoOTPK: PreKeyMessage = {
+    registrationId: 1,
+    preKeyId: undefined,
+    signedPreKeyId: 1,
+    ephemeralKey: new Uint8Array(32).fill(0x99),
+    identityDHKey: new Uint8Array(32).fill(0xaa),
+    message: innerRatchet,
+  };
+  const bytesPreKeyNoOTPK = encodeEnvelope({
+    type: 'prekey',
+    content: preKeyMsgNoOTPK,
+    timestamp: 0,
+    senderAddress: '',
+  });
+
   return [
     {
-      description: 'Wire format: RatchetMessage encoding',
+      description: 'Wire format: RatchetMessage encoding (wire VERSION 0x02 — u32 length-prefixed)',
+      kind: 'ratchet',
       message: {
         dhPublicKey: hex(ratchetMsg.dhPublicKey),
         previousCounter: ratchetMsg.previousCounter,
@@ -170,24 +257,423 @@ async function generateWireFormatVectors(): Promise<Vector[]> {
       },
       encoded: hex(bytesRatchet),
     },
+    {
+      description: 'Wire format: PreKeyMessage with one-time prekey (wire 0x02 type 0x01)',
+      kind: 'prekey',
+      message: {
+        registrationId: preKeyMsgWithOTPK.registrationId,
+        preKeyId: preKeyMsgWithOTPK.preKeyId ?? null,
+        signedPreKeyId: preKeyMsgWithOTPK.signedPreKeyId,
+        ephemeralKey: hex(preKeyMsgWithOTPK.ephemeralKey),
+        identityDHKey: hex(preKeyMsgWithOTPK.identityDHKey),
+        inner: {
+          dhPublicKey: hex(innerRatchet.dhPublicKey),
+          previousCounter: innerRatchet.previousCounter,
+          counter: innerRatchet.counter,
+          ciphertext: hex(innerRatchet.ciphertext),
+          nonce: hex(innerRatchet.nonce),
+        },
+      },
+      encoded: hex(bytesPreKey),
+    },
+    {
+      description: 'Wire format: PreKeyMessage without one-time prekey (preKeyId=null encoded as 0xFFFFFFFF)',
+      kind: 'prekey',
+      message: {
+        registrationId: preKeyMsgNoOTPK.registrationId,
+        preKeyId: null,
+        signedPreKeyId: preKeyMsgNoOTPK.signedPreKeyId,
+        ephemeralKey: hex(preKeyMsgNoOTPK.ephemeralKey),
+        identityDHKey: hex(preKeyMsgNoOTPK.identityDHKey),
+        inner: {
+          dhPublicKey: hex(innerRatchet.dhPublicKey),
+          previousCounter: innerRatchet.previousCounter,
+          counter: innerRatchet.counter,
+          ciphertext: hex(innerRatchet.ciphertext),
+          nonce: hex(innerRatchet.nonce),
+        },
+      },
+      encoded: hex(bytesPreKeyNoOTPK),
+    },
+  ];
+}
+
+// ─── Ratchet step vectors ──────────────────────────────────
+//
+// A ratchet "encrypt step" is fully deterministic given (rootKey, dhSendPriv,
+// dhRemotePub, plaintext, fixed nonce, counters). The vector records every
+// intermediate derivation so each implementation can verify byte-parity at
+// every layer (kdfRootKey → kdfChainKey → header AAD → AES-GCM ciphertext) and
+// also verify decrypt(ciphertext, nonce, aad, messageKey) === plaintext.
+async function generateRatchetStepVectors(): Promise<Vector[]> {
+  // Deterministic inputs
+  const rootKey = new Uint8Array(32).fill(0xa1);
+  const dhSendPriv = new Uint8Array(32).fill(0xb2);
+  const dhSendPub = new Uint8Array(32).fill(0xb3); // not used in derivation, only AAD
+  const dhRemotePub = new Uint8Array(32).fill(0xc4);
+  const plaintext = new TextEncoder().encode('Shade ratchet roundtrip vector v1');
+  const fixedNonce = new Uint8Array(12).fill(0x5e);
+  const previousCounter = 2;
+  const counter = 0;
+
+  // Step 1: DH between local send priv and remote pub
+  const dhOutput = await crypto.x25519(dhSendPriv, dhRemotePub);
+
+  // Step 2: kdfRootKey to advance root + get chain key
+  const { newRootKey, chainKey } = await kdfRootKey(crypto, rootKey, dhOutput);
+
+  // Step 3: kdfChainKey to derive messageKey
+  const { newChainKey, messageKey } = await kdfChainKey(crypto, chainKey);
+
+  // Step 4: Header AAD bytes
+  const aad = encodeRatchetHeader(dhSendPub, previousCounter, counter);
+
+  // Step 5: AES-GCM with deterministic nonce
+  const ciphertext = await aesGcmEncryptDeterministic(messageKey, fixedNonce, plaintext, aad);
+
+  return [
+    {
+      description: 'Ratchet step: deterministic encrypt (kdfRootKey + kdfChainKey + AES-GCM with fixed nonce)',
+      inputs: {
+        rootKey: hex(rootKey),
+        dhSendPrivateKey: hex(dhSendPriv),
+        dhSendPublicKey: hex(dhSendPub),
+        dhRemotePublicKey: hex(dhRemotePub),
+        previousCounter,
+        counter,
+        plaintext: hex(plaintext),
+        nonce: hex(fixedNonce),
+      },
+      derived: {
+        dhOutput: hex(dhOutput),
+        newRootKey: hex(newRootKey),
+        chainKey: hex(chainKey),
+        newChainKey: hex(newChainKey),
+        messageKey: hex(messageKey),
+        aad: hex(aad),
+      },
+      ciphertext: hex(ciphertext),
+    },
+  ];
+}
+
+// ─── Streams 0x11 vectors ──────────────────────────────────
+//
+// Covers the @shade/streams primitives that V3.5 §3 (M-Cross 3) requires
+// Kotlin to mirror byte-for-byte: HKDF labels with embedded NULs, u32-be
+// laneId encoding inside the lane-key info, deterministic (laneId, seq)
+// chunk nonces, the 29-byte chunk AAD, end-to-end chunk encrypt/decrypt,
+// and the wire 0x11 envelope encode/decode.
+async function generateStreamsVectors(): Promise<Vector[]> {
+  const streamSecret = new Uint8Array(32).fill(0xa1);
+  const streamId = new Uint8Array(16).fill(0xb2);
+
+  const streamKey = await deriveStreamKey(crypto, streamSecret, streamId);
+
+  const laneIdsForKeys = [0, 1, 2, 0xffff_ffff];
+  const laneKeyVectors: Array<{ laneId: number; laneKey: string }> = [];
+  for (const laneId of laneIdsForKeys) {
+    const laneKey = await deriveLaneKey(crypto, streamKey, streamId, laneId);
+    laneKeyVectors.push({ laneId, laneKey: hex(laneKey) });
+  }
+
+  const noncePairs: Array<{ laneId: number; seq: bigint }> = [
+    { laneId: 0, seq: 0n },
+    { laneId: 0, seq: 1n },
+    { laneId: 1, seq: 0n },
+    { laneId: 0xffff_ffff, seq: 0xffff_ffff_ffff_fffen },
+  ];
+  const nonceVectors = noncePairs.map((p) => ({
+    laneId: p.laneId,
+    seq: p.seq.toString(),
+    nonce: hex(buildChunkNonce(p.laneId, p.seq)),
+  }));
+
+  const aadCases: Array<{ laneId: number; seq: bigint; isLast: boolean }> = [
+    { laneId: 0, seq: 0n, isLast: false },
+    { laneId: 1, seq: 7n, isLast: true },
+    { laneId: 0xffff_ffff, seq: 0xffff_ffff_ffff_fffen, isLast: false },
+  ];
+  const aadVectors = aadCases.map((c) => ({
+    laneId: c.laneId,
+    seq: c.seq.toString(),
+    isLast: c.isLast,
+    aad: hex(buildChunkAad(streamId, c.laneId, c.seq, c.isLast)),
+  }));
+
+  // End-to-end chunk encrypt with lane 0, seq 0, isLast=true
+  const laneId = 0;
+  const seq = 0n;
+  const isLast = true;
+  const laneKey = await deriveLaneKey(crypto, streamKey, streamId, laneId);
+  const nonce = buildChunkNonce(laneId, seq);
+  const aad = buildChunkAad(streamId, laneId, seq, isLast);
+  const plaintext = new TextEncoder().encode('Shade streams 0x11 chunk vector');
+  const ciphertext = await aesGcmEncryptWithNonce(laneKey, nonce, plaintext, aad);
+
+  // Wire 0x11 envelope (extra-aad field = 0 bytes per current spec)
+  const wire: StreamChunkWire = {
+    streamId,
+    laneId,
+    seq,
+    isLast,
+    nonce,
+    aad: new Uint8Array(0),
+    ciphertext,
+  };
+  const wireBytes = encodeStreamChunk(wire);
+  // Sanity: roundtrip-decode locally so the recorded bytes are always parseable
+  const decoded = decodeStreamChunk(wireBytes);
+  if (hex(decoded.ciphertext) !== hex(ciphertext)) {
+    throw new Error('streams wire 0x11 roundtrip diverged in generator');
+  }
+
+  return [
+    {
+      description: 'deriveStreamKey: HKDF(streamSecret, salt=streamId, info="shade-stream/v1\\0master")',
+      streamSecret: hex(streamSecret),
+      streamId: hex(streamId),
+      streamKey: hex(streamKey),
+    },
+    {
+      description: 'deriveLaneKey: HKDF(streamKey, salt=streamId, info="shade-stream/v1\\0lane\\0" || u32_be(laneId))',
+      streamKey: hex(streamKey),
+      streamId: hex(streamId),
+      lanes: laneKeyVectors,
+    },
+    {
+      description: 'buildChunkNonce(laneId, seq): u32_be(laneId) || u64_be(seq)',
+      nonces: nonceVectors,
+    },
+    {
+      description: 'buildChunkAad(streamId, laneId, seq, isLast): streamId(16) || u32_be(laneId) || u64_be(seq) || u8(isLast)',
+      streamId: hex(streamId),
+      cases: aadVectors,
+    },
+    {
+      description: 'End-to-end chunk encrypt: AES-256-GCM(laneKey, nonce, plaintext, aad)',
+      laneId,
+      seq: seq.toString(),
+      isLast,
+      laneKey: hex(laneKey),
+      nonce: hex(nonce),
+      aad: hex(aad),
+      plaintext: hex(plaintext),
+      ciphertext: hex(ciphertext),
+    },
+    {
+      description: 'Wire 0x11 stream-chunk envelope encode/decode',
+      streamId: hex(streamId),
+      laneId,
+      seq: seq.toString(),
+      isLast,
+      nonce: hex(nonce),
+      extraAad: hex(new Uint8Array(0)),
+      ciphertext: hex(ciphertext),
+      encoded: hex(wireBytes),
+    },
+  ];
+}
+
+// ─── Backup format vectors ─────────────────────────────────
+//
+// Backup v1 derives an AES-256-GCM key from `(passphrase, salt)` via
+// HKDF-SHA256 with info `"ShadeBackupKey"`, then encrypts the payload.
+// The vector pins the HKDF output and an end-to-end encrypt/decrypt for
+// a known plaintext + fixed nonce.
+async function generateBackupVectors(): Promise<Vector[]> {
+  const passphrase = 'correct-horse-battery-staple';
+  const salt = new Uint8Array(32).fill(0xa5);
+  const info = new TextEncoder().encode('ShadeBackupKey');
+  const backupKey = await crypto.hkdf(
+    new TextEncoder().encode(passphrase),
+    salt,
+    info,
+    32,
+  );
+  const plaintext = new TextEncoder().encode(
+    JSON.stringify({ version: 1, identity: null, sessions: [] }),
+  );
+  const fixedNonce = new Uint8Array(12).fill(0xc7);
+  const ciphertext = await aesGcmEncryptDeterministic(
+    backupKey,
+    fixedNonce,
+    plaintext,
+    new Uint8Array(0),
+  );
+  return [
+    {
+      description: 'Backup v1: HKDF(passphrase_utf8, salt, info="ShadeBackupKey", 32) -> backupKey',
+      passphrase,
+      salt: hex(salt),
+      info: 'ShadeBackupKey',
+      backupKey: hex(backupKey),
+    },
+    {
+      description: 'Backup v1: AES-256-GCM(backupKey, plaintext, no AAD) with deterministic nonce',
+      backupKey: hex(backupKey),
+      nonce: hex(fixedNonce),
+      plaintext: hex(plaintext),
+      plaintextUtf8: new TextDecoder().decode(plaintext),
+      ciphertext: hex(ciphertext),
+    },
+  ];
+}
+
+// ─── Group sender-keys vectors ─────────────────────────────
+//
+// Sender-key step pins three things:
+//   1. The 12-byte sender header AAD (`u16_be(gLen) || g || u16_be(sLen) || s || u32_be(iter)`)
+//   2. The chain-key advance (kdfChainKey) producing (newChainKey, messageKey)
+//   3. AES-256-GCM encrypt with deterministic nonce + Ed25519 signature
+//      over `aad || ciphertext`. Ed25519 is deterministic so the signature
+//      bytes are byte-parity-checkable cross-platform.
+async function generateGroupVectors(): Promise<Vector[]> {
+  // Static Ed25519 keypair (RFC 8032 §7.1 test vector 1)
+  const signingPrivateKey = fromHex(
+    '9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60',
+  );
+  const signingPublicKey = fromHex(
+    'd75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a',
+  );
+
+  const groupId = 'group:42';
+  const senderAddress = 'alice@example.com';
+  const iteration = 5;
+  const chainKey = new Uint8Array(32).fill(0x9b);
+
+  const enc = new TextEncoder();
+  const gBytes = enc.encode(groupId);
+  const sBytes = enc.encode(senderAddress);
+  const aad = new Uint8Array(2 + gBytes.length + 2 + sBytes.length + 4);
+  const aadView = new DataView(aad.buffer);
+  let off = 0;
+  aadView.setUint16(off, gBytes.length, false); off += 2;
+  aad.set(gBytes, off); off += gBytes.length;
+  aadView.setUint16(off, sBytes.length, false); off += 2;
+  aad.set(sBytes, off); off += sBytes.length;
+  aadView.setUint32(off, iteration, false);
+
+  const { newChainKey, messageKey } = await kdfChainKey(crypto, chainKey);
+  const fixedNonce = new Uint8Array(12).fill(0x7d);
+  const plaintext = enc.encode('hello group');
+  const ciphertext = await aesGcmEncryptDeterministic(messageKey, fixedNonce, plaintext, aad);
+
+  const signed = new Uint8Array(aad.length + ciphertext.length);
+  signed.set(aad, 0);
+  signed.set(ciphertext, aad.length);
+  const signature = await crypto.sign(signingPrivateKey, signed);
+  // Sanity: verify with the matching public key in the generator
+  const ok = await crypto.verify(signingPublicKey, signed, signature);
+  if (!ok) throw new Error('group sender-key signature verify failed in generator');
+
+  return [
+    {
+      description: 'Sender header AAD: u16_be(gLen) || g || u16_be(sLen) || s || u32_be(iter)',
+      groupId,
+      senderAddress,
+      iteration,
+      aad: hex(aad),
+    },
+    {
+      description: 'Sender-key step: kdfChainKey + deterministic AES-GCM + Ed25519 sign(aad || ct)',
+      chainKey: hex(chainKey),
+      groupId,
+      senderAddress,
+      iteration,
+      plaintext: hex(plaintext),
+      nonce: hex(fixedNonce),
+      signingPrivateKey: hex(signingPrivateKey),
+      signingPublicKey: hex(signingPublicKey),
+      newChainKey: hex(newChainKey),
+      messageKey: hex(messageKey),
+      aad: hex(aad),
+      ciphertext: hex(ciphertext),
+      signature: hex(signature),
+    },
+  ];
+}
+
+// ─── Storage-encryption HKDF parity ────────────────────────
+//
+// `test-vectors/storage-encryption.json` already exists (V3.2). It pins
+// scrypt params + HKDF info templates + AAD templates. The Kotlin port
+// will need scrypt (likely via Bouncy Castle) before the full file can
+// be consumed; for now this generator emits a sub-vector covering only
+// the HKDF-storage-key + HKDF-field-key + deterministic-nonce derivations
+// — those Tink already supports. Bumps the `_ts_subset_version`.
+async function generateStorageEncryptionSubset(): Promise<Vector[]> {
+  const masterKey = new Uint8Array(32);
+  for (let i = 0; i < 32; i++) masterKey[i] = i + 1;
+
+  const storageInfo = new TextEncoder().encode('shade-storage-v1');
+  const storageKey = await crypto.hkdf(masterKey, new Uint8Array(0), storageInfo, 32);
+
+  const fieldCases = [
+    { table: 'sessions', column: 'session' },
+    { table: 'identity', column: 'identity' },
+    { table: 'trusted_identities', column: 'trusted_identity' },
+  ];
+  const fieldKeys: Array<{ table: string; column: string; fieldKey: string }> = [];
+  for (const c of fieldCases) {
+    const info = new TextEncoder().encode(`shade-field-v1:${c.table}:${c.column}`);
+    const k = await crypto.hkdf(storageKey, new Uint8Array(0), info, 32);
+    fieldKeys.push({ table: c.table, column: c.column, fieldKey: hex(k) });
+  }
+
+  const rowKey = new Uint8Array(32).fill(0xcd);
+  const nonceCases = [
+    { table: 'sessions', pk: 'alice' },
+    { table: 'sessions', pk: 'bob' },
+    { table: 'identity', pk: '1' },
+  ];
+  const nonces: Array<{ table: string; pk: string; nonce: string }> = [];
+  for (const c of nonceCases) {
+    const info = new TextEncoder().encode(`shade-row-nonce-v1:${c.table}:${c.pk}`);
+    const n = await crypto.hkdf(rowKey, new Uint8Array(0), info, 12);
+    nonces.push({ table: c.table, pk: c.pk, nonce: hex(n) });
+  }
+
+  return [
+    {
+      description: 'Storage HKDF: storageKey = HKDF(masterKey, salt=0, info="shade-storage-v1", 32)',
+      masterKey: hex(masterKey),
+      storageKey: hex(storageKey),
+    },
+    {
+      description: 'Storage HKDF: fieldKey = HKDF(storageKey, salt=0, info="shade-field-v1:{table}:{column}", 32)',
+      storageKey: hex(storageKey),
+      fields: fieldKeys,
+    },
+    {
+      description: 'Storage HKDF: rowNonce = HKDF(rowKey, salt=0, info="shade-row-nonce-v1:{table}:{pk}", 12)',
+      rowKey: hex(rowKey),
+      nonces,
+    },
   ];
 }
 
 async function main() {
   console.log('Generating cross-platform test vectors…');
 
-  const files: Array<[string, any]> = [
+  const files: Array<[string, { vectors: Vector[] }]> = [
     ['hkdf.json', { vectors: await generateHkdfVectors() }],
     ['kdf-chain.json', { vectors: await generateKdfChainVectors() }],
     ['x3dh.json', { vectors: await generateX3DHVectors() }],
     ['fingerprint.json', { vectors: await generateFingerprintVectors() }],
     ['wire-format.json', { vectors: await generateWireFormatVectors() }],
+    ['ratchet-step.json', { vectors: await generateRatchetStepVectors() }],
+    ['streams.json', { vectors: await generateStreamsVectors() }],
+    ['backup.json', { vectors: await generateBackupVectors() }],
+    ['group.json', { vectors: await generateGroupVectors() }],
+    ['storage-hkdf.json', { vectors: await generateStorageEncryptionSubset() }],
   ];
 
   for (const [name, data] of files) {
     const path = join(OUT_DIR, name);
-    writeFileSync(path, JSON.stringify(data, null, 2) + '\n');
-    console.log(`  ✓ ${name} (${data.vectors.length} vectors)`);
+    const versioned = { version: VECTOR_FILE_VERSION, ...data };
+    writeFileSync(path, JSON.stringify(versioned, null, 2) + '\n');
+    console.log(`  ✓ ${name} (v${VECTOR_FILE_VERSION}, ${data.vectors.length} vectors)`);
   }
 
   console.log('Done.');

scripts/publish-all.ts

@@ -1,36 +1,47 @@
 #!/usr/bin/env bun
 /**
- * Publish all @shade/* packages to the Gitea npm registry.
+ * Headless publisher for all `@shade/*` packages.
  *
- * Expects these env vars:
+ * Use `scripts/publish-shade.sh` for the interactive human flow (token
+ * prompt, conflict detection, version bump-on-conflict). This script is
+ * the env-driven variant — designed for `DRY_RUN=1` smoke tests, CI
+ * pipelines, and any context where prompts are not appropriate.
+ *
+ * Required env (when not DRY_RUN):
  *   GITEA_TOKEN  — publish token from Gitea (Settings → Applications)
- *   GITEA_USER   — Gitea username that owns the registry (e.g. "Stian")
+ *   GITEA_USER   — Gitea username that owns the registry (default: Stian)
  *
  * Optional:
- *   DRY_RUN=1    — build tarballs but don't publish
- *
- * Usage:
- *   bun run scripts/publish-all.ts
+ *   DRY_RUN=1    — pack tarballs but do not publish (no token required)
  */
 import { readFileSync, writeFileSync, existsSync } from 'fs';
 import { join } from 'path';
 import { $ } from 'bun';
 
-// Order matters: each package only depends on packages above it. Publishing
-// in this order means a consumer fetching mid-publish never sees a manifest
-// pointing at an unpublished version.
+// Order matters: each package only depends on packages above it.
+// Mirrors the PACKAGES list in scripts/publish-shade.sh.
 const PACKAGES = [
   'shade-core',
   'shade-proto',
   'shade-crypto-web',
+  'shade-observability',
+  'shade-keychain',
+  'shade-key-transparency',
   'shade-storage-sqlite',
   'shade-storage-postgres',
+  'shade-storage-encrypted',
   'shade-streams',
   'shade-transport',
+  'shade-transport-bridge',
+  'shade-transport-webrtc',
   'shade-server',
+  'shade-inbox-server',
+  'shade-inbox',
   'shade-transfer',
   'shade-files',
+  'shade-recovery',
   'shade-observer',
+  'shade-dashboard',
   'shade-sdk',
   'shade-widgets',
   'shade-cli',
@@ -54,7 +65,6 @@ async function main() {
   console.log(`Dry run: ${dryRun ? 'yes' : 'no'}`);
   console.log();
 
-  // Write a temporary .npmrc at the root
   const npmrcPath = join(ROOT, '.npmrc.publish');
   const npmrc = [
     `@shade:registry=${registryUrl}`,
@@ -62,11 +72,6 @@ async function main() {
   ].filter(Boolean).join('\n');
   writeFileSync(npmrcPath, npmrc);
 
-  // Build a name → version map across all workspace packages so we can rewrite
-  // `workspace:*` (and friends) into concrete `^<version>` specifiers before
-  // publishing. Without this, the registry stores the literal `workspace:*`
-  // string in published package.json, which then fails to resolve in any
-  // consumer (e.g. Dispatch) outside the Shade monorepo.
   const versionByName = new Map<string, string>();
   for (const pkg of PACKAGES) {
     const pkgDir = join(ROOT, 'packages', pkg);
@@ -108,9 +113,6 @@ async function main() {
       const out = `${err instanceof Error ? err.message : String(err)} ${
         (err as { stderr?: { toString(): string } }).stderr?.toString() ?? ''
       } ${(err as { stdout?: { toString(): string } }).stdout?.toString() ?? ''}`;
-      // Gitea (and npm) report already-published versions as 409 / EPUBLISHCONFLICT.
-      // Skip silently rather than failing the whole run — bumping the version
-      // is the user's explicit decision via `bun run version <new>`.
       if (/409|EPUBLISHCONFLICT|already exists|already been published/i.test(out)) {
         alreadyPublished++;
         console.log(`  ⊙ already published — skipping`);
@@ -120,13 +122,10 @@ async function main() {
         process.exitCode = 1;
       }
     } finally {
-      // Always restore the original package.json so the workspace stays usable
-      // for `bun install` after publish, regardless of success or failure.
       writeFileSync(pkgJsonPath, originalPkgJson);
     }
   }
 
-  // Clean up temp npmrc
   try {
     await $`rm ${npmrcPath}`.quiet();
   } catch {}
@@ -137,11 +136,6 @@ async function main() {
   );
 }
 
-/**
- * Rewrite `workspace:*` (and `workspace:^`, `workspace:~`, `workspace:<v>`)
- * specifiers in dependency sections to concrete `^<version>` specifiers.
- * Mutates the passed-in object.
- */
 function rewriteWorkspaceSpecs(
   pkgJson: Record<string, unknown>,
   versionByName: Map<string, string>,

scripts/publish-shade.sh

@@ -10,14 +10,24 @@ PACKAGES=(
   core
   proto
   crypto-web
+  observability
+  keychain
+  key-transparency
   storage-sqlite
   storage-postgres
+  storage-encrypted
   streams
   transport
+  transport-bridge
+  transport-webrtc
   server
+  inbox-server
+  inbox
   transfer
   files
+  recovery
   observer
+  dashboard
   sdk
   widgets
   cli

scripts/soak.ts

@@ -0,0 +1,156 @@
+#!/usr/bin/env bun
+/**
+ * Shade 4.0 GA — combined soak harness.
+ *
+ * Runs the ratchet stack under sustained ping-pong load for a configurable
+ * duration. Designed to be wrapped in `systemd-run` / `nohup` / a Gitea
+ * scheduled job and left running for ≥ 2 weeks before tagging GA-stable.
+ *
+ *   Usage:
+ *     bun run scripts/soak.ts                 # default 1h
+ *     bun run scripts/soak.ts --hours 168     # 1 week
+ *     bun run scripts/soak.ts --hours 336     # 2 weeks (V4.0 §Soak)
+ *     bun run scripts/soak.ts --hours 0.05    # 3 minute smoke (CI gate)
+ */
+import { ShadeSessionManager } from '../packages/shade-core/src/index.js';
+import { SubtleCryptoProvider, MemoryStorage } from '../packages/shade-crypto-web/src/index.js';
+
+interface Args {
+  hours: number;
+  pairs: number;
+  reportSeconds: number;
+}
+
+function parseArgs(argv: string[]): Args {
+  const out: Args = { hours: 1, pairs: 4, reportSeconds: 60 };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === '--hours') out.hours = parseFloat(argv[++i]!);
+    else if (a === '--pairs') out.pairs = parseInt(argv[++i]!, 10);
+    else if (a === '--report-seconds') out.reportSeconds = parseInt(argv[++i]!, 10);
+    else if (a === '--help') {
+      console.log('Usage: bun run scripts/soak.ts [--hours N] [--pairs N] [--report-seconds N]');
+      process.exit(0);
+    }
+  }
+  return out;
+}
+
+interface Counters {
+  ratchetSent: number;
+  ratchetRecv: number;
+  ratchetErrs: number;
+  errors: { workload: string; message: string; ts: number }[];
+}
+
+function newCounters(): Counters {
+  return {
+    ratchetSent: 0,
+    ratchetRecv: 0,
+    ratchetErrs: 0,
+    errors: [],
+  };
+}
+
+interface Pair {
+  id: number;
+  aliceMgr: ShadeSessionManager;
+  bobMgr: ShadeSessionManager;
+  aliceAddr: string;
+  bobAddr: string;
+}
+
+async function setupPair(id: number): Promise<Pair> {
+  const crypto = new SubtleCryptoProvider();
+  const aliceMgr = new ShadeSessionManager(crypto, new MemoryStorage());
+  const bobMgr = new ShadeSessionManager(crypto, new MemoryStorage());
+  await aliceMgr.initialize();
+  await bobMgr.initialize();
+
+  const otpks = await bobMgr.generateOneTimePreKeys(5);
+  const bundle = await bobMgr.createPreKeyBundle();
+  bundle.oneTimePreKey = { keyId: otpks[0]!.keyId, publicKey: otpks[0]!.keyPair.publicKey };
+
+  const aliceAddr = `alice-${id}`;
+  const bobAddr = `bob-${id}`;
+  await aliceMgr.initSessionFromBundle(bobAddr, bundle);
+
+  // Establish the receiver session by sending a first message and decrypting.
+  const init = await aliceMgr.encrypt(bobAddr, 'init');
+  await bobMgr.decrypt(aliceAddr, init);
+
+  return { id, aliceMgr, bobMgr, aliceAddr, bobAddr };
+}
+
+async function ratchetPingPong(pair: Pair, counters: Counters, abort: AbortSignal) {
+  const { aliceMgr, bobMgr, aliceAddr, bobAddr } = pair;
+  let i = 0;
+  while (!abort.aborted) {
+    try {
+      const env = await aliceMgr.encrypt(bobAddr, `msg-${i++}`);
+      counters.ratchetSent++;
+      await bobMgr.decrypt(aliceAddr, env);
+      counters.ratchetRecv++;
+      const reply = await bobMgr.encrypt(aliceAddr, `re-${i}`);
+      counters.ratchetSent++;
+      await aliceMgr.decrypt(bobAddr, reply);
+      counters.ratchetRecv++;
+      if ((i & 0xff) === 0) await new Promise((r) => setTimeout(r, 0));
+    } catch (err) {
+      counters.ratchetErrs++;
+      counters.errors.push({
+        workload: `pair-${pair.id}`,
+        message: (err as Error).message,
+        ts: Date.now(),
+      });
+      if (counters.ratchetErrs > 8) break;
+    }
+  }
+}
+
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  const totalMs = Math.max(60_000, Math.round(args.hours * 3_600_000));
+  const ac = new AbortController();
+  const counters = newCounters();
+  const start = Date.now();
+
+  console.log(`[soak] start — ${args.pairs} pairs, ${args.hours}h, report every ${args.reportSeconds}s`);
+
+  const pairs = await Promise.all(Array.from({ length: args.pairs }, (_, i) => setupPair(i)));
+
+  setTimeout(() => ac.abort(), totalMs).unref();
+  process.on('SIGINT', () => ac.abort());
+  process.on('SIGTERM', () => ac.abort());
+
+  const reporter = setInterval(() => {
+    const elapsedS = ((Date.now() - start) / 1000).toFixed(0);
+    const rps = (counters.ratchetSent / Math.max(1, (Date.now() - start) / 1000)).toFixed(1);
+    console.log(
+      `[soak] t=${elapsedS}s sent=${counters.ratchetSent} recv=${counters.ratchetRecv} errs=${counters.ratchetErrs} (${rps} sent/s)`,
+    );
+    if (counters.errors.length > 32) counters.errors.splice(0, counters.errors.length - 32);
+  }, args.reportSeconds * 1000);
+  reporter.unref();
+
+  await Promise.all(pairs.map((p) => ratchetPingPong(p, counters, ac.signal)));
+
+  clearInterval(reporter);
+
+  const elapsedS = ((Date.now() - start) / 1000).toFixed(0);
+  console.log(`\n[soak] done in ${elapsedS}s`);
+  console.log(`  ratchet sent  ${counters.ratchetSent}`);
+  console.log(`  ratchet recv  ${counters.ratchetRecv}`);
+  console.log(`  ratchet errs  ${counters.ratchetErrs}`);
+
+  if (counters.ratchetErrs > 0) {
+    console.error('\nFAIL: ratchet errors recorded:');
+    for (const e of counters.errors.slice(-10)) {
+      console.error(`  [${new Date(e.ts).toISOString()}] ${e.workload}: ${e.message}`);
+    }
+    process.exit(1);
+  }
+  process.exit(0);
+}
+
+void main();