release(v4.0.0): Shade GA — V3.x consolidation + audit prep

V3.1 → V3.12 consolidated and tagged for the first GA release. Wire format unchanged from 0.4.x — 4.0 peers interoperate with 0.4.x peers byte-for-byte. The version bump is semantic: audit-cycle complete, opt-in surface fully exposed, threat model refreshed for every new surface. Highlights: - All 24 @shade/* packages bumped to 4.0.0 in lockstep. - CHANGELOG 4.0.0 section is the canonical manifest of what landed. - THREAT-MODEL extended (§10 fingerprint gates, §11 WebRTC P2P, §12 Web-Worker boundary) + residual-risks table refreshed. - OpenAPI now covers all 27 routes: prekey, transfer, KT, inbox, bridge, observer, /metrics, /healthz, /ready. - MIGRATION 0.3.x → 4.0 documented + smoke-tested against shade migrate-storage on a real SQLite DB. - docs/audit/REVIEW-BUNDLE.md + SCOPE.md ready for external reviewer. - scripts/soak.ts harness for the GA-stable 2-week soak window. - All V*.md plans archived under docs/archive/ with Status: Done. - Voice/Video carved out into V5.0; 4.0 audit focuses on the frozen non-realtime stack. Tests: TS 1000/1000 + Kotlin 11/11 cross-platform vectors green. Docker: gt.zyon.no/stian/shade-prekey:4.0.0 builds and reports version 4.0.0 on /health. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-03 18:35:35 +02:00
parent 8b055912b7
commit e6fdf31b49
298 changed files with 37909 additions and 256 deletions
--- a/packages/shade-server/tests/kt.test.ts
+++ b/packages/shade-server/tests/kt.test.ts
@@ -0,0 +1,260 @@
+import { describe, test, expect, beforeEach } from 'bun:test';
+import {
+  createPrekeyServerWithKT,
+  KeyTransparencyService,
+  MemoryPrekeyStore,
+  signPayload,
+} from '../src/index.js';
+import { SubtleCryptoProvider } from '@shade/crypto-web';
+import { generateIdentityKeyPair } from '@shade/core';
+import {
+  MemoryKTLogStore,
+  computeBundleHash,
+  ktProofFromWire,
+  sthFromWire,
+  verifyBundleAbsence,
+  verifyBundleInclusion,
+  verifyConsistencyProof,
+  type KTProofWire,
+  type STHWire,
+} from '@shade/key-transparency';
+
+const crypto = new SubtleCryptoProvider();
+
+function b64(bytes: Uint8Array): string {
+  return Buffer.from(bytes).toString('base64');
+}
+
+function fromB64(s: string): Uint8Array {
+  return new Uint8Array(Buffer.from(s, 'base64'));
+}
+
+function randBytes(n: number): Uint8Array {
+  const buf = new Uint8Array(n);
+  globalThis.crypto.getRandomValues(buf);
+  return buf;
+}
+
+async function makeBundleData() {
+  const identity = await generateIdentityKeyPair(crypto);
+  const signedPreKeyPub = randBytes(32);
+  const signedPreKeySig = await crypto.sign(identity.signingPrivateKey, signedPreKeyPub);
+  return {
+    identity,
+    signedPreKey: {
+      keyId: 1,
+      publicKey: signedPreKeyPub,
+      signature: signedPreKeySig,
+    },
+  };
+}
+
+describe('Prekey server with KT enabled', () => {
+  let app: any;
+  let kt: KeyTransparencyService;
+  let logKp: { publicKey: Uint8Array; privateKey: Uint8Array };
+
+  beforeEach(async () => {
+    logKp = await crypto.generateEd25519KeyPair();
+    const result = await createPrekeyServerWithKT({
+      crypto,
+      store: new MemoryPrekeyStore(),
+      disableRateLimit: true,
+      keyTransparency: {
+        store: new MemoryKTLogStore(),
+        signingPrivateKey: logKp.privateKey,
+        signingPublicKey: logKp.publicKey,
+      },
+    });
+    app = result.app;
+    kt = result.kt;
+  });
+
+  async function registerAddress(address: string) {
+    const data = await makeBundleData();
+    const body: any = {
+      address,
+      identitySigningKey: b64(data.identity.signingPublicKey),
+      identityDHKey: b64(data.identity.dhPublicKey),
+      signedPreKey: {
+        keyId: data.signedPreKey.keyId,
+        publicKey: b64(data.signedPreKey.publicKey),
+        signature: b64(data.signedPreKey.signature),
+      },
+      oneTimePreKeys: [{ keyId: 100, publicKey: b64(randBytes(32)) }],
+    };
+    const signed = await signPayload(crypto, data.identity.signingPrivateKey, body);
+    const res = await app.request('/v1/keys/register', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(signed),
+    });
+    expect(res.status).toBe(200);
+    return data;
+  }
+
+  test('GET /v1/kt/log_id returns logId + publicKey', async () => {
+    const res = await app.request('/v1/kt/log_id');
+    expect(res.status).toBe(200);
+    const body = await res.json();
+    expect(body.logId).toBeDefined();
+    expect(body.publicKey).toBeDefined();
+    expect(b64(kt.getSigningPublicKey())).toBe(body.publicKey);
+  });
+
+  test('GET /v1/kt/sth returns latest STH', async () => {
+    await registerAddress('alice');
+    const res = await app.request('/v1/kt/sth');
+    expect(res.status).toBe(200);
+    const wire = (await res.json()) as STHWire;
+    const sth = sthFromWire(wire, fromB64);
+    expect(sth.treeSize).toBe(1);
+  });
+
+  test('bundle response carries verified inclusion proof', async () => {
+    const data = await registerAddress('alice');
+    const res = await app.request('/v1/keys/bundle/alice');
+    expect(res.status).toBe(200);
+    const body = (await res.json()) as { ktProof: KTProofWire } & Record<string, unknown>;
+    expect(body.ktProof).toBeDefined();
+
+    const proof = ktProofFromWire(body.ktProof);
+    expect(proof.body.kind).toBe('inclusion');
+
+    await verifyBundleInclusion(
+      { crypto, logPublicKey: logKp.publicKey },
+      'alice',
+      {
+        identitySigningKey: data.identity.signingPublicKey,
+        identityDHKey: data.identity.dhPublicKey,
+        signedPreKey: data.signedPreKey,
+      },
+      proof,
+    );
+
+    // Sanity: bundle hash matches the proof's index commitment
+    const expected = computeBundleHash({
+      identitySigningKey: data.identity.signingPublicKey,
+      identityDHKey: data.identity.dhPublicKey,
+      signedPreKey: data.signedPreKey,
+    });
+    if (proof.body.kind === 'inclusion') {
+      expect(b64(proof.body.indexProof.entry.bundleHash)).toBe(b64(expected));
+    }
+  });
+
+  test('bundle for unknown address returns 404 + absence proof', async () => {
+    await registerAddress('alice');
+    const res = await app.request('/v1/keys/bundle/zeta');
+    expect(res.status).toBe(404);
+    const body = (await res.json()) as { ktProof?: KTProofWire };
+    expect(body.ktProof).toBeDefined();
+    const proof = ktProofFromWire(body.ktProof!);
+    expect(proof.body.kind).toBe('absence');
+    await verifyBundleAbsence({ crypto, logPublicKey: logKp.publicKey }, 'zeta', proof);
+  });
+
+  test('DELETE /v1/keys/:address commits a tombstone', async () => {
+    const data = await registerAddress('alice');
+    const sthBefore = await kt.getLatestSTH();
+
+    const signed = await signPayload(crypto, data.identity.signingPrivateKey, { address: 'alice' });
+    const res = await app.request('/v1/keys/alice', {
+      method: 'DELETE',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(signed),
+    });
+    expect(res.status).toBe(200);
+
+    const sthAfter = await kt.getLatestSTH();
+    expect(sthAfter.treeSize).toBeGreaterThan(sthBefore.treeSize);
+
+    const fetched = await app.request('/v1/keys/bundle/alice');
+    // Identity row was removed by the prekey-store; absence-proof returned.
+    expect(fetched.status).toBe(404);
+    const body = (await fetched.json()) as { ktProof?: KTProofWire };
+    expect(body.ktProof).toBeDefined();
+  });
+
+  test('GET /v1/kt/consistency returns valid proof', async () => {
+    await registerAddress('alice');
+    const sth1 = await kt.getLatestSTH();
+    await registerAddress('bob');
+    const sth2 = await kt.getLatestSTH();
+
+    const res = await app.request(`/v1/kt/consistency?from=${sth1.treeSize}&to=${sth2.treeSize}`);
+    expect(res.status).toBe(200);
+    const body = (await res.json()) as { proof: string[] };
+    const proofBytes = body.proof.map(fromB64);
+    expect(verifyConsistencyProof(sth1.treeSize, sth2.treeSize, sth1.rootHash, sth2.rootHash, proofBytes)).toBe(true);
+  });
+
+  test('GET /v1/kt/sth/:treeSize returns historical STH', async () => {
+    await registerAddress('alice');
+    const sth1 = await kt.getLatestSTH();
+    await registerAddress('bob');
+
+    const res = await app.request(`/v1/kt/sth/${sth1.treeSize}`);
+    expect(res.status).toBe(200);
+    const wire = (await res.json()) as STHWire;
+    const back = sthFromWire(wire, fromB64);
+    expect(b64(back.rootHash)).toBe(b64(sth1.rootHash));
+  });
+
+  test('rotation: latest STH proof verifies with new bundle, not old', async () => {
+    const v1Data = await registerAddress('alice');
+
+    // Register again with a new identity (rotation)
+    const v2Data = await makeBundleData();
+    const body: any = {
+      address: 'alice',
+      identitySigningKey: b64(v2Data.identity.signingPublicKey),
+      identityDHKey: b64(v2Data.identity.dhPublicKey),
+      signedPreKey: {
+        keyId: v2Data.signedPreKey.keyId,
+        publicKey: b64(v2Data.signedPreKey.publicKey),
+        signature: b64(v2Data.signedPreKey.signature),
+      },
+    };
+    const signed = await signPayload(crypto, v2Data.identity.signingPrivateKey, body);
+    const reRes = await app.request('/v1/keys/register', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(signed),
+    });
+    expect(reRes.status).toBe(200);
+
+    // Now the bundle response should reflect the new identity
+    const res = await app.request('/v1/keys/bundle/alice');
+    const body2 = (await res.json()) as {
+      identitySigningKey: string;
+      ktProof: KTProofWire;
+    };
+    expect(body2.identitySigningKey).toBe(b64(v2Data.identity.signingPublicKey));
+
+    const proof = ktProofFromWire(body2.ktProof);
+    await verifyBundleInclusion(
+      { crypto, logPublicKey: logKp.publicKey },
+      'alice',
+      {
+        identitySigningKey: v2Data.identity.signingPublicKey,
+        identityDHKey: v2Data.identity.dhPublicKey,
+        signedPreKey: v2Data.signedPreKey,
+      },
+      proof,
+    );
+    // Verifying with v1 data should reject
+    await expect(
+      verifyBundleInclusion(
+        { crypto, logPublicKey: logKp.publicKey },
+        'alice',
+        {
+          identitySigningKey: v1Data.identity.signingPublicKey,
+          identityDHKey: v1Data.identity.dhPublicKey,
+          signedPreKey: v1Data.signedPreKey,
+        },
+        proof,
+      ),
+    ).rejects.toThrow();
+  });
+});
--- a/packages/shade-server/tests/openapi-lint.test.ts
+++ b/packages/shade-server/tests/openapi-lint.test.ts
@@ -0,0 +1,146 @@
+import { describe, test, expect } from 'bun:test';
+import { readFileSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+
+const here = dirname(fileURLToPath(import.meta.url));
+const specPath = join(here, '..', 'openapi.yaml');
+const spec = Bun.YAML.parse(readFileSync(specPath, 'utf-8')) as Record<string, unknown>;
+
+describe('openapi.yaml lint', () => {
+  test('declares OpenAPI 3.1', () => {
+    expect(typeof spec.openapi).toBe('string');
+    expect(String(spec.openapi).startsWith('3.1')).toBe(true);
+  });
+
+  test('has the required top-level fields', () => {
+    expect(spec).toHaveProperty('info');
+    expect(spec).toHaveProperty('paths');
+    expect(spec).toHaveProperty('components');
+    const info = spec.info as Record<string, unknown>;
+    expect(typeof info.title).toBe('string');
+    expect(typeof info.version).toBe('string');
+  });
+
+  test('every operation has summary + responses', () => {
+    const paths = (spec.paths ?? {}) as Record<string, Record<string, unknown>>;
+    const httpMethods = new Set([
+      'get',
+      'post',
+      'put',
+      'patch',
+      'delete',
+      'options',
+      'head',
+    ]);
+
+    for (const [routePath, route] of Object.entries(paths)) {
+      for (const [verb, op] of Object.entries(route)) {
+        if (!httpMethods.has(verb)) continue;
+        const operation = op as Record<string, unknown>;
+        expect(operation.summary, `${verb.toUpperCase()} ${routePath} missing summary`).toBeDefined();
+        const responses = operation.responses as Record<string, unknown> | undefined;
+        expect(responses, `${verb.toUpperCase()} ${routePath} missing responses`).toBeDefined();
+        expect(
+          Object.keys(responses ?? {}).length,
+          `${verb.toUpperCase()} ${routePath} has empty responses`,
+        ).toBeGreaterThan(0);
+      }
+    }
+  });
+
+  test('every $ref resolves to a defined component', () => {
+    const refs = collectRefs(spec);
+    expect(refs.length).toBeGreaterThan(0);
+
+    for (const ref of refs) {
+      expect(
+        ref.startsWith('#/'),
+        `non-internal $ref not allowed: ${ref}`,
+      ).toBe(true);
+
+      const segments = ref.slice(2).split('/');
+      let cursor: unknown = spec;
+      for (const segment of segments) {
+        expect(
+          isObject(cursor),
+          `dangling $ref: ${ref} (failed at "${segment}")`,
+        ).toBe(true);
+        cursor = (cursor as Record<string, unknown>)[segment];
+      }
+      expect(cursor, `unresolved $ref: ${ref}`).toBeDefined();
+    }
+  });
+
+  test('every security requirement references a defined scheme', () => {
+    const schemes = (
+      ((spec.components ?? {}) as Record<string, Record<string, unknown>>)
+        .securitySchemes ?? {}
+    ) as Record<string, unknown>;
+    const definedSchemes = new Set(Object.keys(schemes));
+
+    const requirements = collectSecurityRequirements(spec);
+    for (const name of requirements) {
+      expect(
+        definedSchemes.has(name),
+        `security requirement "${name}" not defined under components.securitySchemes`,
+      ).toBe(true);
+    }
+  });
+
+  test('declares the V3.1 transfer surface', () => {
+    const paths = (spec.paths ?? {}) as Record<string, unknown>;
+    expect(paths['/v1/transfer/health']).toBeDefined();
+    expect(paths['/v1/transfer/{streamId}/chunk']).toBeDefined();
+    expect(paths['/v1/transfer/{streamId}/state']).toBeDefined();
+    expect(paths['/v1/transfer/control']).toBeDefined();
+
+    const schemes = (
+      ((spec.components ?? {}) as Record<string, Record<string, unknown>>)
+        .securitySchemes ?? {}
+    ) as Record<string, unknown>;
+    expect(schemes.ShadeTransferAuthenticator).toBeDefined();
+  });
+});
+
+function isObject(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function collectRefs(node: unknown, out: string[] = []): string[] {
+  if (Array.isArray(node)) {
+    for (const item of node) collectRefs(item, out);
+    return out;
+  }
+  if (!isObject(node)) return out;
+  for (const [key, value] of Object.entries(node)) {
+    if (key === '$ref' && typeof value === 'string') {
+      out.push(value);
+      continue;
+    }
+    collectRefs(value, out);
+  }
+  return out;
+}
+
+function collectSecurityRequirements(spec: Record<string, unknown>): Set<string> {
+  const names = new Set<string>();
+  const collectFromList = (list: unknown): void => {
+    if (!Array.isArray(list)) return;
+    for (const entry of list) {
+      if (!isObject(entry)) continue;
+      for (const name of Object.keys(entry)) names.add(name);
+    }
+  };
+
+  collectFromList(spec.security);
+
+  const paths = (spec.paths ?? {}) as Record<string, Record<string, unknown>>;
+  for (const route of Object.values(paths)) {
+    for (const value of Object.values(route)) {
+      if (!isObject(value)) continue;
+      collectFromList(value.security);
+    }
+  }
+  return names;
+}