release(v4.0.0): Shade GA — V3.x consolidation + audit prep

V3.1 → V3.12 consolidated and tagged for the first GA release. Wire
format unchanged from 0.4.x — 4.0 peers interoperate with 0.4.x peers
byte-for-byte. The version bump is semantic: audit-cycle complete,
opt-in surface fully exposed, threat model refreshed for every new
surface.

Highlights:
- All 24 @shade/* packages bumped to 4.0.0 in lockstep.
- CHANGELOG 4.0.0 section is the canonical manifest of what landed.
- THREAT-MODEL extended (§10 fingerprint gates, §11 WebRTC P2P, §12
  Web-Worker boundary) + residual-risks table refreshed.
- OpenAPI now covers all 27 routes: prekey, transfer, KT, inbox,
  bridge, observer, /metrics, /healthz, /ready.
- MIGRATION 0.3.x → 4.0 documented + smoke-tested against
  shade migrate-storage on a real SQLite DB.
- docs/audit/REVIEW-BUNDLE.md + SCOPE.md ready for external reviewer.
- scripts/soak.ts harness for the GA-stable 2-week soak window.
- All V*.md plans archived under docs/archive/ with Status: Done.
- Voice/Video carved out into V5.0; 4.0 audit focuses on the frozen
  non-realtime stack.

Tests: TS 1000/1000 + Kotlin 11/11 cross-platform vectors green.
Docker: gt.zyon.no/stian/shade-prekey:4.0.0 builds and reports
  version 4.0.0 on /health.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

packages/shade-observability/src/attributes.ts

@@ -0,0 +1,122 @@
+/**
+ * Safe-attribute helpers — these are the ONLY attribute keys/values that
+ * Shade internals are allowed to put on spans. The PII-policy guarantees:
+ *
+ *   - No plaintext peer addresses (use `peerHash`).
+ *   - No plaintext payloads.
+ *   - No exact byte counts for stream content (use `bytesBin`).
+ *   - Counters/codes are fine.
+ *
+ * Custom-op authors who want to add their own attributes MUST go through
+ * `safeAttribute()`, which rejects keys/values that look like PII.
+ */
+
+import { sha256 } from '@noble/hashes/sha2.js';
+
+// ─── Standard attribute keys ─────────────────────────────────
+
+export const ATTR_PEER_HASH = 'shade.peer.hash';
+export const ATTR_BYTES_BIN = 'shade.bytes.bin';
+export const ATTR_LANE_COUNT = 'shade.lane.count';
+export const ATTR_LANE_ID = 'shade.lane.id';
+export const ATTR_RETRY_COUNT = 'shade.retry.count';
+export const ATTR_ERROR_CODE = 'shade.error.code';
+export const ATTR_OP = 'shade.op';
+export const ATTR_ROUTE = 'shade.route';
+export const ATTR_HTTP_STATUS = 'shade.http.status';
+export const ATTR_DIRECTION = 'shade.direction';
+export const ATTR_PARTITION = 'shade.partition';
+export const ATTR_RESULT = 'shade.result';
+
+/**
+ * Forbidden substrings — if these appear in attribute keys we refuse.
+ * Mirrors the intent of the PII policy doc: never log addresses or
+ * exact-byte sizes.
+ */
+const FORBIDDEN_KEY_FRAGMENTS = [
+  'peer.address',
+  'peer_address',
+  'plaintext',
+  'payload',
+  'bytes.exact',
+  'bytes_exact',
+];
+
+/** 8-byte stable pseudonym derived from a peer address. */
+export function peerHash(address: string): string {
+  const enc = new TextEncoder().encode(address);
+  const digest = sha256(enc);
+  let hex = '';
+  for (let i = 0; i < 4; i++) hex += digest[i]!.toString(16).padStart(2, '0');
+  return hex;
+}
+
+/** Bin a byte count into a coarse PII-safe bucket. */
+export function bytesBin(n: number): string {
+  if (!Number.isFinite(n) || n < 0) return 'unknown';
+  if (n <= 4 * 1024) return '≤4KB';
+  if (n <= 64 * 1024) return '4–64KB';
+  if (n <= 1024 * 1024) return '64KB–1MB';
+  if (n <= 10 * 1024 * 1024) return '1–10MB';
+  if (n <= 100 * 1024 * 1024) return '10–100MB';
+  if (n <= 1024 * 1024 * 1024) return '100MB–1GB';
+  return '≥1GB';
+}
+
+/** Bin a lane count into a stable bucket. */
+export function laneCountBin(n: number): number {
+  if (n <= 1) return 1;
+  if (n <= 4) return 4;
+  if (n <= 16) return 16;
+  if (n <= 64) return 64;
+  return 64;
+}
+
+export class UnsafeAttributeError extends Error {
+  override readonly name = 'UnsafeAttributeError';
+  constructor(reason: string) {
+    super(reason);
+  }
+}
+
+/**
+ * Validate a user-supplied custom attribute. Returns the key/value pair
+ * untouched on success, or throws `UnsafeAttributeError`. Use this in
+ * any code path that accepts attributes from outside Shade's own
+ * helpers (e.g. plugin-supplied tags).
+ */
+export function safeAttribute(
+  key: string,
+  value: string | number | boolean,
+): { key: string; value: string | number | boolean } {
+  const lower = key.toLowerCase();
+  for (const frag of FORBIDDEN_KEY_FRAGMENTS) {
+    if (lower.includes(frag)) {
+      throw new UnsafeAttributeError(`attribute key "${key}" is PII-unsafe (contains "${frag}")`);
+    }
+  }
+  if (typeof value === 'string') {
+    if (value.length > 256) {
+      throw new UnsafeAttributeError(
+        `attribute "${key}" value too long (${value.length}B); cap at 256B to avoid embedded PII`,
+      );
+    }
+    if (looksLikeAddress(value)) {
+      throw new UnsafeAttributeError(
+        `attribute "${key}" value looks like a peer address — use peerHash() first`,
+      );
+    }
+  }
+  return { key, value };
+}
+
+function looksLikeAddress(s: string): boolean {
+  // Heuristic: emails, "device:UUID", and DID-style identifiers all look
+  // like PII to the grep tester. Hashes (8 hex chars) and small ints are
+  // fine.
+  if (/^[a-f0-9]{1,16}$/i.test(s)) return false;
+  if (/@/.test(s)) return true;
+  if (/^device:/i.test(s)) return true;
+  if (/^did:/i.test(s)) return true;
+  return false;
+}

packages/shade-observability/src/index.ts

@@ -0,0 +1,36 @@
+export type {
+  Attributes,
+  AttrValue,
+  ObservabilityHook,
+  Span,
+} from './types.js';
+export { NOOP_HOOK, noopSpan } from './types.js';
+
+export {
+  ATTR_BYTES_BIN,
+  ATTR_DIRECTION,
+  ATTR_ERROR_CODE,
+  ATTR_HTTP_STATUS,
+  ATTR_LANE_COUNT,
+  ATTR_LANE_ID,
+  ATTR_OP,
+  ATTR_PARTITION,
+  ATTR_PEER_HASH,
+  ATTR_RESULT,
+  ATTR_RETRY_COUNT,
+  ATTR_ROUTE,
+  bytesBin,
+  laneCountBin,
+  peerHash,
+  safeAttribute,
+  UnsafeAttributeError,
+} from './attributes.js';
+
+export {
+  withTracer,
+  type OtelSpanLike,
+  type OtelTracerLike,
+  type WithTracerOptions,
+} from './with-tracer.js';
+
+export { createRecorder, type RecordedSpan, type SpanRecorder } from './recorder.js';

packages/shade-observability/src/recorder.ts

@@ -0,0 +1,99 @@
+/**
+ * In-memory `ObservabilityHook` for tests. Records every span name +
+ * attribute mutation so the PII grep test can scrub the recording for
+ * forbidden values.
+ */
+
+import type { Attributes, AttrValue, ObservabilityHook, Span } from './types.js';
+
+export interface RecordedSpan {
+  name: string;
+  attributes: Attributes;
+  status: 'ok' | 'error' | 'unset';
+  statusMessage?: string;
+  exceptions: unknown[];
+  ended: boolean;
+  startedAtMs: number;
+  endedAtMs?: number;
+}
+
+export interface SpanRecorder extends ObservabilityHook {
+  /** All spans started so far (in order). */
+  readonly spans: readonly RecordedSpan[];
+  /** Drop all recorded spans — convenient between test cases. */
+  clear(): void;
+  /**
+   * Search recorded attributes/values for any of `forbiddenSubstrings`.
+   * Returns the offending hits. Use in PII guard tests.
+   */
+  scanForPII(forbiddenSubstrings: readonly string[]): Array<{
+    spanName: string;
+    key: string;
+    value: AttrValue;
+    match: string;
+  }>;
+}
+
+export function createRecorder(): SpanRecorder {
+  const spans: RecordedSpan[] = [];
+  const hook: ObservabilityHook = {
+    startSpan(name, attrs) {
+      const rec: RecordedSpan = {
+        name,
+        attributes: attrs !== undefined ? { ...attrs } : {},
+        status: 'unset',
+        exceptions: [],
+        ended: false,
+        startedAtMs: Date.now(),
+      };
+      spans.push(rec);
+      const span: Span = {
+        setAttribute(key, value) {
+          if (rec.ended) return;
+          rec.attributes[key] = value;
+        },
+        setAttributes(more) {
+          if (rec.ended) return;
+          for (const [k, v] of Object.entries(more)) rec.attributes[k] = v;
+        },
+        recordException(err) {
+          if (rec.ended) return;
+          rec.exceptions.push(err);
+        },
+        setStatus(status, message) {
+          if (rec.ended) return;
+          rec.status = status;
+          if (message !== undefined) rec.statusMessage = message;
+        },
+        end() {
+          if (rec.ended) return;
+          rec.ended = true;
+          rec.endedAtMs = Date.now();
+        },
+      };
+      return span;
+    },
+  };
+  return Object.assign(hook, {
+    get spans() {
+      return spans as readonly RecordedSpan[];
+    },
+    clear() {
+      spans.length = 0;
+    },
+    scanForPII(forbidden: readonly string[]) {
+      const hits: Array<{ spanName: string; key: string; value: AttrValue; match: string }> = [];
+      for (const span of spans) {
+        for (const [key, value] of Object.entries(span.attributes)) {
+          const haystack = `${key}=${String(value)}`.toLowerCase();
+          for (const f of forbidden) {
+            if (haystack.includes(f.toLowerCase())) {
+              hits.push({ spanName: span.name, key, value, match: f });
+            }
+          }
+        }
+      }
+      return hits;
+    },
+  });
+}

packages/shade-observability/src/types.ts

@@ -0,0 +1,46 @@
+/**
+ * Vendor-neutral observability hook surface.
+ *
+ * Shade's internals never depend on `@opentelemetry/api` directly — they
+ * consume an `ObservabilityHook` through which spans are started/ended.
+ * `withTracer()` adapts an OTel tracer to this hook; tests use
+ * `createRecorder()` for in-memory inspection.
+ */
+
+export type AttrValue = string | number | boolean;
+export type Attributes = Record<string, AttrValue>;
+
+export interface Span {
+  /** Set/overwrite a single attribute on this span. */
+  setAttribute(key: string, value: AttrValue): void;
+  /** Bulk-set attributes. */
+  setAttributes(attrs: Attributes): void;
+  /** Mark the span as failed with the given error. */
+  recordException(err: unknown): void;
+  /** Set the span status. */
+  setStatus(status: 'ok' | 'error', message?: string): void;
+  /** Close the span. Idempotent. */
+  end(): void;
+}
+
+export interface ObservabilityHook {
+  /** Start a new span. The returned object MUST always have `.end()` called. */
+  startSpan(name: string, attrs?: Attributes): Span;
+}
+
+const NOOP_SPAN: Span = {
+  setAttribute: () => undefined,
+  setAttributes: () => undefined,
+  recordException: () => undefined,
+  setStatus: () => undefined,
+  end: () => undefined,
+};
+
+export const NOOP_HOOK: ObservabilityHook = {
+  startSpan: () => NOOP_SPAN,
+};
+
+/** Returned by `Span` factories that get sampled-out. */
+export function noopSpan(): Span {
+  return NOOP_SPAN;
+}

packages/shade-observability/src/with-tracer.ts

@@ -0,0 +1,140 @@
+/**
+ * `withTracer()` — adapt an `@opentelemetry/api` Tracer (or any
+ * structurally-compatible tracer) into the vendor-neutral
+ * `ObservabilityHook` that Shade's internals consume.
+ *
+ * No-op behavior:
+ *   - When `tracer` is `undefined`.
+ *   - When `process.env.SHADE_OTEL_ENABLED` is not set to a truthy value
+ *     ("1"/"true") AND `opts.force` isn't passed.
+ *   - When `opts.sample` is provided and the per-span dice roll fails.
+ *
+ * The OTel structural surface we depend on is intentionally tiny — only
+ * `tracer.startSpan(name)` and the resulting span's
+ * `setAttribute/setStatus/recordException/end` methods. This keeps us
+ * compatible with any OTel-flavoured tracer (Jaeger, Honeycomb, Tempo,
+ * Sentry's OTel adapter, etc.) without locking into one vendor.
+ */
+
+import {
+  NOOP_HOOK,
+  noopSpan,
+  type Attributes,
+  type AttrValue,
+  type ObservabilityHook,
+  type Span,
+} from './types.js';
+
+/** The structural subset of an OTel `Span` that we use. */
+export interface OtelSpanLike {
+  setAttribute(key: string, value: AttrValue): unknown;
+  setAttributes?(attrs: Attributes): unknown;
+  recordException?(err: unknown): unknown;
+  setStatus?(status: { code: number; message?: string }): unknown;
+  end(endTime?: number): unknown;
+}
+
+/** The structural subset of an OTel `Tracer` that we use. */
+export interface OtelTracerLike {
+  startSpan(name: string, options?: { attributes?: Attributes }): OtelSpanLike;
+}
+
+export interface WithTracerOptions {
+  /**
+   * Per-span sample rate in `[0, 1]`. Default `1` (sample everything when
+   * the hook is active). Sampled-out spans return a no-op span.
+   */
+  sample?: number;
+  /**
+   * Bypass the `SHADE_OTEL_ENABLED` env-var gate. Useful for tests and
+   * for embeddings where the hosting application makes the on/off
+   * decision itself.
+   */
+  force?: boolean;
+  /** Override the env-var name. Defaults to `SHADE_OTEL_ENABLED`. */
+  envVar?: string;
+  /** Override the random source (used in tests for determinism). */
+  random?: () => number;
+}
+
+/**
+ * Per the OTel spec: `SpanStatusCode.OK = 1`, `ERROR = 2`. We hardcode
+ * the integers so we don't pull in `@opentelemetry/api` at runtime.
+ */
+const OTEL_STATUS_OK = 1;
+const OTEL_STATUS_ERROR = 2;
+
+export function withTracer(
+  tracer: OtelTracerLike | null | undefined,
+  opts: WithTracerOptions = {},
+): ObservabilityHook {
+  if (tracer === null || tracer === undefined) return NOOP_HOOK;
+
+  const envVar = opts.envVar ?? 'SHADE_OTEL_ENABLED';
+  if (!opts.force && !envEnabled(envVar)) return NOOP_HOOK;
+
+  const sample = opts.sample ?? 1;
+  if (sample <= 0) return NOOP_HOOK;
+  const random = opts.random ?? Math.random;
+
+  return {
+    startSpan(name: string, attrs?: Attributes): Span {
+      if (sample < 1 && random() >= sample) return noopSpan();
+      const otelSpan = attrs !== undefined
+        ? tracer.startSpan(name, { attributes: attrs })
+        : tracer.startSpan(name);
+      return adaptSpan(otelSpan);
+    },
+  };
+}
+
+function adaptSpan(otel: OtelSpanLike): Span {
+  let ended = false;
+  return {
+    setAttribute(key, value) {
+      if (ended) return;
+      otel.setAttribute(key, value);
+    },
+    setAttributes(attrs) {
+      if (ended) return;
+      if (typeof otel.setAttributes === 'function') {
+        otel.setAttributes(attrs);
+      } else {
+        for (const [k, v] of Object.entries(attrs)) otel.setAttribute(k, v);
+      }
+    },
+    recordException(err) {
+      if (ended) return;
+      if (typeof otel.recordException === 'function') {
+        otel.recordException(err);
+      } else {
+        otel.setAttribute(
+          'exception.message',
+          err instanceof Error ? err.message : String(err),
+        );
+      }
+    },
+    setStatus(status, message) {
+      if (ended) return;
+      if (typeof otel.setStatus === 'function') {
+        const code = status === 'ok' ? OTEL_STATUS_OK : OTEL_STATUS_ERROR;
+        otel.setStatus(message !== undefined ? { code, message } : { code });
+      }
+    },
+    end() {
+      if (ended) return;
+      ended = true;
+      otel.end();
+    },
+  };
+}
+
+function envEnabled(name: string): boolean {
+  const proc =
+    typeof globalThis !== 'undefined'
+      ? (globalThis as unknown as { process?: { env?: Record<string, string | undefined> } }).process
+      : undefined;
+  const v = proc?.env?.[name];
+  if (v === undefined || v === '') return false;
+  return v === '1' || v.toLowerCase() === 'true';
+}