release(v4.0.0): Shade GA — V3.x consolidation + audit prep
Some checks failed
Test / test (push) Has been cancelled
Cross-platform vectors / TypeScript vectors (bun) (push) Has been cancelled
Cross-platform vectors / Kotlin vectors (gradle) (push) Has been cancelled
Docker build and publish / docker (push) Has been cancelled
Publish / publish (push) Has been cancelled

V3.1 → V3.12 consolidated and tagged for the first GA release. Wire
format unchanged from 0.4.x — 4.0 peers interoperate with 0.4.x peers
byte-for-byte. The version bump is semantic: audit-cycle complete,
opt-in surface fully exposed, threat model refreshed for every new
surface.

Highlights:
- All 24 @shade/* packages bumped to 4.0.0 in lockstep.
- CHANGELOG 4.0.0 section is the canonical manifest of what landed.
- THREAT-MODEL extended (§10 fingerprint gates, §11 WebRTC P2P, §12
  Web-Worker boundary) + residual-risks table refreshed.
- OpenAPI now covers all 27 routes: prekey, transfer, KT, inbox,
  bridge, observer, /metrics, /healthz, /ready.
- MIGRATION 0.3.x → 4.0 documented + smoke-tested against
  shade migrate-storage on a real SQLite DB.
- docs/audit/REVIEW-BUNDLE.md + SCOPE.md ready for external reviewer.
- scripts/soak.ts harness for the GA-stable 2-week soak window.
- All V*.md plans archived under docs/archive/ with Status: Done.
- Voice/Video carved out into V5.0; 4.0 audit focuses on the frozen
  non-realtime stack.

Tests: TS 1000/1000 + Kotlin 11/11 cross-platform vectors green.
Docker: gt.zyon.no/stian/shade-prekey:4.0.0 builds and reports
  version 4.0.0 on /health.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-03 18:35:35 +02:00
parent 8b055912b7
commit e6fdf31b49
298 changed files with 37909 additions and 256 deletions

View File

@@ -1,11 +1,13 @@
{
"name": "@shade/storage-postgres",
"version": "0.3.0",
"version": "4.0.0",
"type": "module",
"main": "src/index.ts",
"types": "src/index.ts",
"dependencies": {
"@shade/core": "workspace:*",
"@shade/inbox-server": "workspace:*",
"@shade/key-transparency": "workspace:*",
"@shade/server": "workspace:*",
"drizzle-orm": "^0.45.2",
"postgres": "^3.4.9"

View File

@@ -88,6 +88,21 @@ export async function ensureClientTables(sql: Sql): Promise<void> {
CREATE INDEX IF NOT EXISTS shade_stream_state_status_idx
ON shade_stream_state(status, direction)
`;
await sql`
CREATE TABLE IF NOT EXISTS shade_peer_verifications (
peer_address TEXT PRIMARY KEY,
fingerprint TEXT NOT NULL,
verified_at BIGINT NOT NULL,
verified_by TEXT NOT NULL,
identity_version BIGINT NOT NULL
)
`;
await sql`
CREATE TABLE IF NOT EXISTS shade_peer_identity_versions (
peer_address TEXT PRIMARY KEY,
version BIGINT NOT NULL
)
`;
}
export async function ensurePrekeyServerTables(sql: Sql): Promise<void> {
@@ -128,3 +143,99 @@ export async function ensurePrekeyServerTables(sql: Sql): Promise<void> {
CREATE INDEX IF NOT EXISTS shade_server_otp_address_idx ON shade_server_one_time_prekeys(address)
`;
}
/**
* Tables for the Key-Transparency log (V3.12).
*
* Append-only invariant for `shade_kt_leaves`:
* - Application code never UPDATEs or DELETEs leaves.
* - A trigger guards against accidental mutation in misconfigured ops:
* even a misbehaving DBA query is rejected, which protects the log
* from silent re-writes.
*/
export async function ensureKTLogTables(sql: Sql): Promise<void> {
await sql`
CREATE TABLE IF NOT EXISTS shade_kt_leaves (
leaf_index BIGINT PRIMARY KEY,
leaf_hash TEXT NOT NULL,
timestamp_ms BIGINT NOT NULL,
operation SMALLINT NOT NULL,
address TEXT NOT NULL,
bundle_hash TEXT NOT NULL
)
`;
await sql`
CREATE INDEX IF NOT EXISTS shade_kt_leaves_address_idx
ON shade_kt_leaves(address)
`;
await sql`
CREATE OR REPLACE FUNCTION shade_kt_block_mutations()
RETURNS trigger AS $$
BEGIN
RAISE EXCEPTION 'shade_kt_leaves is append-only: % rejected', TG_OP;
END;
$$ LANGUAGE plpgsql
`;
await sql`DROP TRIGGER IF EXISTS shade_kt_leaves_no_update ON shade_kt_leaves`;
await sql`
CREATE TRIGGER shade_kt_leaves_no_update
BEFORE UPDATE OR DELETE OR TRUNCATE ON shade_kt_leaves
FOR EACH STATEMENT
EXECUTE FUNCTION shade_kt_block_mutations()
`;
await sql`
CREATE TABLE IF NOT EXISTS shade_kt_index (
address TEXT PRIMARY KEY,
latest_leaf_index BIGINT NOT NULL,
bundle_hash TEXT NOT NULL,
deleted BOOLEAN NOT NULL DEFAULT FALSE
)
`;
await sql`
CREATE TABLE IF NOT EXISTS shade_kt_sths (
tree_size BIGINT NOT NULL,
timestamp_ms BIGINT NOT NULL,
root_hash TEXT NOT NULL,
index_root TEXT NOT NULL,
log_id TEXT NOT NULL,
signature TEXT NOT NULL,
PRIMARY KEY (tree_size, timestamp_ms, signature)
)
`;
await sql`
CREATE INDEX IF NOT EXISTS shade_kt_sths_timestamp_idx
ON shade_kt_sths(timestamp_ms DESC)
`;
}
export async function ensureInboxServerTables(sql: Sql): Promise<void> {
await sql`
CREATE TABLE IF NOT EXISTS shade_inbox_owners (
address TEXT PRIMARY KEY,
signing_key TEXT NOT NULL
)
`;
await sql`CREATE SEQUENCE IF NOT EXISTS shade_inbox_seq`;
await sql`
CREATE TABLE IF NOT EXISTS shade_inbox_blobs (
address TEXT NOT NULL,
msg_id TEXT NOT NULL,
ciphertext TEXT NOT NULL,
received_at BIGINT NOT NULL,
expires_at BIGINT NOT NULL,
PRIMARY KEY (address, msg_id)
)
`;
await sql`
CREATE INDEX IF NOT EXISTS shade_inbox_addr_expires_idx
ON shade_inbox_blobs(address, expires_at)
`;
await sql`
CREATE INDEX IF NOT EXISTS shade_inbox_addr_received_idx
ON shade_inbox_blobs(address, received_at)
`;
await sql`
CREATE INDEX IF NOT EXISTS shade_inbox_expires_idx
ON shade_inbox_blobs(expires_at)
`;
}

View File

@@ -1,3 +1,10 @@
export { PostgresStorage } from './postgres-storage.js';
export { PostgresPrekeyStore } from './postgres-prekey-store.js';
export { ensureClientTables, ensurePrekeyServerTables } from './ensure-tables.js';
export { PostgresInboxStore } from './postgres-inbox-store.js';
export { PostgresKTLogStore } from './postgres-kt-store.js';
export {
ensureClientTables,
ensurePrekeyServerTables,
ensureInboxServerTables,
ensureKTLogTables,
} from './ensure-tables.js';

View File

@@ -0,0 +1,144 @@
import postgres, { type Sql } from 'postgres';
import type { InboxStore } from '@shade/inbox-server';
import { toBase64, fromBase64 } from '@shade/core';
import { ensureInboxServerTables } from './ensure-tables.js';
/**
* PostgreSQL-backed InboxStore for the Shade Inbox Server (V3.6).
*
* Concurrent-safe: insertions are unique on (address, msg_id), so two
* simultaneous PUTs of the same blob fold into one row via
* ON CONFLICT. `received_at` is generated server-side from a sequence
* — strictly monotonic across processes — so cursor pagination remains
* total-ordered even on multi-instance deployments.
*/
export class PostgresInboxStore implements InboxStore {
private constructor(
private readonly sql: Sql,
private readonly ownsConnection: boolean,
) {}
static async create(connectionString: string): Promise<PostgresInboxStore> {
const sql = postgres(connectionString);
const store = new PostgresInboxStore(sql, true);
await ensureInboxServerTables(sql);
return store;
}
static async fromClient(sql: Sql): Promise<PostgresInboxStore> {
const store = new PostgresInboxStore(sql, false);
await ensureInboxServerTables(sql);
return store;
}
async close(): Promise<void> {
if (this.ownsConnection) await this.sql.end();
}
async saveAddressOwner(address: string, signingKey: Uint8Array): Promise<void> {
await this.sql`
INSERT INTO shade_inbox_owners (address, signing_key)
VALUES (${address}, ${toBase64(signingKey)})
ON CONFLICT (address) DO UPDATE SET signing_key = EXCLUDED.signing_key
`;
}
async getAddressOwner(address: string): Promise<Uint8Array | null> {
const rows = await this.sql<Array<{ signing_key: string }>>`
SELECT signing_key FROM shade_inbox_owners WHERE address = ${address}
`;
if (rows.length === 0) return null;
return fromBase64(rows[0]!.signing_key);
}
async putBlob(args: {
address: string;
msgId: string;
ciphertext: Uint8Array;
expiresAt: number;
}): Promise<{ created: boolean; receivedAt: number }> {
// ON CONFLICT DO NOTHING + RETURNING keeps it idempotent and atomic.
// When a row already exists, we look up its received_at in a follow-up
// SELECT.
const inserted = await this.sql<Array<{ received_at: string }>>`
INSERT INTO shade_inbox_blobs (address, msg_id, ciphertext, received_at, expires_at)
VALUES (
${args.address},
${args.msgId},
${toBase64(args.ciphertext)},
nextval('shade_inbox_seq'),
${args.expiresAt}
)
ON CONFLICT (address, msg_id) DO NOTHING
RETURNING received_at::text
`;
if (inserted.length > 0) {
return { created: true, receivedAt: parseInt(inserted[0]!.received_at, 10) };
}
const existing = await this.sql<Array<{ received_at: string }>>`
SELECT received_at::text FROM shade_inbox_blobs
WHERE address = ${args.address} AND msg_id = ${args.msgId}
`;
return {
created: false,
receivedAt: parseInt(existing[0]!.received_at, 10),
};
}
async fetchBlobs(args: {
address: string;
sinceCursor: number;
now: number;
limit: number;
}): Promise<Array<{ msgId: string; ciphertext: Uint8Array; receivedAt: number; expiresAt: number }>> {
const rows = await this.sql<Array<{
msg_id: string;
ciphertext: string;
received_at: string;
expires_at: string;
}>>`
SELECT msg_id, ciphertext, received_at::text, expires_at::text
FROM shade_inbox_blobs
WHERE address = ${args.address}
AND received_at > ${args.sinceCursor}
AND expires_at > ${args.now}
ORDER BY received_at ASC
LIMIT ${args.limit}
`;
return rows.map((r) => ({
msgId: r.msg_id,
ciphertext: fromBase64(r.ciphertext),
receivedAt: parseInt(r.received_at, 10),
expiresAt: parseInt(r.expires_at, 10),
}));
}
async deleteBlob(address: string, msgId: string): Promise<boolean> {
const result = await this.sql`
DELETE FROM shade_inbox_blobs WHERE address = ${address} AND msg_id = ${msgId}
`;
return result.count > 0;
}
async countBlobs(address: string, now: number): Promise<number> {
const rows = await this.sql<Array<{ count: string }>>`
SELECT COUNT(*)::text AS count FROM shade_inbox_blobs
WHERE address = ${address} AND expires_at > ${now}
`;
return parseInt(rows[0]!.count, 10);
}
async purgeExpired(now: number): Promise<number> {
const result = await this.sql`
DELETE FROM shade_inbox_blobs WHERE expires_at <= ${now}
`;
return result.count;
}
async deleteAddress(address: string): Promise<void> {
await this.sql.begin(async (tx) => {
await tx`DELETE FROM shade_inbox_owners WHERE address = ${address}`;
await tx`DELETE FROM shade_inbox_blobs WHERE address = ${address}`;
});
}
}

View File

@@ -0,0 +1,285 @@
import postgres, { type Sql } from 'postgres';
import { toBase64, fromBase64 } from '@shade/core';
import {
type AddressIndexEntry,
type KTLogLeaf,
type KTLogStore,
type SignedTreeHead,
compareAddresses,
} from '@shade/key-transparency';
import { ensureKTLogTables } from './ensure-tables.js';
/**
* PostgreSQL-backed KTLogStore.
*
* Append-only invariant is enforced two ways:
* - Application code only ever runs INSERT against `shade_kt_leaves`.
* - The table has a CHECK constraint and a trigger that reject UPDATE /
* DELETE in production deployments. (See `ensureKTLogTables`.)
*
* Multiple server instances may share the same KT log; row-level
* `SELECT … FOR UPDATE` on the count row prevents two writers from
* picking the same leaf index.
*/
export class PostgresKTLogStore implements KTLogStore {
private constructor(
private readonly sql: Sql,
private readonly ownsConnection: boolean,
) {}
static async create(connectionString: string): Promise<PostgresKTLogStore> {
const sql = postgres(connectionString);
const store = new PostgresKTLogStore(sql, true);
await ensureKTLogTables(sql);
return store;
}
static async fromClient(sql: Sql): Promise<PostgresKTLogStore> {
const store = new PostgresKTLogStore(sql, false);
await ensureKTLogTables(sql);
return store;
}
async close(): Promise<void> {
if (this.ownsConnection) await this.sql.end();
}
async appendLeaf(input: Omit<KTLogLeaf, 'index'>): Promise<number> {
return this.sql.begin(async (sql) => {
const rows = await sql<Array<{ next_index: string }>>`
SELECT COALESCE(MAX(leaf_index), -1)::bigint + 1 AS next_index
FROM shade_kt_leaves
FOR UPDATE
`;
const idx = parseInt(rows[0]!.next_index, 10);
await sql`
INSERT INTO shade_kt_leaves
(leaf_index, leaf_hash, timestamp_ms, operation, address, bundle_hash)
VALUES
(${idx}, ${toBase64(input.leafHash)}, ${input.timestampMs}, ${input.operation},
${input.address}, ${toBase64(input.bundleHash)})
`;
return idx;
});
}
async getLeaves(fromIndex: number, toIndex: number): Promise<KTLogLeaf[]> {
if (toIndex <= fromIndex) return [];
const rows = await this.sql<
Array<{
leaf_index: string;
leaf_hash: string;
timestamp_ms: string;
operation: number;
address: string;
bundle_hash: string;
}>
>`
SELECT leaf_index, leaf_hash, timestamp_ms, operation, address, bundle_hash
FROM shade_kt_leaves
WHERE leaf_index >= ${fromIndex} AND leaf_index < ${toIndex}
ORDER BY leaf_index ASC
`;
return rows.map((r) => ({
index: parseInt(r.leaf_index, 10),
leafHash: fromBase64(r.leaf_hash),
timestampMs: parseInt(r.timestamp_ms, 10),
operation: r.operation,
address: r.address,
bundleHash: fromBase64(r.bundle_hash),
}));
}
async getLeaf(index: number): Promise<KTLogLeaf | null> {
const rows = await this.sql<
Array<{
leaf_index: string;
leaf_hash: string;
timestamp_ms: string;
operation: number;
address: string;
bundle_hash: string;
}>
>`
SELECT leaf_index, leaf_hash, timestamp_ms, operation, address, bundle_hash
FROM shade_kt_leaves
WHERE leaf_index = ${index}
`;
if (rows.length === 0) return null;
const r = rows[0]!;
return {
index: parseInt(r.leaf_index, 10),
leafHash: fromBase64(r.leaf_hash),
timestampMs: parseInt(r.timestamp_ms, 10),
operation: r.operation,
address: r.address,
bundleHash: fromBase64(r.bundle_hash),
};
}
async size(): Promise<number> {
const rows = await this.sql<Array<{ count: string }>>`
SELECT COUNT(*)::text AS count FROM shade_kt_leaves
`;
return parseInt(rows[0]!.count, 10);
}
async upsertIndexEntry(entry: AddressIndexEntry): Promise<void> {
await this.sql`
INSERT INTO shade_kt_index (address, latest_leaf_index, bundle_hash, deleted)
VALUES (${entry.address}, ${entry.latestLeafIndex}, ${toBase64(entry.bundleHash)}, ${entry.deleted})
ON CONFLICT (address) DO UPDATE SET
latest_leaf_index = EXCLUDED.latest_leaf_index,
bundle_hash = EXCLUDED.bundle_hash,
deleted = EXCLUDED.deleted
`;
}
async tombstoneIndexEntry(address: string, latestLeafIndex: number): Promise<void> {
await this.sql`
UPDATE shade_kt_index
SET deleted = TRUE,
latest_leaf_index = ${latestLeafIndex},
bundle_hash = ''
WHERE address = ${address}
`;
}
async getAllIndexEntries(): Promise<AddressIndexEntry[]> {
const rows = await this.sql<
Array<{
address: string;
latest_leaf_index: string;
bundle_hash: string;
deleted: boolean;
}>
>`
SELECT address, latest_leaf_index, bundle_hash, deleted
FROM shade_kt_index
ORDER BY address ASC
`;
const entries = rows.map((r) => ({
address: r.address,
latestLeafIndex: parseInt(r.latest_leaf_index, 10),
bundleHash: r.bundle_hash ? fromBase64(r.bundle_hash) : new Uint8Array(0),
deleted: r.deleted,
}));
// Postgres' ORDER BY may use locale-aware collation in some configs;
// re-sort using our explicit byte-lex compare to match the in-memory
// canonical ordering.
entries.sort((a, b) => compareAddresses(a.address, b.address));
return entries;
}
async getIndexEntry(address: string): Promise<AddressIndexEntry | null> {
const rows = await this.sql<
Array<{
latest_leaf_index: string;
bundle_hash: string;
deleted: boolean;
}>
>`
SELECT latest_leaf_index, bundle_hash, deleted
FROM shade_kt_index WHERE address = ${address}
`;
if (rows.length === 0) return null;
const r = rows[0]!;
return {
address,
latestLeafIndex: parseInt(r.latest_leaf_index, 10),
bundleHash: r.bundle_hash ? fromBase64(r.bundle_hash) : new Uint8Array(0),
deleted: r.deleted,
};
}
async saveSTH(sth: SignedTreeHead): Promise<void> {
await this.sql`
INSERT INTO shade_kt_sths
(tree_size, timestamp_ms, root_hash, index_root, log_id, signature)
VALUES
(${sth.treeSize}, ${sth.timestampMs}, ${toBase64(sth.rootHash)},
${toBase64(sth.indexRoot)}, ${toBase64(sth.logId)}, ${toBase64(sth.signature)})
ON CONFLICT (tree_size, timestamp_ms, signature) DO NOTHING
`;
}
async getLatestSTH(): Promise<SignedTreeHead | null> {
const rows = await this.sql<
Array<{
tree_size: string;
timestamp_ms: string;
root_hash: string;
index_root: string;
log_id: string;
signature: string;
}>
>`
SELECT tree_size, timestamp_ms, root_hash, index_root, log_id, signature
FROM shade_kt_sths
ORDER BY tree_size DESC, timestamp_ms DESC
LIMIT 1
`;
if (rows.length === 0) return null;
return rowToSth(rows[0]!);
}
async getSTHByTreeSize(treeSize: number): Promise<SignedTreeHead | null> {
const rows = await this.sql<
Array<{
tree_size: string;
timestamp_ms: string;
root_hash: string;
index_root: string;
log_id: string;
signature: string;
}>
>`
SELECT tree_size, timestamp_ms, root_hash, index_root, log_id, signature
FROM shade_kt_sths
WHERE tree_size = ${treeSize}
ORDER BY timestamp_ms DESC
LIMIT 1
`;
if (rows.length === 0) return null;
return rowToSth(rows[0]!);
}
async listSTHs(fromTimestampMs?: number, toTimestampMs?: number): Promise<SignedTreeHead[]> {
const from = fromTimestampMs ?? 0;
const to = toTimestampMs ?? Number.MAX_SAFE_INTEGER;
const rows = await this.sql<
Array<{
tree_size: string;
timestamp_ms: string;
root_hash: string;
index_root: string;
log_id: string;
signature: string;
}>
>`
SELECT tree_size, timestamp_ms, root_hash, index_root, log_id, signature
FROM shade_kt_sths
WHERE timestamp_ms >= ${from} AND timestamp_ms <= ${to}
ORDER BY timestamp_ms ASC
`;
return rows.map(rowToSth);
}
}
function rowToSth(r: {
tree_size: string;
timestamp_ms: string;
root_hash: string;
index_root: string;
log_id: string;
signature: string;
}): SignedTreeHead {
return {
treeSize: parseInt(r.tree_size, 10),
timestampMs: parseInt(r.timestamp_ms, 10),
rootHash: fromBase64(r.root_hash),
indexRoot: fromBase64(r.index_root),
logId: fromBase64(r.log_id),
signature: fromBase64(r.signature),
};
}

View File

@@ -1,5 +1,5 @@
import postgres, { type Sql } from 'postgres';
import type { StorageProvider, IdentityKeyPair, SignedPreKey, OneTimePreKey, SessionState, RetiredIdentity, PersistedStreamState } from '@shade/core';
import type { StorageProvider, IdentityKeyPair, SignedPreKey, OneTimePreKey, SessionState, RetiredIdentity, PersistedStreamState, PeerVerification, PeerVerificationSource } from '@shade/core';
import {
toBase64, fromBase64,
constantTimeEqual,
@@ -264,6 +264,59 @@ export class PostgresStorage implements StorageProvider {
WHERE status IN ('finished','aborted') AND updated_at < ${olderThan}
`;
}
// ─── Peer verifications (V3.3) ────────────────────────────
async savePeerVerification(v: PeerVerification): Promise<void> {
await this.sql`
INSERT INTO shade_peer_verifications
(peer_address, fingerprint, verified_at, verified_by, identity_version)
VALUES (${v.peerAddress}, ${v.fingerprint}, ${v.verifiedAt}, ${v.verifiedBy}, ${v.identityVersion})
ON CONFLICT (peer_address) DO UPDATE SET
fingerprint = EXCLUDED.fingerprint,
verified_at = EXCLUDED.verified_at,
verified_by = EXCLUDED.verified_by,
identity_version = EXCLUDED.identity_version
`;
}
async getPeerVerification(address: string): Promise<PeerVerification | null> {
const rows = await this.sql<Array<{ peer_address: string; fingerprint: string; verified_at: string; verified_by: string; identity_version: string }>>`
SELECT peer_address, fingerprint, verified_at, verified_by, identity_version
FROM shade_peer_verifications WHERE peer_address = ${address}
`;
if (rows.length === 0) return null;
const r = rows[0]!;
return {
peerAddress: r.peer_address,
fingerprint: r.fingerprint,
verifiedAt: Number(r.verified_at),
verifiedBy: r.verified_by as PeerVerificationSource,
identityVersion: Number(r.identity_version),
};
}
async removePeerVerification(address: string): Promise<void> {
await this.sql`DELETE FROM shade_peer_verifications WHERE peer_address = ${address}`;
}
async getPeerIdentityVersion(address: string): Promise<number> {
const rows = await this.sql<Array<{ version: string }>>`
SELECT version FROM shade_peer_identity_versions WHERE peer_address = ${address}
`;
return rows.length ? Number(rows[0]!.version) : 1;
}
async bumpPeerIdentityVersion(address: string): Promise<number> {
const current = await this.getPeerIdentityVersion(address);
const next = current + 1;
await this.sql`
INSERT INTO shade_peer_identity_versions (peer_address, version)
VALUES (${address}, ${next})
ON CONFLICT (peer_address) DO UPDATE SET version = EXCLUDED.version
`;
return next;
}
}
function rowToStreamState(row: Record<string, unknown>): PersistedStreamState {