Files
Shade/packages/shade-storage-postgres/src/ensure-tables.ts

283 lines
8.4 KiB
TypeScript
Raw Normal View History

import type { Sql } from 'postgres';
/**
* Auto-create all Shade tables if they don't exist.
*
* Called on PostgresStorage / PostgresPrekeyStore construction.
* Uses raw SQL (not Drizzle migrations) for zero-config deployment.
*
* All tables prefixed with `shade_` to avoid collisions when sharing
* a PostgreSQL instance with another project.
*/
export async function ensureClientTables(sql: Sql): Promise<void> {
await sql`
CREATE TABLE IF NOT EXISTS shade_identity (
id INTEGER PRIMARY KEY CHECK (id = 1),
signing_public_key TEXT NOT NULL,
signing_private_key TEXT NOT NULL,
dh_public_key TEXT NOT NULL,
dh_private_key TEXT NOT NULL
)
`;
await sql`
CREATE TABLE IF NOT EXISTS shade_config (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
)
`;
await sql`
CREATE TABLE IF NOT EXISTS shade_signed_prekeys (
key_id INTEGER PRIMARY KEY,
data_json TEXT NOT NULL
)
`;
await sql`
CREATE TABLE IF NOT EXISTS shade_one_time_prekeys (
key_id INTEGER PRIMARY KEY,
data_json TEXT NOT NULL
)
`;
await sql`
CREATE TABLE IF NOT EXISTS shade_sessions (
address TEXT PRIMARY KEY,
state_json TEXT NOT NULL
)
`;
fix(session): remember where aliasSession moved a session aliasSession knew that two labels name the same peer, then threw that knowledge away. The binding lived only in the caller's memory, so a restart lost it — and the peer could not repair it from its side. First contact forces the receiver to label a session by the only sender hint a relay surfaces, an 8-byte signing-key fingerprint (`fp:<hex>`). Once the peer announces its canonical address, aliasSession moves the session there. But the peer keeps sending under `fp:<hex>`, because its transport derives the same label from the same hint every time. After a restart the session sat under the canonical address, inbound frames resolved to `fp:<hex>`, and nothing matched. The peer held a valid session so it never re-ran X3DH: the failure was permanent, and only a manual re-link cleared it. Observed in Prism as `No session for address: fp:579c3b335d66e2c0` on every receive for three days, with a phone whose every RPC timed out. StorageProvider gains saveSessionAlias / getSessionAlias / removeSessionAliasesFor, optional so third-party implementations keep compiling, and implemented across all seven backends. Lookups resolve through resolveLabel(), which runs BEFORE the peer mutex — locking the alias while mutating the canonical session would let an aliased and a canonical caller ratchet the same state concurrently. A live session under a label always wins over an alias, and prekey envelopes never resolve: both keep a re-link establishing a fresh session instead of being redirected into the stale one. Aliases are dropped in resetSession and acceptIdentityChange, and memoized so the hot path costs no extra read. The sdk.test.ts case that asserted a dead fp-label encoded the old behaviour; it now pins the new contract. Verified: 1166 tests pass (from 1160). With alias persistence disabled as a negative control, 5 of the 6 new tests fail, including both restart cases. Also drops `baseUrl` from the consumer-strict tsconfig — removed in TS 6.0, and it was failing the typecheck that gates publishing. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-13 19:36:31 +02:00
await sql`
CREATE TABLE IF NOT EXISTS shade_session_aliases (
alias TEXT PRIMARY KEY,
canonical TEXT NOT NULL
)
`;
await sql`
CREATE INDEX IF NOT EXISTS idx_shade_session_aliases_canonical
ON shade_session_aliases(canonical)
`;
await sql`
CREATE TABLE IF NOT EXISTS shade_trusted_identities (
address TEXT PRIMARY KEY,
identity_key TEXT NOT NULL
)
`;
await sql`
CREATE TABLE IF NOT EXISTS shade_retired_identities (
id SERIAL PRIMARY KEY,
data_json TEXT NOT NULL,
retired_at BIGINT NOT NULL
)
`;
await sql`
CREATE INDEX IF NOT EXISTS shade_retired_at_idx ON shade_retired_identities(retired_at)
`;
feat(files): @shade/files 0.3.0 — E2EE filesystem RPC primitive M-Files-1..6 land the full files-RPC layer + everything 0.3.0 needs to ship. Apps keep their own UI; this layer ships the typed RPC, the streams bridge for content I/O, and production hooks (rate limit, retention, fingerprint gate, metrics). @shade/files (NEW) - Standard ops: list/stat/mkdir/delete/move/read/write/getThumbnail with Zod-validated wire schemas + clean user-handler types. - Custom ops: typed via TypeScript declaration merging on CustomOpsMap + per-op Zod schemas; client.custom('app.foo', {...}) is fully typed. - Content I/O: inline (≤ 256 KiB plaintext) base64-in-RPC; streams (> 256 KiB) ride @shade/transfer via userMetadata.shadeFilesWriteId / shadeFilesReadStreamId correlation. Server-side TransformStream bridges accept inbound transfers immediately (engine rejects chunks that arrive before accept) and park the readable for the matching RPC. - Directory ops: walk(path, opts) async-iterable depth-first walker; uploadDirectory()/downloadDirectory() with bounded concurrency pool (default 4, cap 16), aggregated progress, abort. - Production hooks (callback-based, vendor-neutral): rate-limit (op + byte), idempotency cache (LRU + TTL + in-flight de-dupe), path policy (traversal + percent-decode hardening), fingerprint gate (required/optional/reject), pluggable Ed25519 sig verification with ±5 min replay window, onMetric sink (standard names). - React hooks (subpath @shade/files/react): ShadeFilesProvider, useShadeFiles, useFileList, useFileTransfer/Upload/Download. - Shade.files.serve(handler) + Shade.files.client(peer) high-level entrypoint in @shade/sdk; lazy + memoized; one handler per Shade. Wire format bump - @shade/proto wire VERSION 0x01 → 0x02. Length prefixes changed from u16 to u32. The previous u16 silently truncated payloads above 64 KiB — a hard correctness ceiling that blocked inline file ops up to 256 KiB. Wire-incompatible with 0.2.x peers; new sessions only. Cross-platform Kotlin port (android/shade-android) updated to match; test-vectors/wire-format.json regenerated. Concurrency safety - ShadeSessionManager.encrypt/.decrypt now run under per-peer mutex. Concurrent decryptions of the same peer raced ratchet state (manifested as sporadic "Failed to decrypt — wrong key or tampered data" under load — surfaced once concurrent uploadDirectory pumped many writes in flight). Encrypt was already serialized via Shade.send's encryptChains; decrypt is now serialized at the manager layer too. @shade/streams extension - StreamMetadata.userMetadata?: Record<string, string> for application-level key/value pairs that round-trip verbatim through stream-init plaintext. Used by @shade/files for write/read correlation; available to any consumer. @shade/sdk extension - Shade.files getter (lazy + memoized). - BackgroundHooks.onPruneFiles + periodic timer (default 5 min) + BackgroundTasks.setHook(name, fn) for runtime hook registration. Bundles in-flight 0.2.0 work - packages/shade-streams/, packages/shade-transfer/, related shade-sdk streams-bridge + shade-widgets transfer hooks were uncommitted prior to this session. Including them keeps the workspace consistent at 0.3.0 since @shade/files depends on them. Tests - 74 new tests in @shade/files (572 → 646 workspace pass; 0 fail; 3× stable). Coverage spans unit (inline-threshold + concurrency), integration (read-write inline + streams up to 1 MiB, walk + upload/download directory, custom-op, metrics, SDK namespace end-to-end), and security (tampered-envelope sig verification, replay window, fingerprint gate, rate-limit + quota). Release artifacts - All packages bumped to 0.3.0 via scripts/bump-version.ts. - scripts/publish-all.ts PACKAGES updated with shade-files in topological order (after shade-transfer, before shade-sdk). - bun run publish:dry clean (14 packed, 0 failed). - examples/08-files-browser/ — three-process CLI demo (prekey + Bob server + Alice CLI) covering list/stat/mkdir/delete/upload/download. - docs/files.md — full API + design doc. - CHANGELOG.md 0.3.0 entry. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-02 14:00:01 +02:00
await sql`
CREATE TABLE IF NOT EXISTS shade_stream_state (
stream_id TEXT PRIMARY KEY,
direction TEXT NOT NULL CHECK (direction IN ('send','receive')),
peer_address TEXT NOT NULL,
status TEXT NOT NULL CHECK (status IN ('active','paused','finished','aborted')),
metadata_json TEXT NOT NULL,
partition_json TEXT NOT NULL,
lane_state_json TEXT NOT NULL,
io_descriptor_json TEXT NOT NULL,
secret_enc BYTEA NOT NULL,
secret_nonce BYTEA NOT NULL,
overall_hash_state TEXT,
created_at BIGINT NOT NULL,
updated_at BIGINT NOT NULL
)
`;
await sql`
CREATE INDEX IF NOT EXISTS shade_stream_state_peer_idx
ON shade_stream_state(peer_address)
`;
await sql`
CREATE INDEX IF NOT EXISTS shade_stream_state_updated_idx
ON shade_stream_state(updated_at)
`;
await sql`
CREATE INDEX IF NOT EXISTS shade_stream_state_status_idx
ON shade_stream_state(status, direction)
`;
await sql`
CREATE TABLE IF NOT EXISTS shade_peer_verifications (
peer_address TEXT PRIMARY KEY,
fingerprint TEXT NOT NULL,
verified_at BIGINT NOT NULL,
verified_by TEXT NOT NULL,
identity_version BIGINT NOT NULL
)
`;
await sql`
CREATE TABLE IF NOT EXISTS shade_peer_identity_versions (
peer_address TEXT PRIMARY KEY,
version BIGINT NOT NULL
)
`;
}
export async function ensurePrekeyServerTables(sql: Sql): Promise<void> {
await sql`
CREATE TABLE IF NOT EXISTS shade_server_identities (
address TEXT PRIMARY KEY,
identity_signing_key TEXT NOT NULL,
feat(container): M-Box 1-8 — stack-agnostic standalone Docker container Shade now ships as a self-contained Docker image. Deploy one container per project, any stack (Bun, Python, Go, Rust, Kotlin) can talk to it via plain HTTP. Zero coupling to consumer codebases. M-Box 1: Stale identity cleanup API - touchIdentity + purgeStaleIdentities on PrekeyStore interface - Implemented for Memory, SQLite, and Postgres backends - SQLite adds last_activity_at column with migration ALTER for existing DBs - Postgres adds the same via raw SQL with IF NOT EXISTS guards - Routes call touchIdentity on register, bundle fetch, replenish - 4 new tests for the cleanup API M-Box 2: Stale cleanup background task - StaleCleanupTask runs purge on startup + every 24h (configurable) - Reads SHADE_STALE_DAYS (default 30) and SHADE_CLEANUP_INTERVAL_HOURS - Wired into standalone.ts, stopped on graceful shutdown - 5 new tests for the task M-Box 3: Observer baked into the container - standalone.ts conditionally mounts @shade/observer at /shade-observer when SHADE_OBSERVER_TOKEN is set (and >= 16 chars) - Shared PrekeyServerEvents emitter feeds both routes and observer - @shade/observer added as optional dependency of @shade/server M-Box 4: Dockerfile with dashboard build - Multi-stage build: oven/bun:1 builder → oven/bun:1-alpine runtime - COPY packages/ wholesale so workspace lockfile resolves cleanly - RUN bun run build inside shade-dashboard → dist/ → observer/dist/ - Non-root shade user, /data volume, healthcheck, env defaults - Final image: 260 MB M-Box 5: OpenAPI spec for stack-agnostic clients - packages/shade-server/openapi.yaml documents all 9 endpoints with request/response schemas, security (Ed25519 signatures + bearer token) - createOpenApiRoutes serves /openapi.yaml and /docs (Redoc viewer) - Any language can generate a client with openapi-generator M-Box 6: Docker CI pipeline - .gitea/workflows/docker.yml builds + pushes on git tag v* - scripts/build-docker.ts for local builds, supports --push with GITEA_TOKEN - Root package.json: build:docker, publish:docker scripts M-Box 7: Deployment documentation - packages/shade-server/README rewritten: 5-line quickstart with the image - docs/DEPLOYMENT.md: full reference, env vars, backup, Dokploy, PG setup - examples/05-dokploy-deployment/docker-compose.yml updated to pull published image (gt.zyon.no/stian/shade-prekey:latest) - Root README deployment section rewritten M-Box 8: End-to-end verification - Image builds locally (bun run build:docker) - /health, /openapi.yaml, /docs, /metrics, /shade-observer all respond - 401 without observer token, 200 with - Real SDK client round-trip: Alice → container → Bob → reply → Alice - Persistence: identity + prekeys survive container restart (count 20→18 as expected from two bundle fetches) 285 tests passing, 0 failures. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-11 14:29:00 +02:00
identity_dh_key TEXT NOT NULL,
last_activity_at BIGINT NOT NULL DEFAULT 0
)
`;
feat(container): M-Box 1-8 — stack-agnostic standalone Docker container Shade now ships as a self-contained Docker image. Deploy one container per project, any stack (Bun, Python, Go, Rust, Kotlin) can talk to it via plain HTTP. Zero coupling to consumer codebases. M-Box 1: Stale identity cleanup API - touchIdentity + purgeStaleIdentities on PrekeyStore interface - Implemented for Memory, SQLite, and Postgres backends - SQLite adds last_activity_at column with migration ALTER for existing DBs - Postgres adds the same via raw SQL with IF NOT EXISTS guards - Routes call touchIdentity on register, bundle fetch, replenish - 4 new tests for the cleanup API M-Box 2: Stale cleanup background task - StaleCleanupTask runs purge on startup + every 24h (configurable) - Reads SHADE_STALE_DAYS (default 30) and SHADE_CLEANUP_INTERVAL_HOURS - Wired into standalone.ts, stopped on graceful shutdown - 5 new tests for the task M-Box 3: Observer baked into the container - standalone.ts conditionally mounts @shade/observer at /shade-observer when SHADE_OBSERVER_TOKEN is set (and >= 16 chars) - Shared PrekeyServerEvents emitter feeds both routes and observer - @shade/observer added as optional dependency of @shade/server M-Box 4: Dockerfile with dashboard build - Multi-stage build: oven/bun:1 builder → oven/bun:1-alpine runtime - COPY packages/ wholesale so workspace lockfile resolves cleanly - RUN bun run build inside shade-dashboard → dist/ → observer/dist/ - Non-root shade user, /data volume, healthcheck, env defaults - Final image: 260 MB M-Box 5: OpenAPI spec for stack-agnostic clients - packages/shade-server/openapi.yaml documents all 9 endpoints with request/response schemas, security (Ed25519 signatures + bearer token) - createOpenApiRoutes serves /openapi.yaml and /docs (Redoc viewer) - Any language can generate a client with openapi-generator M-Box 6: Docker CI pipeline - .gitea/workflows/docker.yml builds + pushes on git tag v* - scripts/build-docker.ts for local builds, supports --push with GITEA_TOKEN - Root package.json: build:docker, publish:docker scripts M-Box 7: Deployment documentation - packages/shade-server/README rewritten: 5-line quickstart with the image - docs/DEPLOYMENT.md: full reference, env vars, backup, Dokploy, PG setup - examples/05-dokploy-deployment/docker-compose.yml updated to pull published image (gt.zyon.no/stian/shade-prekey:latest) - Root README deployment section rewritten M-Box 8: End-to-end verification - Image builds locally (bun run build:docker) - /health, /openapi.yaml, /docs, /metrics, /shade-observer all respond - 401 without observer token, 200 with - Real SDK client round-trip: Alice → container → Bob → reply → Alice - Persistence: identity + prekeys survive container restart (count 20→18 as expected from two bundle fetches) 285 tests passing, 0 failures. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-11 14:29:00 +02:00
// Migrate existing deployments (no-op if column exists)
await sql`
ALTER TABLE shade_server_identities
ADD COLUMN IF NOT EXISTS last_activity_at BIGINT NOT NULL DEFAULT 0
`;
await sql`
CREATE INDEX IF NOT EXISTS shade_server_identities_activity_idx
ON shade_server_identities(last_activity_at)
`;
await sql`
CREATE TABLE IF NOT EXISTS shade_server_signed_prekeys (
address TEXT PRIMARY KEY,
key_id INTEGER NOT NULL,
public_key TEXT NOT NULL,
signature TEXT NOT NULL
)
`;
await sql`
CREATE TABLE IF NOT EXISTS shade_server_one_time_prekeys (
id SERIAL PRIMARY KEY,
address TEXT NOT NULL,
key_id INTEGER NOT NULL,
public_key TEXT NOT NULL
)
`;
await sql`
CREATE INDEX IF NOT EXISTS shade_server_otp_address_idx ON shade_server_one_time_prekeys(address)
`;
}
/**
* Tables for the Key-Transparency log (V3.12).
*
* Append-only invariant for `shade_kt_leaves`:
* - Application code never UPDATEs or DELETEs leaves.
* - A trigger guards against accidental mutation in misconfigured ops:
* even a misbehaving DBA query is rejected, which protects the log
* from silent re-writes.
*/
export async function ensureKTLogTables(sql: Sql): Promise<void> {
await sql`
CREATE TABLE IF NOT EXISTS shade_kt_leaves (
leaf_index BIGINT PRIMARY KEY,
leaf_hash TEXT NOT NULL,
timestamp_ms BIGINT NOT NULL,
operation SMALLINT NOT NULL,
address TEXT NOT NULL,
bundle_hash TEXT NOT NULL
)
`;
await sql`
CREATE INDEX IF NOT EXISTS shade_kt_leaves_address_idx
ON shade_kt_leaves(address)
`;
await sql`
CREATE OR REPLACE FUNCTION shade_kt_block_mutations()
RETURNS trigger AS $$
BEGIN
RAISE EXCEPTION 'shade_kt_leaves is append-only: % rejected', TG_OP;
END;
$$ LANGUAGE plpgsql
`;
await sql`DROP TRIGGER IF EXISTS shade_kt_leaves_no_update ON shade_kt_leaves`;
await sql`
CREATE TRIGGER shade_kt_leaves_no_update
BEFORE UPDATE OR DELETE OR TRUNCATE ON shade_kt_leaves
FOR EACH STATEMENT
EXECUTE FUNCTION shade_kt_block_mutations()
`;
await sql`
CREATE TABLE IF NOT EXISTS shade_kt_index (
address TEXT PRIMARY KEY,
latest_leaf_index BIGINT NOT NULL,
bundle_hash TEXT NOT NULL,
deleted BOOLEAN NOT NULL DEFAULT FALSE
)
`;
await sql`
CREATE TABLE IF NOT EXISTS shade_kt_sths (
tree_size BIGINT NOT NULL,
timestamp_ms BIGINT NOT NULL,
root_hash TEXT NOT NULL,
index_root TEXT NOT NULL,
log_id TEXT NOT NULL,
signature TEXT NOT NULL,
PRIMARY KEY (tree_size, timestamp_ms, signature)
)
`;
await sql`
CREATE INDEX IF NOT EXISTS shade_kt_sths_timestamp_idx
ON shade_kt_sths(timestamp_ms DESC)
`;
}
/**
* V4.9 — encrypted-blob primitive (`/v1/blob/<slotId>`). One row per
* slot, keyed on the 64-hex slotId. ETag is a sequence value so it's
* unique and monotonic across writers (matches the inbox `received_at`
* pattern). The blob column holds base64-encoded AEAD ciphertext —
* the relay never decrypts.
*/
export async function ensureBlobServerTables(sql: Sql): Promise<void> {
await sql`CREATE SEQUENCE IF NOT EXISTS shade_blob_seq`;
await sql`
CREATE TABLE IF NOT EXISTS shade_blob_slots (
slot_id TEXT PRIMARY KEY,
owner_pubkey TEXT NOT NULL,
blob TEXT NOT NULL,
etag BIGINT NOT NULL,
updated_at BIGINT NOT NULL
)
`;
await sql`
CREATE INDEX IF NOT EXISTS shade_blob_updated_idx
ON shade_blob_slots(updated_at)
`;
}
export async function ensureInboxServerTables(sql: Sql): Promise<void> {
await sql`
CREATE TABLE IF NOT EXISTS shade_inbox_owners (
address TEXT PRIMARY KEY,
signing_key TEXT NOT NULL
)
`;
await sql`CREATE SEQUENCE IF NOT EXISTS shade_inbox_seq`;
await sql`
CREATE TABLE IF NOT EXISTS shade_inbox_blobs (
address TEXT NOT NULL,
msg_id TEXT NOT NULL,
ciphertext TEXT NOT NULL,
received_at BIGINT NOT NULL,
expires_at BIGINT NOT NULL,
sender_fp TEXT,
PRIMARY KEY (address, msg_id)
)
`;
// V4.8 — sender fingerprint column. Idempotent ADD COLUMN for live
// databases that came up under a 4.7-or-earlier schema.
await sql`
ALTER TABLE shade_inbox_blobs
ADD COLUMN IF NOT EXISTS sender_fp TEXT
`;
await sql`
CREATE INDEX IF NOT EXISTS shade_inbox_addr_expires_idx
ON shade_inbox_blobs(address, expires_at)
`;
await sql`
CREATE INDEX IF NOT EXISTS shade_inbox_addr_received_idx
ON shade_inbox_blobs(address, received_at)
`;
await sql`
CREATE INDEX IF NOT EXISTS shade_inbox_expires_idx
ON shade_inbox_blobs(expires_at)
`;
}