diff --git a/docs/vault.md b/docs/vault.md new file mode 100644 index 0000000..1d8ff13 --- /dev/null +++ b/docs/vault.md @@ -0,0 +1,126 @@ +# `@shade/vault` — server-side encrypted file store + +V4.13. Where the blob primitive (V4.9) holds one small blob per account, a +vault holds a whole **collection** — a workspace, a mailbox, a drive folder — +as content-addressed objects plus an append-only log of manifests. + +It is the piece Shade was missing to let an app offer backup, or to let a +client read data while the peer that owns it is switched off. + +## The model + +**Objects are content-addressed.** An object's name is the SHA-256 of its +*ciphertext*, so the relay can store, dedupe and verify without a key: it +recomputes the hash on upload and rejects anything mislabelled. + +**A manifest names the collection.** Paths live inside the encrypted manifest, +never in object names — the relay must not learn that a user has a file called +`Projects/Divorce/plan.md`. The manifest is itself stored as an object. + +**The log is append-only.** Each commit adds a row pointing at a manifest. +History, rollback and "what changed last Tuesday" all fall out of that. + +## Keys + +Three deterministic derivations off the account master key, in their own HKDF +branch: + +``` +vaultId = HKDF(masterKey, "shade-vault-id-v1:") +contentKey = HKDF(masterKey, "shade-vault-content-v1:") +sigSeed = HKDF(masterKey, "shade-vault-sig-v1:") +``` + +Separate from `shade-blob-*-v1` on purpose: a vault key reads every file, a +profile-blob key reads a host list. Sharing a derivation would turn one +compromise into the other. + +Because the derivation is deterministic, **recovery is credentials**. A fresh +device that can derive the profile master can derive these, and the relay hands +over ciphertext it has never been able to read. The flip side is that recovery +strength is password strength — see G1's argon2id item. + +## Client + +```ts +import { SubtleCryptoProvider } from '@shade/crypto-web'; +import { VaultClient, HttpVaultTransport, deriveVaultKeys } from '@shade/vault'; + +const crypto = new SubtleCryptoProvider(); +const keys = await deriveVaultKeys(masterKey, 'scaffold'); +const client = new VaultClient(crypto, keys, new HttpVaultTransport('https://vault.example')); + +// Back up. Unchanged files are not re-uploaded. +const res = await client.push(files, Date.now(), 'nightly'); +// → { seq: 4, uploaded: 2, reused: 489, bytesUploaded: 1_204 } + +// Restore — the newest, or any earlier version. +const { manifest, files: restored } = await client.pull(); +const old = await client.pull(2); + +// History. +const log = await client.history(); +``` + +`push` takes `at` rather than reading the clock, so a queued backup records +when it was *taken* rather than when it finally reached the relay. + +## Server + +```ts +import { createVaultRoutes } from '@shade/vault/server'; +import { SqliteVaultStore } from '@shade/storage-sqlite'; + +app.route('/', createVaultRoutes(new SqliteVaultStore('/data/shade-vault.db'), crypto)); +``` + +Routes: + +| | | +|---|---| +| `GET /v1/vault/:id/log` | `{ entries, head }` | +| `HEAD /v1/vault/:id/object/:hash` | 200 / 404 — the have-check | +| `GET /v1/vault/:id/object/:hash` | raw ciphertext | +| `PUT /v1/vault/:id/object/:hash` | signed; verifies hash matches bytes | +| `POST /v1/vault/:id/commit` | signed; refuses a stale `seq` or a missing object | + +Auth is TOFU-Ed25519, as for blobs: the first signed write pins a pubkey, and +every later write must be signed by it. The pubkey travels **inside** the +signed payload — appended afterwards it would not be covered by the signature, +and could be swapped in transit on the pinning write. + +### Configuration + +``` +SHADE_VAULT_DB_PATH=/data/shade-vault.db # required — no in-memory fallback +SHADE_DISABLE_VAULT=1 # optional: off entirely +``` + +**There is deliberately no in-memory fallback.** The blob store has one, and on +2026-08-12 a routine redeploy destroyed every profile because the path was +never set in compose: the fallback worked perfectly right up until the +container was recreated, and nothing ever failed loudly. A vault that forgets +is worse than no vault, so an unset path means the routes are simply not +mounted, with a warning that says why. + +### Limits + +Per object 8 MiB, per vault 512 MiB, both overridable via `VaultRoutesOptions`. + +## What the relay learns + +Object sizes, how many objects there are, when commits happen, and that some +opaque 64-hex id is active. Not contents, not paths, not filenames, and not +which user a vaultId belongs to — the id is itself derived from a secret. + +Deduplication is **within** a version chain, not across independent seals: +sealing uses a fresh nonce, so identical plaintext yields different names. +Convergent encryption would dedupe globally but leak which files two users +share, which is exactly what a blind store must not do. + +## Status + +Client, routes, `MemoryVaultStore` and `SqliteVaultStore` are implemented and +tested (16 unit + 7 store tests, plus an end-to-end suite against a real +workspace of 491 files). Not yet: a Postgres store, rate limiting on the vault +routes, and garbage collection of objects no manifest references any more. diff --git a/packages/shade-server/package.json b/packages/shade-server/package.json index 4d31225..0a7f98b 100644 --- a/packages/shade-server/package.json +++ b/packages/shade-server/package.json @@ -9,6 +9,8 @@ "@shade/inbox-server": "workspace:*", "@shade/key-transparency": "workspace:*", "@shade/observability": "workspace:*", + "@shade/storage-sqlite": "workspace:*", + "@shade/vault": "workspace:*", "hono": "^4.12.12" }, "optionalDependencies": { diff --git a/packages/shade-server/src/standalone.ts b/packages/shade-server/src/standalone.ts index ba32df5..282b30e 100644 --- a/packages/shade-server/src/standalone.ts +++ b/packages/shade-server/src/standalone.ts @@ -83,6 +83,30 @@ async function createInboxStore(): Promise void | P * also opt the blob store *off* entirely via `SHADE_DISABLE_BLOB=1` — * useful for relays that only want the inbox surface. */ +/** + * Build the vault store, or `null` when none is configured. + * + * Deliberately NOT falling back to an in-memory store the way the blob + * primitive does. A vault holds the user's actual files: an in-memory one + * accepts every write, serves every read, and loses the lot on the next + * container recreate — a backup that reports success and is not there. The + * blob store's silent fallback destroyed Prism's profile on 2026-08-12 + * precisely because nothing failed loudly. Here, no path means no vault. + */ +async function createVaultStore(): Promise { + const sqlitePath = process.env.SHADE_VAULT_DB_PATH; + if (!sqlitePath) { + logger.warn( + 'Vault not enabled — set SHADE_VAULT_DB_PATH to a path on a persistent volume. ' + + 'There is no in-memory fallback on purpose: a vault that forgets is not a backup.', + ); + return null; + } + const { SqliteVaultStore } = await import('@shade/storage-sqlite'); + logger.info('Using SQLite vault store', { path: sqlitePath }); + return new SqliteVaultStore(sqlitePath); +} + async function createBlobStore(): Promise void | Promise }> { const sqlitePath = process.env.SHADE_BLOB_DB_PATH; const pgUrl = process.env.SHADE_BLOB_PG_URL ?? process.env.SHADE_PREKEY_PG_URL; @@ -255,6 +279,21 @@ if (blobDisabled) { logger.info('Blob primitive enabled', { route: '/v1/blob/:slotId' }); } +// V4.13 — vault: server-side encrypted file store. Where the blob primitive +// holds one small blob per account, a vault holds a whole collection with an +// append-only version log. Off unless a store is configured, because a vault +// that silently lives in RAM is worse than no vault at all — see the warning +// on SHADE_VAULT_DB_PATH below. +const vaultDisabled = process.env.SHADE_DISABLE_VAULT === '1'; +const vaultStore = vaultDisabled ? null : await createVaultStore(); +if (vaultDisabled) { + logger.info('Vault disabled (SHADE_DISABLE_VAULT=1)'); +} else if (vaultStore) { + const { createVaultRoutes } = await import('@shade/vault/server'); + app.route('/', createVaultRoutes(vaultStore, crypto)); + logger.info('Vault enabled', { route: '/v1/vault/:vaultId' }); +} + // ─── Optional: Observer + Dashboard ────────────────────────── const observerToken = process.env.SHADE_OBSERVER_TOKEN; diff --git a/packages/shade-storage-sqlite/package.json b/packages/shade-storage-sqlite/package.json index 4dd8f6b..eec6f49 100644 --- a/packages/shade-storage-sqlite/package.json +++ b/packages/shade-storage-sqlite/package.json @@ -8,6 +8,7 @@ "@shade/core": "workspace:*", "@shade/crypto-web": "workspace:*", "@shade/inbox-server": "workspace:*", - "@shade/server": "workspace:*" + "@shade/server": "workspace:*", + "@shade/vault": "workspace:*" } } diff --git a/packages/shade-storage-sqlite/src/index.ts b/packages/shade-storage-sqlite/src/index.ts index f52e442..26b169a 100644 --- a/packages/shade-storage-sqlite/src/index.ts +++ b/packages/shade-storage-sqlite/src/index.ts @@ -2,3 +2,4 @@ export { SQLiteStorage } from './sqlite-storage.js'; export { SqlitePrekeyStore } from './sqlite-prekey-store.js'; export { SqliteInboxStore } from './sqlite-inbox-store.js'; export { SqliteBlobStore } from './sqlite-blob-store.js'; +export { SqliteVaultStore } from './sqlite-vault-store.js'; diff --git a/packages/shade-storage-sqlite/src/sqlite-vault-store.ts b/packages/shade-storage-sqlite/src/sqlite-vault-store.ts new file mode 100644 index 0000000..154bb8b --- /dev/null +++ b/packages/shade-storage-sqlite/src/sqlite-vault-store.ts @@ -0,0 +1,154 @@ +import { Database } from 'bun:sqlite'; +import type { VaultLogEntry, VaultStore } from '@shade/vault'; + +/** + * SQLite-backed VaultStore for the V4.13 encrypted file store. + * + * Three tables, mirroring the model: an owner key per vault, the + * content-addressed objects, and the append-only manifest log. The relay + * never decrypts anything — it enforces auth, verifies that an object's bytes + * hash to its name, and refuses a commit whose objects are not all present. + * + * Objects are stored as BLOBs rather than base64 text. A workspace backup is + * a few hundred files where the blob primitive holds one small profile, so the + * 33% base64 overhead stops being a rounding error. + * + * Docker usage: set `SHADE_VAULT_DB_PATH` (falls back to `/data/shade-vault.db`). + * **Set it.** The blob store's equivalent was missing from `docker-compose.yml` + * for months and nobody noticed, because the in-memory fallback works + * perfectly until the container is recreated — at which point everything is + * gone, with no error anywhere. That happened on 2026-08-12. + */ +export class SqliteVaultStore implements VaultStore { + private db: Database; + private stmts!: { + getOwner: ReturnType; + setOwner: ReturnType; + hasObject: ReturnType; + putObject: ReturnType; + getObject: ReturnType; + head: ReturnType; + log: ReturnType; + logLimit: ReturnType; + appendLog: ReturnType; + usage: ReturnType; + }; + + constructor(dbPath?: string) { + const path = dbPath ?? process.env.SHADE_VAULT_DB_PATH ?? '/data/shade-vault.db'; + this.db = new Database(path, { create: true }); + this.db.exec('PRAGMA journal_mode=WAL'); + this.ensureTables(); + this.prepareStatements(); + } + + private ensureTables() { + this.db.exec(` + CREATE TABLE IF NOT EXISTS shade_vault_owners ( + vault_id TEXT PRIMARY KEY, + owner_pubkey BLOB NOT NULL, + created_at INTEGER NOT NULL + ); + + CREATE TABLE IF NOT EXISTS shade_vault_objects ( + vault_id TEXT NOT NULL, + hash TEXT NOT NULL, + bytes BLOB NOT NULL, + size INTEGER NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (vault_id, hash) + ); + + CREATE TABLE IF NOT EXISTS shade_vault_log ( + vault_id TEXT NOT NULL, + seq INTEGER NOT NULL, + manifest TEXT NOT NULL, + at INTEGER NOT NULL, + bytes INTEGER NOT NULL, + PRIMARY KEY (vault_id, seq) + ); + `); + } + + private prepareStatements() { + this.stmts = { + getOwner: this.db.prepare('SELECT owner_pubkey FROM shade_vault_owners WHERE vault_id = ?'), + setOwner: this.db.prepare( + 'INSERT OR REPLACE INTO shade_vault_owners (vault_id, owner_pubkey, created_at) VALUES (?, ?, ?)', + ), + hasObject: this.db.prepare( + 'SELECT 1 FROM shade_vault_objects WHERE vault_id = ? AND hash = ? LIMIT 1', + ), + // An object's name IS its content hash, so a repeat upload is the same + // bytes by definition — ignoring it is correct, not lossy. + putObject: this.db.prepare( + 'INSERT OR IGNORE INTO shade_vault_objects (vault_id, hash, bytes, size, created_at) VALUES (?, ?, ?, ?, ?)', + ), + getObject: this.db.prepare( + 'SELECT bytes FROM shade_vault_objects WHERE vault_id = ? AND hash = ?', + ), + head: this.db.prepare('SELECT MAX(seq) AS head FROM shade_vault_log WHERE vault_id = ?'), + log: this.db.prepare( + 'SELECT seq, manifest, at, bytes FROM shade_vault_log WHERE vault_id = ? ORDER BY seq ASC', + ), + logLimit: this.db.prepare( + 'SELECT seq, manifest, at, bytes FROM (SELECT seq, manifest, at, bytes FROM shade_vault_log WHERE vault_id = ? ORDER BY seq DESC LIMIT ?) ORDER BY seq ASC', + ), + appendLog: this.db.prepare( + 'INSERT INTO shade_vault_log (vault_id, seq, manifest, at, bytes) VALUES (?, ?, ?, ?, ?)', + ), + usage: this.db.prepare( + 'SELECT COALESCE(SUM(size), 0) AS total FROM shade_vault_objects WHERE vault_id = ?', + ), + }; + } + + async getOwner(vaultId: string): Promise { + const row = this.stmts.getOwner.get(vaultId) as { owner_pubkey: Uint8Array } | null; + return row ? new Uint8Array(row.owner_pubkey) : null; + } + + async setOwner(vaultId: string, publicKey: Uint8Array): Promise { + this.stmts.setOwner.run(vaultId, publicKey, Date.now()); + } + + async hasObject(vaultId: string, hash: string): Promise { + return this.stmts.hasObject.get(vaultId, hash) !== null; + } + + async putObject(vaultId: string, hash: string, bytes: Uint8Array): Promise { + this.stmts.putObject.run(vaultId, hash, bytes, bytes.length, Date.now()); + } + + async getObject(vaultId: string, hash: string): Promise { + const row = this.stmts.getObject.get(vaultId, hash) as { bytes: Uint8Array } | null; + return row ? new Uint8Array(row.bytes) : null; + } + + async head(vaultId: string): Promise { + const row = this.stmts.head.get(vaultId) as { head: number | null } | null; + return row?.head ?? 0; + } + + async log(vaultId: string, limit?: number): Promise { + const rows = ( + limit === undefined + ? this.stmts.log.all(vaultId) + : this.stmts.logLimit.all(vaultId, limit) + ) as VaultLogEntry[]; + return rows; + } + + async appendLog(vaultId: string, entry: VaultLogEntry): Promise { + this.stmts.appendLog.run(vaultId, entry.seq, entry.manifest, entry.at, entry.bytes); + } + + async usage(vaultId: string): Promise { + const row = this.stmts.usage.get(vaultId) as { total: number } | null; + return row?.total ?? 0; + } + + close(): void { + this.db.close(); + } +} diff --git a/packages/shade-storage-sqlite/tests/sqlite-vault-store.test.ts b/packages/shade-storage-sqlite/tests/sqlite-vault-store.test.ts new file mode 100644 index 0000000..851df00 --- /dev/null +++ b/packages/shade-storage-sqlite/tests/sqlite-vault-store.test.ts @@ -0,0 +1,120 @@ +/** + * The SQLite vault store, with persistence as the headline property. + * + * This exists because of 2026-08-12: the blob store fell back to memory when + * its path was unset, worked perfectly, and lost every profile on the next + * container recreate — with no error anywhere. A backup store that forgets is + * worse than none, so "survives a close and reopen" is tested directly rather + * than assumed from the fact that SQLite is involved. + */ +import { describe, test, expect, afterEach } from 'bun:test'; +import { unlinkSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { SqliteVaultStore } from '../src/sqlite-vault-store.js'; + +const paths: string[] = []; +function scratch(name: string): string { + const p = join(tmpdir(), `shade-vault-${name}-${process.pid}-${Math.random().toString(36).slice(2)}.db`); + paths.push(p); + return p; +} + +afterEach(() => { + for (const p of paths.splice(0)) { + for (const suffix of ['', '-wal', '-shm']) { + try { + unlinkSync(p + suffix); + } catch { + // Not every WAL sidecar exists; absence is fine. + } + } + } +}); + +const VAULT = 'a'.repeat(64); +const HASH = 'b'.repeat(64); + +describe('persistence', () => { + test('objects, owner and log survive a close and reopen', async () => { + const path = scratch('reopen'); + const first = new SqliteVaultStore(path); + await first.setOwner(VAULT, new Uint8Array([1, 2, 3, 4])); + await first.putObject(VAULT, HASH, new Uint8Array([9, 8, 7])); + await first.appendLog(VAULT, { seq: 1, manifest: HASH, at: 1700, bytes: 3 }); + first.close(); + + const second = new SqliteVaultStore(path); + expect(await second.getOwner(VAULT)).toEqual(new Uint8Array([1, 2, 3, 4])); + expect(await second.getObject(VAULT, HASH)).toEqual(new Uint8Array([9, 8, 7])); + expect(await second.head(VAULT)).toBe(1); + expect((await second.log(VAULT))[0]!.manifest).toBe(HASH); + second.close(); + }); + + test('binary content round-trips without base64 mangling', async () => { + // Objects are ciphertext: every byte value occurs, including 0x00. + const path = scratch('binary'); + const store = new SqliteVaultStore(path); + const bytes = new Uint8Array(256); + for (let i = 0; i < 256; i++) bytes[i] = i; + await store.putObject(VAULT, HASH, bytes); + store.close(); + + const reopened = new SqliteVaultStore(path); + expect(await reopened.getObject(VAULT, HASH)).toEqual(bytes); + reopened.close(); + }); +}); + +describe('semantics', () => { + test('an empty vault has head 0 and no owner', async () => { + const store = new SqliteVaultStore(scratch('empty')); + expect(await store.head(VAULT)).toBe(0); + expect(await store.getOwner(VAULT)).toBeNull(); + expect(await store.log(VAULT)).toEqual([]); + expect(await store.usage(VAULT)).toBe(0); + store.close(); + }); + + test('re-putting the same hash is a no-op, not a duplicate', async () => { + // The name IS the content hash, so a repeat is the same bytes by + // definition. Usage must not double. + const store = new SqliteVaultStore(scratch('dupe')); + await store.putObject(VAULT, HASH, new Uint8Array(100)); + await store.putObject(VAULT, HASH, new Uint8Array(100)); + expect(await store.usage(VAULT)).toBe(100); + store.close(); + }); + + test('vaults are isolated from each other', async () => { + const store = new SqliteVaultStore(scratch('isolated')); + const other = 'c'.repeat(64); + await store.putObject(VAULT, HASH, new Uint8Array([1])); + expect(await store.hasObject(other, HASH)).toBe(false); + expect(await store.getObject(other, HASH)).toBeNull(); + expect(await store.usage(other)).toBe(0); + store.close(); + }); + + test('the log is ordered oldest-first and limit counts back from the head', async () => { + const store = new SqliteVaultStore(scratch('log')); + for (let seq = 1; seq <= 5; seq++) { + await store.appendLog(VAULT, { seq, manifest: `${seq}`.repeat(64), at: seq, bytes: seq }); + } + expect((await store.log(VAULT)).map((e) => e.seq)).toEqual([1, 2, 3, 4, 5]); + // A client asking for the last two wants 4 and 5, still in order. + expect((await store.log(VAULT, 2)).map((e) => e.seq)).toEqual([4, 5]); + expect(await store.head(VAULT)).toBe(5); + store.close(); + }); + + test('a duplicate seq is rejected by the primary key', async () => { + // The route layer refuses this first, but the store is the last line: + // two rows with the same seq would make history ambiguous. + const store = new SqliteVaultStore(scratch('seq')); + await store.appendLog(VAULT, { seq: 1, manifest: HASH, at: 1, bytes: 1 }); + expect(store.appendLog(VAULT, { seq: 1, manifest: HASH, at: 2, bytes: 1 })).rejects.toThrow(); + store.close(); + }); +}); diff --git a/packages/shade-vault/package.json b/packages/shade-vault/package.json new file mode 100644 index 0000000..af591c1 --- /dev/null +++ b/packages/shade-vault/package.json @@ -0,0 +1,25 @@ +{ + "name": "@shade/vault", + "version": "4.13.0", + "description": "Server-side encrypted file store for Shade \u2014 content-addressed objects with an append-only manifest log", + "type": "module", + "main": "src/index.ts", + "types": "src/index.ts", + "exports": { + ".": "./src/index.ts", + "./server": "./src/server.ts" + }, + "dependencies": { + "@noble/hashes": "^2.0.1", + "@shade/core": "workspace:*", + "@shade/crypto-web": "workspace:*", + "@shade/observability": "workspace:*", + "@shade/server": "workspace:*", + "@shade/storage-encrypted": "workspace:*", + "hono": "^4.12.18" + }, + "scripts": { + "test": "bun test", + "typecheck": "tsc --noEmit" + } +} diff --git a/packages/shade-vault/src/client.ts b/packages/shade-vault/src/client.ts new file mode 100644 index 0000000..56e67f4 --- /dev/null +++ b/packages/shade-vault/src/client.ts @@ -0,0 +1,225 @@ +/** + * The client half: turn a set of files into an encrypted, versioned vault, + * and turn it back again on a machine that has nothing but the credentials. + * + * The push is deliberately have-check-first. A workspace changes a few lines + * at a time, so asking "do you already have these hashes?" and uploading only + * the misses turns a full backup into a handful of small writes. Everything + * unchanged keeps its hash from the previous commit and costs nothing. + */ + +import type { CryptoProvider } from '@shade/core'; +import { toBase64 } from '@shade/core'; +// The pubkey derivation is not on `CryptoProvider` — it is a pure function of +// the seed, and lives with the curve implementation. +import { ed25519PublicKeyFromSeed } from '@shade/crypto-web'; +import { signPayload } from '@shade/server'; +import { + deriveVaultContentKey, + deriveVaultId, + deriveVaultSigningSeed, +} from '@shade/storage-encrypted'; +import { openManifest, openObject, sealManifest, sealObject, toHex } from './crypto.js'; +import type { VaultEntry, VaultLog, VaultManifest } from './types.js'; + +/** A file as the app sees it, before encryption. */ +export interface VaultFile { + path: string; + bytes: Uint8Array; + mode?: string; +} + +export interface VaultKeys { + vaultId: string; + contentKey: Uint8Array; + signingSeed: Uint8Array; + publicKey: Uint8Array; +} + +/** + * Derive everything a vault needs from the account master key. + * + * This is what makes recovery "username and password": a fresh device that + * can derive the profile master can derive these too, and the relay hands + * over ciphertext it has never been able to read. + */ +export async function deriveVaultKeys(masterKey: Uint8Array, app: string): Promise { + const signingSeed = deriveVaultSigningSeed(masterKey, app); + const publicKey = ed25519PublicKeyFromSeed(signingSeed); + return { + vaultId: toHex(deriveVaultId(masterKey, app)), + contentKey: deriveVaultContentKey(masterKey, app), + signingSeed, + publicKey, + }; +} + +export interface VaultTransport { + /** Does the relay already hold this object? */ + has(vaultId: string, hash: string): Promise; + put(vaultId: string, hash: string, body: unknown): Promise; + get(vaultId: string, hash: string): Promise; + log(vaultId: string, limit?: number): Promise; + commit(vaultId: string, body: unknown): Promise<{ seq: number }>; +} + +export interface PushResult { + seq: number; + /** Objects actually sent — the rest were already on the relay. */ + uploaded: number; + /** Objects skipped because the relay had them. */ + reused: number; + bytesUploaded: number; +} + +export class VaultClient { + constructor( + private readonly crypto: CryptoProvider, + private readonly keys: VaultKeys, + private readonly transport: VaultTransport, + ) {} + + get vaultId(): string { + return this.keys.vaultId; + } + + /** + * Sign a request body, with the pubkey inside the signed payload. + * + * The key has to be part of what gets signed, not appended afterwards: + * `verifyPayload` canonicalises every field, so an appended key would both + * break the signature and — if the scheme ignored it — let anyone swap the + * claimed identity in transit on a first, TOFU-pinning write. + */ + private async sign(body: Record): Promise> { + return signPayload(this.crypto, this.keys.signingSeed, { + ...body, + publicKey: toBase64(this.keys.publicKey), + }); + } + + /** + * Encrypt and upload `files`, then commit a manifest naming them. + * + * `at` is passed in rather than read from the clock so a caller can make a + * commit reproducible in tests, and so a queued backup records when it was + * *taken* rather than when it finally reached the relay. + */ + async push(files: VaultFile[], at: number, message?: string): Promise { + const { head } = await this.transport.log(this.keys.vaultId, 1); + + const entries: VaultEntry[] = []; + let uploaded = 0; + let reused = 0; + let bytesUploaded = 0; + + // Reuse the previous manifest's hashes for unchanged content: sealing is + // nondeterministic (fresh nonce), so re-sealing an untouched file would + // produce a new hash and a pointless upload every single time. + const previous = head > 0 ? await this.pull().catch(() => null) : null; + const byPath = new Map(previous?.manifest.entries.map((e) => [e.path, e]) ?? []); + const priorPlain = previous?.files ?? new Map(); + + for (const file of files) { + const prior = byPath.get(file.path); + const priorBytes = priorPlain.get(file.path); + if (prior && priorBytes && sameBytes(priorBytes, file.bytes)) { + entries.push({ ...prior, size: file.bytes.length }); + reused++; + continue; + } + + const sealed = await sealObject( + this.crypto, + this.keys.contentKey, + this.keys.vaultId, + file.bytes, + ); + if (!(await this.transport.has(this.keys.vaultId, sealed.hash))) { + await this.transport.put( + this.keys.vaultId, + sealed.hash, + await this.sign({ data: toBase64(sealed.bytes) }), + ); + uploaded++; + bytesUploaded += sealed.bytes.length; + } else { + reused++; + } + const entry: VaultEntry = { path: file.path, hash: sealed.hash, size: file.bytes.length }; + if (file.mode !== undefined) entry.mode = file.mode; + entries.push(entry); + } + + const manifest: VaultManifest = { version: 1, seq: head + 1, at, entries }; + if (message !== undefined) manifest.message = message; + + const sealedManifest = await sealManifest( + this.crypto, + this.keys.contentKey, + this.keys.vaultId, + manifest, + ); + await this.transport.put( + this.keys.vaultId, + sealedManifest.hash, + await this.sign({ data: toBase64(sealedManifest.bytes) }), + ); + + const { seq } = await this.transport.commit( + this.keys.vaultId, + await this.sign({ + manifest: sealedManifest.hash, + seq: manifest.seq, + at, + hashes: entries.map((e) => e.hash), + }), + ); + + return { seq, uploaded, reused, bytesUploaded }; + } + + /** + * Fetch a version and decrypt it. Defaults to the newest. + * + * Restoring is the whole point of the exercise, so this returns the plain + * files rather than a handle: a caller that has to make a second round of + * decisions to get their data back does not have a backup. + */ + async pull(seq?: number): Promise<{ manifest: VaultManifest; files: Map }> { + const log = await this.transport.log(this.keys.vaultId); + const wanted = seq ?? log.head; + const row = log.entries.find((e) => e.seq === wanted); + if (!row) throw new Error(`vault has no version ${wanted}`); + + const manifestBytes = await this.transport.get(this.keys.vaultId, row.manifest); + const manifest = await openManifest( + this.crypto, + this.keys.contentKey, + this.keys.vaultId, + row.manifest, + manifestBytes, + ); + + const files = new Map(); + for (const entry of manifest.entries) { + const bytes = await this.transport.get(this.keys.vaultId, entry.hash); + files.set( + entry.path, + await openObject(this.crypto, this.keys.contentKey, this.keys.vaultId, entry.hash, bytes), + ); + } + return { manifest, files }; + } + + /** Version history, newest last. */ + async history(limit?: number): Promise { + return this.transport.log(this.keys.vaultId, limit); + } +} + +function sameBytes(a: Uint8Array, b: Uint8Array): boolean { + if (a.length !== b.length) return false; + for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false; + return true; +} diff --git a/packages/shade-vault/src/crypto.ts b/packages/shade-vault/src/crypto.ts new file mode 100644 index 0000000..aca5385 --- /dev/null +++ b/packages/shade-vault/src/crypto.ts @@ -0,0 +1,129 @@ +/** + * Sealing and naming for vault objects. + * + * Every object on the relay is `nonce(12) || ciphertext||tag`, named by the + * SHA-256 of those bytes. Hashing the *ciphertext* rather than the plaintext + * is what lets the relay verify that an upload is what it claims to be + * without holding a key — it recomputes the hash and compares. + * + * The cost is that identical plaintext yields different names on each seal, + * because the nonce is fresh. Deduplication is therefore *within* a version + * chain (an unchanged file keeps its hash across commits because the client + * reuses the object it already uploaded), not across independent seals. That + * is the right trade: convergent encryption would leak which files two users + * share, which is exactly what a blind store must not do. + */ + +import { sha256 } from '@noble/hashes/sha2.js'; +import type { CryptoProvider } from '@shade/core'; +import type { VaultManifest } from './types.js'; + +const NONCE_LEN = 12; + +/** Lowercase hex SHA-256 — the name of an object. */ +export function objectHash(sealed: Uint8Array): string { + return Array.from(sha256(sealed)) + .map((b) => b.toString(16).padStart(2, '0')) + .join(''); +} + +/** + * AAD for an object. + * + * Binds the vault it belongs to, so a ciphertext lifted from one vault cannot + * be planted in another even by someone who can write to both. The content + * hash is deliberately NOT in the AAD: it is derived from the very bytes + * being sealed, so it cannot be known before sealing, and the relay's own + * hash check already covers substitution. + */ +function objectAad(vaultId: string): Uint8Array { + return new TextEncoder().encode(`shade-vault-object-v1|${vaultId}`); +} + +export interface SealedObject { + hash: string; + bytes: Uint8Array; +} + +/** Encrypt one object and give it its name. */ +export async function sealObject( + crypto: CryptoProvider, + contentKey: Uint8Array, + vaultId: string, + plaintext: Uint8Array, +): Promise { + const { ciphertext, nonce } = await crypto.aesGcmEncrypt( + contentKey, + plaintext, + objectAad(vaultId), + ); + const bytes = new Uint8Array(NONCE_LEN + ciphertext.length); + bytes.set(nonce, 0); + bytes.set(ciphertext, NONCE_LEN); + return { hash: objectHash(bytes), bytes }; +} + +/** + * Decrypt one object. + * + * Throws when the hash does not match the bytes: a relay that returned the + * wrong object would otherwise surface as an AEAD failure, which reads like + * key corruption and sends the user looking in the wrong place. + */ +export async function openObject( + crypto: CryptoProvider, + contentKey: Uint8Array, + vaultId: string, + expectedHash: string, + bytes: Uint8Array, +): Promise { + if (bytes.length < NONCE_LEN + 16) { + throw new Error('vault object too short to be a sealed object'); + } + const actual = objectHash(bytes); + if (actual !== expectedHash) { + throw new Error(`vault object hash mismatch: expected ${expectedHash}, got ${actual}`); + } + return crypto.aesGcmDecrypt( + contentKey, + bytes.subarray(NONCE_LEN), + bytes.subarray(0, NONCE_LEN), + objectAad(vaultId), + ); +} + +/** A manifest is stored as an ordinary object; this is the JSON step around it. */ +export async function sealManifest( + crypto: CryptoProvider, + contentKey: Uint8Array, + vaultId: string, + manifest: VaultManifest, +): Promise { + const json = new TextEncoder().encode(JSON.stringify(manifest)); + return sealObject(crypto, contentKey, vaultId, json); +} + +export async function openManifest( + crypto: CryptoProvider, + contentKey: Uint8Array, + vaultId: string, + expectedHash: string, + bytes: Uint8Array, +): Promise { + const plain = await openObject(crypto, contentKey, vaultId, expectedHash, bytes); + const parsed = JSON.parse(new TextDecoder().decode(plain)) as VaultManifest; + if (parsed.version !== 1) { + // Refusing beats guessing: a future manifest may mean a file the reader + // cannot represent, and silently dropping it would lose data on the next + // commit made from this client. + throw new Error(`unsupported vault manifest version: ${parsed.version}`); + } + return parsed; +} + +/** Lowercase hex of arbitrary bytes — used for vaultId and pubkeys on the wire. */ +export function toHex(bytes: Uint8Array): string { + return Array.from(bytes) + .map((b) => b.toString(16).padStart(2, '0')) + .join(''); +} diff --git a/packages/shade-vault/src/http-transport.ts b/packages/shade-vault/src/http-transport.ts new file mode 100644 index 0000000..00bfa6c --- /dev/null +++ b/packages/shade-vault/src/http-transport.ts @@ -0,0 +1,80 @@ +/** + * The default transport: plain HTTP against a relay running the vault routes. + * + * Takes a `fetch` rather than calling the global one, so the same client works + * against a live server, a Hono app under test, and anything else that can + * answer a Request — which is how the tests exercise the real route handlers + * instead of a mock that agrees with them by construction. + */ + +import { fromBase64 } from '@shade/core'; +import type { VaultTransport } from './client.js'; +import type { VaultLog } from './types.js'; + +type FetchLike = (input: string, init?: RequestInit) => Promise; + +export class HttpVaultTransport implements VaultTransport { + constructor( + private readonly baseUrl: string, + private readonly fetchImpl: FetchLike = globalThis.fetch.bind(globalThis), + ) {} + + private url(path: string): string { + return `${this.baseUrl.replace(/\/$/, '')}${path}`; + } + + /** Turn a non-2xx into an Error that names the relay's own code. */ + private async fail(res: Response, what: string): Promise { + let detail = res.statusText; + try { + const body = (await res.json()) as { error?: { code?: string; message?: string } }; + if (body?.error) detail = `${body.error.code}: ${body.error.message}`; + } catch { + // Non-JSON error body; the status line is all we have. + } + throw new Error(`vault ${what} failed (${res.status}) — ${detail}`); + } + + async has(vaultId: string, hash: string): Promise { + const res = await this.fetchImpl(this.url(`/v1/vault/${vaultId}/object/${hash}`), { + method: 'HEAD', + }); + if (res.status === 200) return true; + if (res.status === 404) return false; + return this.fail(res, 'have-check'); + } + + async put(vaultId: string, hash: string, body: unknown): Promise { + const res = await this.fetchImpl(this.url(`/v1/vault/${vaultId}/object/${hash}`), { + method: 'PUT', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + }); + if (!res.ok) return this.fail(res, 'upload'); + } + + async get(vaultId: string, hash: string): Promise { + const res = await this.fetchImpl(this.url(`/v1/vault/${vaultId}/object/${hash}`)); + if (!res.ok) return this.fail(res, 'download'); + return new Uint8Array(await res.arrayBuffer()); + } + + async log(vaultId: string, limit?: number): Promise { + const q = limit === undefined ? '' : `?limit=${limit}`; + const res = await this.fetchImpl(this.url(`/v1/vault/${vaultId}/log${q}`)); + if (!res.ok) return this.fail(res, 'log'); + return (await res.json()) as VaultLog; + } + + async commit(vaultId: string, body: unknown): Promise<{ seq: number }> { + const res = await this.fetchImpl(this.url(`/v1/vault/${vaultId}/commit`), { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + }); + if (!res.ok) return this.fail(res, 'commit'); + return (await res.json()) as { seq: number }; + } +} + +export { fromBase64 }; diff --git a/packages/shade-vault/src/index.ts b/packages/shade-vault/src/index.ts new file mode 100644 index 0000000..31df631 --- /dev/null +++ b/packages/shade-vault/src/index.ts @@ -0,0 +1,6 @@ +export * from './types.js'; +export * from './crypto.js'; +export * from './client.js'; +export { MemoryVaultStore } from './store.js'; +export type { VaultStore } from './store.js'; +export { HttpVaultTransport } from './http-transport.js'; diff --git a/packages/shade-vault/src/server.ts b/packages/shade-vault/src/server.ts new file mode 100644 index 0000000..b4e0432 --- /dev/null +++ b/packages/shade-vault/src/server.ts @@ -0,0 +1,226 @@ +/** + * Relay-side vault routes. + * + * GET /v1/vault/:vaultId/log → { entries, head } + * GET /v1/vault/:vaultId/object/:hash → raw ciphertext bytes + * HEAD /v1/vault/:vaultId/object/:hash → 200 | 404 (have-check) + * PUT /v1/vault/:vaultId/object/:hash → { stored } (signed) + * POST /v1/vault/:vaultId/commit → { seq } (signed) + * + * Auth is the same TOFU-Ed25519 scheme the blob primitive uses: the first + * signed write pins a pubkey for the vault, and every later write must be + * signed by it. There is no account, no password, and nothing for the relay + * to leak — it cannot even tell which user a vaultId belongs to. + */ + +import { Hono } from 'hono'; +import type { ContentfulStatusCode } from 'hono/utils/http-status'; +import type { CryptoProvider } from '@shade/core'; +import { fromBase64 } from '@shade/core'; +import { verifyPayload } from '@shade/server'; +import { objectHash } from './crypto.js'; +import type { VaultStore } from './store.js'; +import type { VaultManifest } from './types.js'; + +const ID_REGEX = /^[0-9a-f]{64}$/; +const HASH_REGEX = /^[0-9a-f]{64}$/; + +export interface VaultRoutesOptions { + /** Per-object ceiling. Defaults to 8 MiB. */ + maxObjectBytes?: number; + /** Whole-vault ceiling. Defaults to 512 MiB. */ + maxVaultBytes?: number; +} + +const DEFAULT_MAX_OBJECT = 8 * 1024 * 1024; +const DEFAULT_MAX_VAULT = 512 * 1024 * 1024; + +function fail(code: string, message: string, status: ContentfulStatusCode) { + return { body: { error: { code, message } }, status }; +} + +export function createVaultRoutes( + store: VaultStore, + crypto: CryptoProvider, + options: VaultRoutesOptions = {}, +): Hono { + const app = new Hono(); + const maxObject = options.maxObjectBytes ?? DEFAULT_MAX_OBJECT; + const maxVault = options.maxVaultBytes ?? DEFAULT_MAX_VAULT; + + /** + * Check a signed request against the vault's pinned key, pinning it on the + * first write. `publicKey` is only honoured when nothing is pinned yet — + * otherwise anyone could rotate the owner by simply asserting a new key. + */ + async function authorize( + vaultId: string, + payload: Record, + ): Promise< + { ok: true } | { ok: false; code: string; message: string; status: ContentfulStatusCode } + > { + const claimed = typeof payload.publicKey === 'string' ? payload.publicKey : null; + const pinned = await store.getOwner(vaultId); + + if (!pinned) { + if (!claimed) { + return { ok: false, code: 'UNAUTHORIZED', message: 'first write must carry publicKey', status: 401 }; + } + const key = fromBase64(claimed); + try { + await verifyPayload(crypto, key, payload); + } catch (e) { + return { ok: false, code: 'UNAUTHORIZED', message: String((e as Error).message), status: 401 }; + } + await store.setOwner(vaultId, key); + return { ok: true }; + } + + try { + await verifyPayload(crypto, pinned, payload); + } catch (e) { + return { ok: false, code: 'UNAUTHORIZED', message: String((e as Error).message), status: 401 }; + } + return { ok: true }; + } + + app.get('/v1/vault/:vaultId/log', async (c) => { + const vaultId = c.req.param('vaultId'); + if (!ID_REGEX.test(vaultId)) { + const f = fail('BAD_REQUEST', 'vaultId must be 64 lowercase hex chars', 400); + return c.json(f.body, f.status); + } + const limitRaw = c.req.query('limit'); + const limit = limitRaw ? Number(limitRaw) : undefined; + const entries = await store.log(vaultId, Number.isFinite(limit) ? limit : undefined); + return c.json({ entries, head: await store.head(vaultId) }); + }); + + // A have-check before uploading. This is what makes an unchanged file free: + // the client asks about every hash in the new manifest and only sends the + // ones the relay is missing. + app.on(['HEAD', 'GET'], '/v1/vault/:vaultId/object/:hash', async (c) => { + const vaultId = c.req.param('vaultId'); + const hash = c.req.param('hash'); + if (!ID_REGEX.test(vaultId) || !HASH_REGEX.test(hash)) { + const f = fail('BAD_REQUEST', 'bad vaultId or hash', 400); + return c.json(f.body, f.status); + } + if (c.req.method === 'HEAD') { + return c.body(null, (await store.hasObject(vaultId, hash)) ? 200 : 404); + } + const bytes = await store.getObject(vaultId, hash); + if (!bytes) { + const f = fail('NOT_FOUND', 'no such object', 404); + return c.json(f.body, f.status); + } + return c.body(bytes as unknown as ArrayBuffer, 200, { + 'content-type': 'application/octet-stream', + }); + }); + + app.put('/v1/vault/:vaultId/object/:hash', async (c) => { + const vaultId = c.req.param('vaultId'); + const hash = c.req.param('hash'); + if (!ID_REGEX.test(vaultId) || !HASH_REGEX.test(hash)) { + const f = fail('BAD_REQUEST', 'bad vaultId or hash', 400); + return c.json(f.body, f.status); + } + + const body = (await c.req.json().catch(() => null)) as Record | null; + if (!body || typeof body.data !== 'string') { + const f = fail('BAD_REQUEST', 'expected { data, signedAt, signature }', 400); + return c.json(f.body, f.status); + } + + const auth = await authorize(vaultId, body); + if (!auth.ok) { + const f = fail(auth.code, auth.message, auth.status); + return c.json(f.body, f.status); + } + + const bytes = fromBase64(body.data); + if (bytes.length > maxObject) { + const f = fail('TOO_LARGE', `object exceeds ${maxObject} bytes`, 413); + return c.json(f.body, f.status); + } + if ((await store.usage(vaultId)) + bytes.length > maxVault) { + const f = fail('TOO_LARGE', `vault exceeds ${maxVault} bytes`, 413); + return c.json(f.body, f.status); + } + + // The name must be the hash of the bytes. Without this the store would + // accept a mislabelled object, and every later read of that name would + // fail decryption somewhere far away from the cause. + const actual = objectHash(bytes); + if (actual !== hash) { + const f = fail('BAD_REQUEST', `hash mismatch: bytes hash to ${actual}`, 400); + return c.json(f.body, f.status); + } + + await store.putObject(vaultId, hash, bytes); + return c.json({ stored: true, hash }); + }); + + app.post('/v1/vault/:vaultId/commit', async (c) => { + const vaultId = c.req.param('vaultId'); + if (!ID_REGEX.test(vaultId)) { + const f = fail('BAD_REQUEST', 'vaultId must be 64 lowercase hex chars', 400); + return c.json(f.body, f.status); + } + + const body = (await c.req.json().catch(() => null)) as Record | null; + if (!body || typeof body.manifest !== 'string' || typeof body.seq !== 'number') { + const f = fail('BAD_REQUEST', 'expected { manifest, seq, hashes, signedAt, signature }', 400); + return c.json(f.body, f.status); + } + + const auth = await authorize(vaultId, body); + if (!auth.ok) { + const f = fail(auth.code, auth.message, auth.status); + return c.json(f.body, f.status); + } + + const head = await store.head(vaultId); + if (body.seq !== head + 1) { + // Two devices committed from the same head. The loser has to re-read + // and re-commit; retrying the same seq would overwrite a version that + // is already part of the history. + const f = fail('SEQ_CONFLICT', `expected seq ${head + 1}, got ${body.seq}`, 409); + return c.json({ ...f.body, head }, f.status); + } + + if (!(await store.hasObject(vaultId, body.manifest))) { + const f = fail('MISSING_OBJECTS', 'manifest object not uploaded', 409); + return c.json(f.body, f.status); + } + + // Every object the manifest references must already be here, or the + // commit would publish a version that cannot be restored. Checking at + // commit time is what makes the log trustworthy. + const hashes = Array.isArray(body.hashes) ? (body.hashes as string[]) : []; + const missing: string[] = []; + for (const h of hashes) { + if (!HASH_REGEX.test(h) || !(await store.hasObject(vaultId, h))) missing.push(h); + } + if (missing.length > 0) { + const f = fail('MISSING_OBJECTS', `${missing.length} referenced object(s) missing`, 409); + return c.json({ ...f.body, missing: missing.slice(0, 20) }, f.status); + } + + await store.appendLog(vaultId, { + seq: body.seq, + manifest: body.manifest, + at: typeof body.at === 'number' ? body.at : Date.now(), + bytes: await store.usage(vaultId), + }); + + return c.json({ seq: body.seq, head: body.seq }); + }); + + return app; +} + +export type { VaultStore } from './store.js'; +export { MemoryVaultStore } from './store.js'; +export type { VaultManifest }; diff --git a/packages/shade-vault/src/store.ts b/packages/shade-vault/src/store.ts new file mode 100644 index 0000000..ed36f87 --- /dev/null +++ b/packages/shade-vault/src/store.ts @@ -0,0 +1,90 @@ +/** + * Relay-side storage for a vault. + * + * The interface is deliberately dumb: put bytes under a hash, list a log, + * append to it. Nothing here can decrypt, and nothing needs to — which is + * what makes a SQLite, Postgres or object-store implementation equivalent. + */ + +import type { VaultLogEntry } from './types.js'; + +export interface VaultStore { + /** Ed25519 pubkey pinned on first write (TOFU), or null for a fresh vault. */ + getOwner(vaultId: string): Promise; + setOwner(vaultId: string, publicKey: Uint8Array): Promise; + + hasObject(vaultId: string, hash: string): Promise; + putObject(vaultId: string, hash: string, bytes: Uint8Array): Promise; + getObject(vaultId: string, hash: string): Promise; + + /** Highest committed seq, or 0 when empty. */ + head(vaultId: string): Promise; + /** Newest last. `limit` counts back from the head. */ + log(vaultId: string, limit?: number): Promise; + appendLog(vaultId: string, entry: VaultLogEntry): Promise; + + /** Total stored bytes, for quota decisions. */ + usage(vaultId: string): Promise; +} + +/** + * In-memory store. Real for tests, a footgun in production — the same shape + * as `MemoryBlobStore`, and the same warning applies: a restart loses + * everything, and it will do so without an error. + */ +export class MemoryVaultStore implements VaultStore { + private owners = new Map(); + private objects = new Map>(); + private logs = new Map(); + + async getOwner(vaultId: string): Promise { + return this.owners.get(vaultId) ?? null; + } + + async setOwner(vaultId: string, publicKey: Uint8Array): Promise { + this.owners.set(vaultId, publicKey); + } + + private bucket(vaultId: string): Map { + let b = this.objects.get(vaultId); + if (!b) { + b = new Map(); + this.objects.set(vaultId, b); + } + return b; + } + + async hasObject(vaultId: string, hash: string): Promise { + return this.bucket(vaultId).has(hash); + } + + async putObject(vaultId: string, hash: string, bytes: Uint8Array): Promise { + this.bucket(vaultId).set(hash, bytes); + } + + async getObject(vaultId: string, hash: string): Promise { + return this.bucket(vaultId).get(hash) ?? null; + } + + async head(vaultId: string): Promise { + const l = this.logs.get(vaultId); + return l && l.length > 0 ? l[l.length - 1]!.seq : 0; + } + + async log(vaultId: string, limit?: number): Promise { + const l = this.logs.get(vaultId) ?? []; + return limit === undefined ? [...l] : l.slice(Math.max(0, l.length - limit)); + } + + async appendLog(vaultId: string, entry: VaultLogEntry): Promise { + const l = this.logs.get(vaultId) ?? []; + l.push(entry); + this.logs.set(vaultId, l); + } + + async usage(vaultId: string): Promise { + let total = 0; + for (const bytes of this.bucket(vaultId).values()) total += bytes.length; + return total; + } +} diff --git a/packages/shade-vault/src/types.ts b/packages/shade-vault/src/types.ts new file mode 100644 index 0000000..cf5c924 --- /dev/null +++ b/packages/shade-vault/src/types.ts @@ -0,0 +1,79 @@ +/** + * The vault's data model. + * + * A vault is a collection of files that lives encrypted on the relay: the + * pieces Shade was missing to let an app offer backup, or to let a client + * read data while the peer that owns it is switched off. + * + * Three ideas, and the shape follows from them: + * + * 1. **Objects are content-addressed.** An object's name is the hash of its + * *ciphertext*, so the relay can store and dedupe without understanding + * anything. Re-uploading an unchanged file is free, which matters when a + * workspace of several hundred files changes three lines at a time. + * + * 2. **A manifest names the collection.** Paths live in the manifest, not in + * object names — the relay must not learn that a user has a file called + * `Projects/Divorce/plan.md`. The manifest is itself encrypted and stored + * as an object. + * + * 3. **The log is append-only.** Each commit adds an entry pointing at a + * manifest. History, rollback and "what changed last Tuesday" all fall out + * of that, which is the versioning half of the requirement. + */ + +/** One file in a manifest. `hash` names the ciphertext object. */ +export interface VaultEntry { + /** Path within the collection, as the app understands it. */ + path: string; + /** Lowercase hex SHA-256 of the ciphertext object. */ + hash: string; + /** Plaintext byte length, for progress reporting and sanity checks. */ + size: number; + /** Optional app-defined mode/metadata, opaque to the vault. */ + mode?: string; +} + +/** The full contents of a collection at one point in time. */ +export interface VaultManifest { + /** Schema version — a client that does not know it must refuse, not guess. */ + version: 1; + /** Monotonic, assigned by the client and checked by the relay. */ + seq: number; + /** Epoch millis when the commit was made. */ + at: number; + /** Optional human note, e.g. "nattlig backup". */ + message?: string; + entries: VaultEntry[]; +} + +/** One row of the append-only log. */ +export interface VaultLogEntry { + seq: number; + /** Hash of the encrypted manifest object. */ + manifest: string; + at: number; + /** Total bytes of all objects the manifest references, for quota display. */ + bytes: number; +} + +/** What `GET /v1/vault/:id/log` returns. */ +export interface VaultLog { + entries: VaultLogEntry[]; + /** Highest seq the relay holds; `0` when the vault is empty. */ + head: number; +} + +/** + * Error codes clients branch on. + * + * `SEQ_CONFLICT` is the important one: two devices committed from the same + * head, and the loser must re-read and re-commit rather than retry blindly. + */ +export type VaultErrorCode = + | 'BAD_REQUEST' + | 'UNAUTHORIZED' + | 'NOT_FOUND' + | 'SEQ_CONFLICT' + | 'TOO_LARGE' + | 'MISSING_OBJECTS'; diff --git a/packages/shade-vault/tests/scaffoldd-e2e.test.ts b/packages/shade-vault/tests/scaffoldd-e2e.test.ts new file mode 100644 index 0000000..c452659 --- /dev/null +++ b/packages/shade-vault/tests/scaffoldd-e2e.test.ts @@ -0,0 +1,153 @@ +/** + * The whole backup chain, against real data. + * + * Talks to a running `scaffoldd` over its unix socket, asks what the backup + * set is, reads those files off disk, pushes them through the vault, and pulls + * them back — then compares byte-for-byte. + * + * This is the test that would catch the things unit tests cannot: a path that + * survives the round trip wrong, a 600 KB log that trips a size ceiling, a + * workspace whose file count makes the manifest too large to commit. It is + * skipped when no daemon is listening, so it never blocks an ordinary run. + * + * scaffoldd --socket /tmp/scaffoldd-e2e.sock & + * SCAFFOLDD_SOCKET=/tmp/scaffoldd-e2e.sock bun test scaffoldd-e2e + */ +import { describe, test, expect } from 'bun:test'; +import { connect } from 'node:net'; +import { readFile } from 'node:fs/promises'; +import { SubtleCryptoProvider } from '@shade/crypto-web'; +import { createVaultRoutes } from '../src/server.js'; +import { MemoryVaultStore } from '../src/store.js'; +import { HttpVaultTransport } from '../src/http-transport.js'; +import { VaultClient, deriveVaultKeys } from '../src/client.js'; +import type { VaultFile } from '../src/client.js'; + +const SOCKET = process.env.SCAFFOLDD_SOCKET; +const crypto = new SubtleCryptoProvider(); + +interface BackupFile { + path: string; + source: string; + size: number; +} + +function ask(socketPath: string, request: object): Promise { + return new Promise((resolve, reject) => { + const sock = connect(socketPath); + let buf = ''; + const timer = setTimeout(() => { + sock.destroy(); + reject(new Error('scaffoldd svarte ikke innen 20 s')); + }, 20_000); + // Deliberately no `end()` after writing: Bun's socket closes both + // directions, so the daemon's reply is discarded before it arrives. The + // protocol is line-delimited, so read until the first complete line and + // close from here instead. + sock.on('connect', () => { + sock.write(`${JSON.stringify(request)}\n`); + }); + sock.on('data', (c) => { + buf += c.toString(); + const nl = buf.indexOf('\n'); + if (nl === -1) return; + clearTimeout(timer); + sock.destroy(); + const res = JSON.parse(buf.slice(0, nl)); + res.ok ? resolve(res.result) : reject(new Error(`${res.code}: ${res.error}`)); + }); + sock.on('end', () => { + clearTimeout(timer); + if (!buf.includes('\n')) reject(new Error('tomt svar')); + }); + sock.on('error', (e) => { + clearTimeout(timer); + reject(e); + }); + }); +} + +async function harness() { + const store = new MemoryVaultStore(); + const routes = createVaultRoutes(store, crypto); + const transport = new HttpVaultTransport('http://vault.test', (i, init) => + routes.fetch(new Request(i, init)), + ); + const keys = await deriveVaultKeys(new Uint8Array(32).fill(42), 'scaffold-e2e'); + return { store, client: new VaultClient(crypto, keys, transport) }; +} + +describe.skipIf(!SOCKET)('scaffoldd → vault, real workspace', () => { + test('the whole backup set round-trips byte-for-byte', async () => { + const set = (await ask(SOCKET!, { op: 'backupSet' })) as { + files: BackupFile[]; + count: number; + bytes: number; + }; + expect(set.count).toBeGreaterThan(50); + + const files: VaultFile[] = []; + for (const f of set.files) { + try { + files.push({ path: f.path, bytes: new Uint8Array(await readFile(f.source)) }); + } catch { + // Matches the bridge: one unreadable file is skipped, not fatal. + } + } + + const { client } = await harness(); + const push = await client.push(files, 1_786_600_000_000, 'e2e'); + expect(push.seq).toBe(1); + expect(push.uploaded).toBe(files.length); + + const { files: back } = await client.pull(); + expect(back.size).toBe(files.length); + + // Every single file, not a sample: a backup that restores 99% of a + // workspace is not a backup. + for (const f of files) { + const restored = back.get(f.path); + expect(restored).toBeDefined(); + expect(restored!.length).toBe(f.bytes.length); + expect(Buffer.from(restored!).equals(Buffer.from(f.bytes))).toBe(true); + } + }, 120_000); + + test('a second push after one edit sends almost nothing', async () => { + // The property that decides whether hourly backup is affordable. + const set = (await ask(SOCKET!, { op: 'backupSet' })) as { files: BackupFile[] }; + const files: VaultFile[] = []; + for (const f of set.files.slice(0, 120)) { + try { + files.push({ path: f.path, bytes: new Uint8Array(await readFile(f.source)) }); + } catch { + /* skipped */ + } + } + + const { client } = await harness(); + const first = await client.push(files, 1_786_600_000_000); + + const edited = files.map((f, i) => + i === 0 ? { path: f.path, bytes: new TextEncoder().encode('endret én linje') } : f, + ); + const second = await client.push(edited, 1_786_600_001_000); + + expect(first.uploaded).toBe(files.length); + expect(second.uploaded).toBe(1); + expect(second.reused).toBe(files.length - 1); + }, 120_000); + + test('the largest file in the workspace survives the round trip', async () => { + // Terra's log.md is ~617 KB. Nothing in the chain may quietly cap it. + const set = (await ask(SOCKET!, { op: 'backupSet' })) as { files: BackupFile[] }; + const biggest = set.files.reduce((a, b) => (a.size > b.size ? a : b)); + expect(biggest.size).toBeGreaterThan(100_000); + + const bytes = new Uint8Array(await readFile(biggest.source)); + const { client } = await harness(); + await client.push([{ path: biggest.path, bytes }], 1_786_600_000_000); + const { files } = await client.pull(); + expect(files.get(biggest.path)!.length).toBe(bytes.length); + }, 120_000); +}); diff --git a/packages/shade-vault/tests/vault.test.ts b/packages/shade-vault/tests/vault.test.ts new file mode 100644 index 0000000..c0836ae --- /dev/null +++ b/packages/shade-vault/tests/vault.test.ts @@ -0,0 +1,313 @@ +/** + * End-to-end tests for the vault. + * + * The client talks to the REAL route handlers through Hono's `fetch`, not to a + * mock. A mock transport would agree with the client by construction and prove + * nothing about the wire contract — which is exactly the surface a phone and a + * daemon have to share. + */ +import { describe, test, expect } from 'bun:test'; +import { SubtleCryptoProvider } from '@shade/crypto-web'; +import { createVaultRoutes } from '../src/server.js'; +import { MemoryVaultStore } from '../src/store.js'; +import { HttpVaultTransport } from '../src/http-transport.js'; +import { VaultClient, deriveVaultKeys } from '../src/client.js'; +import { objectHash } from '../src/crypto.js'; + +const crypto = new SubtleCryptoProvider(); +const AT = 1_786_600_000_000; + +function enc(s: string): Uint8Array { + return new TextEncoder().encode(s); +} +function dec(b: Uint8Array): string { + return new TextDecoder().decode(b); +} + +/** A client wired to a fresh in-memory relay. */ +async function harness(masterKey = new Uint8Array(32).fill(7), app = 'scaffold') { + const store = new MemoryVaultStore(); + const routes = createVaultRoutes(store, crypto); + const transport = new HttpVaultTransport('http://vault.test', (input, init) => + routes.fetch(new Request(input, init)), + ); + const keys = await deriveVaultKeys(masterKey, app); + return { store, keys, client: new VaultClient(crypto, keys, transport) }; +} + +describe('round trip', () => { + test('files pushed can be pulled back byte-for-byte', async () => { + const { client } = await harness(); + const files = [ + { path: '.scaffold/plan.md', bytes: enc('# Plan\n\n## Nå\n- [ ] noe\n') }, + { path: '.scaffold/tasks.yaml', bytes: enc('tasks: []\n') }, + ]; + + const res = await client.push(files, AT, 'første backup'); + expect(res.seq).toBe(1); + expect(res.uploaded).toBe(2); + + const { manifest, files: back } = await client.pull(); + expect(manifest.seq).toBe(1); + expect(manifest.message).toBe('første backup'); + expect(dec(back.get('.scaffold/plan.md')!)).toBe('# Plan\n\n## Nå\n- [ ] noe\n'); + expect(dec(back.get('.scaffold/tasks.yaml')!)).toBe('tasks: []\n'); + }); + + test('a fresh device with only the credentials can restore', async () => { + // The recovery story: same master key, nothing else carried over. + const master = new Uint8Array(32).fill(11); + const { client, store } = await harness(master); + await client.push([{ path: 'notes.yaml', bytes: enc('notes: [en, to]\n') }], AT); + + const routes = createVaultRoutes(store, crypto); + const transport = new HttpVaultTransport('http://vault.test', (i, init) => + routes.fetch(new Request(i, init)), + ); + const keys = await deriveVaultKeys(master, 'scaffold'); + const fresh = new VaultClient(crypto, keys, transport); + + const { files } = await fresh.pull(); + expect(dec(files.get('notes.yaml')!)).toBe('notes: [en, to]\n'); + }); + + test('a different master key cannot read the vault', async () => { + const { store } = await harness(new Uint8Array(32).fill(1)); + const routes = createVaultRoutes(store, crypto); + const transport = new HttpVaultTransport('http://vault.test', (i, init) => + routes.fetch(new Request(i, init)), + ); + const wrong = await deriveVaultKeys(new Uint8Array(32).fill(2), 'scaffold'); + const intruder = new VaultClient(crypto, wrong, transport); + // A different master derives a different vaultId, so there is nothing + // there to read in the first place — the id is itself a secret. + await expect(intruder.pull()).rejects.toThrow(); + }); +}); + +describe('versioning', () => { + test('each push is a new version and old ones stay readable', async () => { + const { client } = await harness(); + await client.push([{ path: 'plan.md', bytes: enc('versjon 1') }], AT); + await client.push([{ path: 'plan.md', bytes: enc('versjon 2') }], AT + 1000); + await client.push([{ path: 'plan.md', bytes: enc('versjon 3') }], AT + 2000); + + const log = await client.history(); + expect(log.head).toBe(3); + expect(log.entries.map((e) => e.seq)).toEqual([1, 2, 3]); + + // Rollback: the whole point of keeping the log. + expect(dec((await client.pull(1)).files.get('plan.md')!)).toBe('versjon 1'); + expect(dec((await client.pull(2)).files.get('plan.md')!)).toBe('versjon 2'); + expect(dec((await client.pull()).files.get('plan.md')!)).toBe('versjon 3'); + }); + + test('unchanged files are not re-uploaded', async () => { + // The property that makes backing up a 9 MB workspace on every change + // affordable: only what actually moved goes over the wire. + const { client } = await harness(); + const stable = { path: 'stor-logg.md', bytes: enc('x'.repeat(50_000)) }; + + const first = await client.push([stable, { path: 'plan.md', bytes: enc('en') }], AT); + expect(first.uploaded).toBe(2); + + const second = await client.push([stable, { path: 'plan.md', bytes: enc('to') }], AT + 1); + expect(second.uploaded).toBe(1); + expect(second.reused).toBe(1); + expect(second.bytesUploaded).toBeLessThan(1000); + + // And the reused file is still intact in the new version. + const { files } = await client.pull(); + expect(files.get('stor-logg.md')!.length).toBe(50_000); + }); + + test('a deleted file is absent from the new version but present in the old', async () => { + const { client } = await harness(); + await client.push( + [ + { path: 'a.md', bytes: enc('A') }, + { path: 'b.md', bytes: enc('B') }, + ], + AT, + ); + await client.push([{ path: 'a.md', bytes: enc('A') }], AT + 1); + + expect((await client.pull()).files.has('b.md')).toBe(false); + expect(dec((await client.pull(1)).files.get('b.md')!)).toBe('B'); + }); +}); + +describe('the relay is blind', () => { + test('stored objects contain no plaintext', async () => { + const { client, store } = await harness(); + await client.push([{ path: 'hemmelig/plan.md', bytes: enc('SENSITIVT INNHOLD') }], AT); + + const log = await store.log(client.vaultId); + const manifestBytes = await store.getObject(client.vaultId, log[0]!.manifest); + const asText = dec(manifestBytes!); + // Neither the contents nor the path leaks: paths live inside the + // encrypted manifest, not in object names. + expect(asText).not.toContain('SENSITIVT'); + expect(asText).not.toContain('hemmelig'); + }); + + test('object names are the hash of the ciphertext, so the relay can verify', async () => { + const { client, store } = await harness(); + await client.push([{ path: 'x', bytes: enc('hei') }], AT); + const log = await store.log(client.vaultId); + const bytes = await store.getObject(client.vaultId, log[0]!.manifest); + expect(objectHash(bytes!)).toBe(log[0]!.manifest); + }); + + test('an object whose bytes do not match its name is rejected', async () => { + const store = new MemoryVaultStore(); + const routes = createVaultRoutes(store, crypto); + const keys = await deriveVaultKeys(new Uint8Array(32).fill(3), 'scaffold'); + const { signPayload } = await import('@shade/server'); + const { toBase64 } = await import('@shade/core'); + + const body = await signPayload(crypto, keys.signingSeed, { + data: toBase64(enc('juks')), + publicKey: toBase64(keys.publicKey), + }); + const res = await routes.fetch( + new Request(`http://v/v1/vault/${keys.vaultId}/object/${'0'.repeat(64)}`, { + method: 'PUT', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + }), + ); + expect(res.status).toBe(400); + expect((await res.json()).error.code).toBe('BAD_REQUEST'); + }); +}); + +describe('concurrent writers', () => { + test('a commit from a stale head is refused', async () => { + // Two devices push from the same version. The second must not be able to + // overwrite a sequence number that is already history. + const { client, keys, store } = await harness(); + await client.push([{ path: 'plan.md', bytes: enc('en')}], AT); + + const routes = createVaultRoutes(store, crypto); + const { signPayload } = await import('@shade/server'); + const { toBase64 } = await import('@shade/core'); + const log = await store.log(keys.vaultId); + const body = await signPayload(crypto, keys.signingSeed, { + manifest: log[0]!.manifest, + seq: 1, // already taken + at: AT, + hashes: [], + publicKey: toBase64(keys.publicKey), + }); + + const res = await routes.fetch( + new Request(`http://v/v1/vault/${keys.vaultId}/commit`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + }), + ); + expect(res.status).toBe(409); + const err = await res.json(); + expect(err.error.code).toBe('SEQ_CONFLICT'); + expect(err.head).toBe(1); + }); + + test('a commit referencing a missing object is refused', async () => { + // Otherwise the log would publish a version that cannot be restored. + const { client, keys, store } = await harness(); + await client.push([{ path: 'plan.md', bytes: enc('en') }], AT); + + const routes = createVaultRoutes(store, crypto); + const { signPayload } = await import('@shade/server'); + const { toBase64 } = await import('@shade/core'); + const log = await store.log(keys.vaultId); + const body = await signPayload(crypto, keys.signingSeed, { + manifest: log[0]!.manifest, + seq: 2, + at: AT, + hashes: ['a'.repeat(64)], + publicKey: toBase64(keys.publicKey), + }); + + const res = await routes.fetch( + new Request(`http://v/v1/vault/${keys.vaultId}/commit`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + }), + ); + expect(res.status).toBe(409); + expect((await res.json()).error.code).toBe('MISSING_OBJECTS'); + }); +}); + +describe('authorisation', () => { + test('a second key cannot write to a vault another key pinned', async () => { + const { client, keys, store } = await harness(); + await client.push([{ path: 'plan.md', bytes: enc('mitt') }], AT); + + const routes = createVaultRoutes(store, crypto); + const { signPayload } = await import('@shade/server'); + const { toBase64 } = await import('@shade/core'); + const attacker = await deriveVaultKeys(new Uint8Array(32).fill(9), 'scaffold'); + + // Signed correctly — but by the wrong key, and asserting its own pubkey. + const body = await signPayload(crypto, attacker.signingSeed, { + data: toBase64(enc('tull')), + publicKey: toBase64(attacker.publicKey), + }); + const res = await routes.fetch( + new Request( + `http://v/v1/vault/${keys.vaultId}/object/${objectHash(enc('tull'))}`, + { + method: 'PUT', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + }, + ), + ); + expect(res.status).toBe(401); + }); + + test('an unsigned write is refused', async () => { + const store = new MemoryVaultStore(); + const routes = createVaultRoutes(store, crypto); + const res = await routes.fetch( + new Request(`http://v/v1/vault/${'a'.repeat(64)}/object/${'b'.repeat(64)}`, { + method: 'PUT', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ data: 'aGk=' }), + }), + ); + expect(res.status).toBe(401); + }); +}); + +describe('key derivation', () => { + test('the same credentials derive the same vault, different ones do not', async () => { + const a = await deriveVaultKeys(new Uint8Array(32).fill(4), 'scaffold'); + const b = await deriveVaultKeys(new Uint8Array(32).fill(4), 'scaffold'); + const c = await deriveVaultKeys(new Uint8Array(32).fill(5), 'scaffold'); + expect(a.vaultId).toBe(b.vaultId); + expect(a.vaultId).not.toBe(c.vaultId); + }); + + test('two apps under one master do not share a vault', async () => { + const master = new Uint8Array(32).fill(6); + const scaffold = await deriveVaultKeys(master, 'scaffold'); + const mail = await deriveVaultKeys(master, 'mail'); + expect(scaffold.vaultId).not.toBe(mail.vaultId); + expect(scaffold.contentKey).not.toEqual(mail.contentKey); + }); + + test('the vault branch is separate from the profile-blob branch', async () => { + // A vault key reads every file; a profile-blob key reads a host list. + // Sharing a derivation would make one compromise into the other. + const master = new Uint8Array(32).fill(8); + const { deriveBlobKey } = await import('@shade/storage-encrypted'); + const vault = await deriveVaultKeys(master, 'prism'); + expect(vault.contentKey).not.toEqual(deriveBlobKey(master, 'prism')); + }); +}); diff --git a/packages/shade-vault/tsconfig.json b/packages/shade-vault/tsconfig.json new file mode 100644 index 0000000..a181690 --- /dev/null +++ b/packages/shade-vault/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { "outDir": "dist", "rootDir": "src", "lib": ["ES2022", "DOM"] }, + "include": ["src"] +}