feat(vault): server-side kryptert fillager (V4.13)

Shade kunne flytte filer mellom peers (@shade/files) og lagre én liten
profil-blob per konto, men hadde ingen alltid-på lagring av krypterte filer.
Uten den kan ingen Shade-app tilby backup, og ingen klient lese data mens
peeren som eier dem er avslått.

Objekter er innholdsadresserte på hashen av CHIFFERTEKSTEN, så relayen kan
lagre, deduplisere og verifisere uten nøkkel — den regner om hashen ved
opplasting og avviser feilnavngitte objekter. Stier bor inne i det krypterte
manifestet, aldri i objektnavn: relayen skal ikke lære hva filene heter.
Loggen er append-only, så historikk og rollback følger av modellen.

SqliteVaultStore har med vilje INGEN minne-fallback, i motsetning til
blob-storen. Den fallbacken slettet Prisms profil ved en rutine-redeploy
2026-08-12 fordi den fungerte helt til containeren ble recreated, uten en
eneste feilmelding. En backup som glemmer er verre enn ingen backup, så uten
SHADE_VAULT_DB_PATH mountes rutene ikke — med en logglinje som sier hvorfor.

Én feil fanget av testene: pubkeyen ble først lagt på UTENFOR signaturen,
som både brøt verifyPayload og ville latt hvem som helst bytte identitet i
transit på den TOFU-pinnende førsteskrivingen.

16 vault- + 7 store-tester, alle mot de ekte rutehåndtererne gjennom Honos
fetch. Kjeden er dessuten kjørt mot en ekte HTTP-server med et ekte
workspace: 491 filer / 7,7 MB, alle bit-identiske etter gjenoppretting,
og andre push etter én endring sendte 0 KB.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-08-14 11:58:37 +02:00
parent 012d7f5289
commit 84d3166ca1
18 changed files with 1775 additions and 1 deletions

View File

@@ -0,0 +1,153 @@
/**
* The whole backup chain, against real data.
*
* Talks to a running `scaffoldd` over its unix socket, asks what the backup
* set is, reads those files off disk, pushes them through the vault, and pulls
* them back — then compares byte-for-byte.
*
* This is the test that would catch the things unit tests cannot: a path that
* survives the round trip wrong, a 600 KB log that trips a size ceiling, a
* workspace whose file count makes the manifest too large to commit. It is
* skipped when no daemon is listening, so it never blocks an ordinary run.
*
* scaffoldd --socket /tmp/scaffoldd-e2e.sock &
* SCAFFOLDD_SOCKET=/tmp/scaffoldd-e2e.sock bun test scaffoldd-e2e
*/
import { describe, test, expect } from 'bun:test';
import { connect } from 'node:net';
import { readFile } from 'node:fs/promises';
import { SubtleCryptoProvider } from '@shade/crypto-web';
import { createVaultRoutes } from '../src/server.js';
import { MemoryVaultStore } from '../src/store.js';
import { HttpVaultTransport } from '../src/http-transport.js';
import { VaultClient, deriveVaultKeys } from '../src/client.js';
import type { VaultFile } from '../src/client.js';
const SOCKET = process.env.SCAFFOLDD_SOCKET;
const crypto = new SubtleCryptoProvider();
interface BackupFile {
path: string;
source: string;
size: number;
}
function ask(socketPath: string, request: object): Promise<any> {
return new Promise((resolve, reject) => {
const sock = connect(socketPath);
let buf = '';
const timer = setTimeout(() => {
sock.destroy();
reject(new Error('scaffoldd svarte ikke innen 20 s'));
}, 20_000);
// Deliberately no `end()` after writing: Bun's socket closes both
// directions, so the daemon's reply is discarded before it arrives. The
// protocol is line-delimited, so read until the first complete line and
// close from here instead.
sock.on('connect', () => {
sock.write(`${JSON.stringify(request)}\n`);
});
sock.on('data', (c) => {
buf += c.toString();
const nl = buf.indexOf('\n');
if (nl === -1) return;
clearTimeout(timer);
sock.destroy();
const res = JSON.parse(buf.slice(0, nl));
res.ok ? resolve(res.result) : reject(new Error(`${res.code}: ${res.error}`));
});
sock.on('end', () => {
clearTimeout(timer);
if (!buf.includes('\n')) reject(new Error('tomt svar'));
});
sock.on('error', (e) => {
clearTimeout(timer);
reject(e);
});
});
}
async function harness() {
const store = new MemoryVaultStore();
const routes = createVaultRoutes(store, crypto);
const transport = new HttpVaultTransport('http://vault.test', (i, init) =>
routes.fetch(new Request(i, init)),
);
const keys = await deriveVaultKeys(new Uint8Array(32).fill(42), 'scaffold-e2e');
return { store, client: new VaultClient(crypto, keys, transport) };
}
describe.skipIf(!SOCKET)('scaffoldd → vault, real workspace', () => {
test('the whole backup set round-trips byte-for-byte', async () => {
const set = (await ask(SOCKET!, { op: 'backupSet' })) as {
files: BackupFile[];
count: number;
bytes: number;
};
expect(set.count).toBeGreaterThan(50);
const files: VaultFile[] = [];
for (const f of set.files) {
try {
files.push({ path: f.path, bytes: new Uint8Array(await readFile(f.source)) });
} catch {
// Matches the bridge: one unreadable file is skipped, not fatal.
}
}
const { client } = await harness();
const push = await client.push(files, 1_786_600_000_000, 'e2e');
expect(push.seq).toBe(1);
expect(push.uploaded).toBe(files.length);
const { files: back } = await client.pull();
expect(back.size).toBe(files.length);
// Every single file, not a sample: a backup that restores 99% of a
// workspace is not a backup.
for (const f of files) {
const restored = back.get(f.path);
expect(restored).toBeDefined();
expect(restored!.length).toBe(f.bytes.length);
expect(Buffer.from(restored!).equals(Buffer.from(f.bytes))).toBe(true);
}
}, 120_000);
test('a second push after one edit sends almost nothing', async () => {
// The property that decides whether hourly backup is affordable.
const set = (await ask(SOCKET!, { op: 'backupSet' })) as { files: BackupFile[] };
const files: VaultFile[] = [];
for (const f of set.files.slice(0, 120)) {
try {
files.push({ path: f.path, bytes: new Uint8Array(await readFile(f.source)) });
} catch {
/* skipped */
}
}
const { client } = await harness();
const first = await client.push(files, 1_786_600_000_000);
const edited = files.map((f, i) =>
i === 0 ? { path: f.path, bytes: new TextEncoder().encode('endret én linje') } : f,
);
const second = await client.push(edited, 1_786_600_001_000);
expect(first.uploaded).toBe(files.length);
expect(second.uploaded).toBe(1);
expect(second.reused).toBe(files.length - 1);
}, 120_000);
test('the largest file in the workspace survives the round trip', async () => {
// Terra's log.md is ~617 KB. Nothing in the chain may quietly cap it.
const set = (await ask(SOCKET!, { op: 'backupSet' })) as { files: BackupFile[] };
const biggest = set.files.reduce((a, b) => (a.size > b.size ? a : b));
expect(biggest.size).toBeGreaterThan(100_000);
const bytes = new Uint8Array(await readFile(biggest.source));
const { client } = await harness();
await client.push([{ path: biggest.path, bytes }], 1_786_600_000_000);
const { files } = await client.pull();
expect(files.get(biggest.path)!.length).toBe(bytes.length);
}, 120_000);
});

View File

@@ -0,0 +1,313 @@
/**
* End-to-end tests for the vault.
*
* The client talks to the REAL route handlers through Hono's `fetch`, not to a
* mock. A mock transport would agree with the client by construction and prove
* nothing about the wire contract — which is exactly the surface a phone and a
* daemon have to share.
*/
import { describe, test, expect } from 'bun:test';
import { SubtleCryptoProvider } from '@shade/crypto-web';
import { createVaultRoutes } from '../src/server.js';
import { MemoryVaultStore } from '../src/store.js';
import { HttpVaultTransport } from '../src/http-transport.js';
import { VaultClient, deriveVaultKeys } from '../src/client.js';
import { objectHash } from '../src/crypto.js';
const crypto = new SubtleCryptoProvider();
const AT = 1_786_600_000_000;
function enc(s: string): Uint8Array {
return new TextEncoder().encode(s);
}
function dec(b: Uint8Array): string {
return new TextDecoder().decode(b);
}
/** A client wired to a fresh in-memory relay. */
async function harness(masterKey = new Uint8Array(32).fill(7), app = 'scaffold') {
const store = new MemoryVaultStore();
const routes = createVaultRoutes(store, crypto);
const transport = new HttpVaultTransport('http://vault.test', (input, init) =>
routes.fetch(new Request(input, init)),
);
const keys = await deriveVaultKeys(masterKey, app);
return { store, keys, client: new VaultClient(crypto, keys, transport) };
}
describe('round trip', () => {
test('files pushed can be pulled back byte-for-byte', async () => {
const { client } = await harness();
const files = [
{ path: '.scaffold/plan.md', bytes: enc('# Plan\n\n## Nå\n- [ ] noe\n') },
{ path: '.scaffold/tasks.yaml', bytes: enc('tasks: []\n') },
];
const res = await client.push(files, AT, 'første backup');
expect(res.seq).toBe(1);
expect(res.uploaded).toBe(2);
const { manifest, files: back } = await client.pull();
expect(manifest.seq).toBe(1);
expect(manifest.message).toBe('første backup');
expect(dec(back.get('.scaffold/plan.md')!)).toBe('# Plan\n\n## Nå\n- [ ] noe\n');
expect(dec(back.get('.scaffold/tasks.yaml')!)).toBe('tasks: []\n');
});
test('a fresh device with only the credentials can restore', async () => {
// The recovery story: same master key, nothing else carried over.
const master = new Uint8Array(32).fill(11);
const { client, store } = await harness(master);
await client.push([{ path: 'notes.yaml', bytes: enc('notes: [en, to]\n') }], AT);
const routes = createVaultRoutes(store, crypto);
const transport = new HttpVaultTransport('http://vault.test', (i, init) =>
routes.fetch(new Request(i, init)),
);
const keys = await deriveVaultKeys(master, 'scaffold');
const fresh = new VaultClient(crypto, keys, transport);
const { files } = await fresh.pull();
expect(dec(files.get('notes.yaml')!)).toBe('notes: [en, to]\n');
});
test('a different master key cannot read the vault', async () => {
const { store } = await harness(new Uint8Array(32).fill(1));
const routes = createVaultRoutes(store, crypto);
const transport = new HttpVaultTransport('http://vault.test', (i, init) =>
routes.fetch(new Request(i, init)),
);
const wrong = await deriveVaultKeys(new Uint8Array(32).fill(2), 'scaffold');
const intruder = new VaultClient(crypto, wrong, transport);
// A different master derives a different vaultId, so there is nothing
// there to read in the first place — the id is itself a secret.
await expect(intruder.pull()).rejects.toThrow();
});
});
describe('versioning', () => {
test('each push is a new version and old ones stay readable', async () => {
const { client } = await harness();
await client.push([{ path: 'plan.md', bytes: enc('versjon 1') }], AT);
await client.push([{ path: 'plan.md', bytes: enc('versjon 2') }], AT + 1000);
await client.push([{ path: 'plan.md', bytes: enc('versjon 3') }], AT + 2000);
const log = await client.history();
expect(log.head).toBe(3);
expect(log.entries.map((e) => e.seq)).toEqual([1, 2, 3]);
// Rollback: the whole point of keeping the log.
expect(dec((await client.pull(1)).files.get('plan.md')!)).toBe('versjon 1');
expect(dec((await client.pull(2)).files.get('plan.md')!)).toBe('versjon 2');
expect(dec((await client.pull()).files.get('plan.md')!)).toBe('versjon 3');
});
test('unchanged files are not re-uploaded', async () => {
// The property that makes backing up a 9 MB workspace on every change
// affordable: only what actually moved goes over the wire.
const { client } = await harness();
const stable = { path: 'stor-logg.md', bytes: enc('x'.repeat(50_000)) };
const first = await client.push([stable, { path: 'plan.md', bytes: enc('en') }], AT);
expect(first.uploaded).toBe(2);
const second = await client.push([stable, { path: 'plan.md', bytes: enc('to') }], AT + 1);
expect(second.uploaded).toBe(1);
expect(second.reused).toBe(1);
expect(second.bytesUploaded).toBeLessThan(1000);
// And the reused file is still intact in the new version.
const { files } = await client.pull();
expect(files.get('stor-logg.md')!.length).toBe(50_000);
});
test('a deleted file is absent from the new version but present in the old', async () => {
const { client } = await harness();
await client.push(
[
{ path: 'a.md', bytes: enc('A') },
{ path: 'b.md', bytes: enc('B') },
],
AT,
);
await client.push([{ path: 'a.md', bytes: enc('A') }], AT + 1);
expect((await client.pull()).files.has('b.md')).toBe(false);
expect(dec((await client.pull(1)).files.get('b.md')!)).toBe('B');
});
});
describe('the relay is blind', () => {
test('stored objects contain no plaintext', async () => {
const { client, store } = await harness();
await client.push([{ path: 'hemmelig/plan.md', bytes: enc('SENSITIVT INNHOLD') }], AT);
const log = await store.log(client.vaultId);
const manifestBytes = await store.getObject(client.vaultId, log[0]!.manifest);
const asText = dec(manifestBytes!);
// Neither the contents nor the path leaks: paths live inside the
// encrypted manifest, not in object names.
expect(asText).not.toContain('SENSITIVT');
expect(asText).not.toContain('hemmelig');
});
test('object names are the hash of the ciphertext, so the relay can verify', async () => {
const { client, store } = await harness();
await client.push([{ path: 'x', bytes: enc('hei') }], AT);
const log = await store.log(client.vaultId);
const bytes = await store.getObject(client.vaultId, log[0]!.manifest);
expect(objectHash(bytes!)).toBe(log[0]!.manifest);
});
test('an object whose bytes do not match its name is rejected', async () => {
const store = new MemoryVaultStore();
const routes = createVaultRoutes(store, crypto);
const keys = await deriveVaultKeys(new Uint8Array(32).fill(3), 'scaffold');
const { signPayload } = await import('@shade/server');
const { toBase64 } = await import('@shade/core');
const body = await signPayload(crypto, keys.signingSeed, {
data: toBase64(enc('juks')),
publicKey: toBase64(keys.publicKey),
});
const res = await routes.fetch(
new Request(`http://v/v1/vault/${keys.vaultId}/object/${'0'.repeat(64)}`, {
method: 'PUT',
headers: { 'content-type': 'application/json' },
body: JSON.stringify(body),
}),
);
expect(res.status).toBe(400);
expect((await res.json()).error.code).toBe('BAD_REQUEST');
});
});
describe('concurrent writers', () => {
test('a commit from a stale head is refused', async () => {
// Two devices push from the same version. The second must not be able to
// overwrite a sequence number that is already history.
const { client, keys, store } = await harness();
await client.push([{ path: 'plan.md', bytes: enc('en')}], AT);
const routes = createVaultRoutes(store, crypto);
const { signPayload } = await import('@shade/server');
const { toBase64 } = await import('@shade/core');
const log = await store.log(keys.vaultId);
const body = await signPayload(crypto, keys.signingSeed, {
manifest: log[0]!.manifest,
seq: 1, // already taken
at: AT,
hashes: [],
publicKey: toBase64(keys.publicKey),
});
const res = await routes.fetch(
new Request(`http://v/v1/vault/${keys.vaultId}/commit`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify(body),
}),
);
expect(res.status).toBe(409);
const err = await res.json();
expect(err.error.code).toBe('SEQ_CONFLICT');
expect(err.head).toBe(1);
});
test('a commit referencing a missing object is refused', async () => {
// Otherwise the log would publish a version that cannot be restored.
const { client, keys, store } = await harness();
await client.push([{ path: 'plan.md', bytes: enc('en') }], AT);
const routes = createVaultRoutes(store, crypto);
const { signPayload } = await import('@shade/server');
const { toBase64 } = await import('@shade/core');
const log = await store.log(keys.vaultId);
const body = await signPayload(crypto, keys.signingSeed, {
manifest: log[0]!.manifest,
seq: 2,
at: AT,
hashes: ['a'.repeat(64)],
publicKey: toBase64(keys.publicKey),
});
const res = await routes.fetch(
new Request(`http://v/v1/vault/${keys.vaultId}/commit`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify(body),
}),
);
expect(res.status).toBe(409);
expect((await res.json()).error.code).toBe('MISSING_OBJECTS');
});
});
describe('authorisation', () => {
test('a second key cannot write to a vault another key pinned', async () => {
const { client, keys, store } = await harness();
await client.push([{ path: 'plan.md', bytes: enc('mitt') }], AT);
const routes = createVaultRoutes(store, crypto);
const { signPayload } = await import('@shade/server');
const { toBase64 } = await import('@shade/core');
const attacker = await deriveVaultKeys(new Uint8Array(32).fill(9), 'scaffold');
// Signed correctly — but by the wrong key, and asserting its own pubkey.
const body = await signPayload(crypto, attacker.signingSeed, {
data: toBase64(enc('tull')),
publicKey: toBase64(attacker.publicKey),
});
const res = await routes.fetch(
new Request(
`http://v/v1/vault/${keys.vaultId}/object/${objectHash(enc('tull'))}`,
{
method: 'PUT',
headers: { 'content-type': 'application/json' },
body: JSON.stringify(body),
},
),
);
expect(res.status).toBe(401);
});
test('an unsigned write is refused', async () => {
const store = new MemoryVaultStore();
const routes = createVaultRoutes(store, crypto);
const res = await routes.fetch(
new Request(`http://v/v1/vault/${'a'.repeat(64)}/object/${'b'.repeat(64)}`, {
method: 'PUT',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ data: 'aGk=' }),
}),
);
expect(res.status).toBe(401);
});
});
describe('key derivation', () => {
test('the same credentials derive the same vault, different ones do not', async () => {
const a = await deriveVaultKeys(new Uint8Array(32).fill(4), 'scaffold');
const b = await deriveVaultKeys(new Uint8Array(32).fill(4), 'scaffold');
const c = await deriveVaultKeys(new Uint8Array(32).fill(5), 'scaffold');
expect(a.vaultId).toBe(b.vaultId);
expect(a.vaultId).not.toBe(c.vaultId);
});
test('two apps under one master do not share a vault', async () => {
const master = new Uint8Array(32).fill(6);
const scaffold = await deriveVaultKeys(master, 'scaffold');
const mail = await deriveVaultKeys(master, 'mail');
expect(scaffold.vaultId).not.toBe(mail.vaultId);
expect(scaffold.contentKey).not.toEqual(mail.contentKey);
});
test('the vault branch is separate from the profile-blob branch', async () => {
// A vault key reads every file; a profile-blob key reads a host list.
// Sharing a derivation would make one compromise into the other.
const master = new Uint8Array(32).fill(8);
const { deriveBlobKey } = await import('@shade/storage-encrypted');
const vault = await deriveVaultKeys(master, 'prism');
expect(vault.contentKey).not.toEqual(deriveBlobKey(master, 'prism'));
});
});