From 53ce776c0286c40fad18f4109a6cd97b40ad776f Mon Sep 17 00:00:00 2001 From: Sterister Date: Tue, 8 Sep 2026 16:33:40 +0200 Subject: [PATCH] test: stop guessing port numbers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Twenty test servers picked a port with `NNNN + Math.floor(Math.random() * 500)` and hoped it was free. Several files guessed inside the *same* range — sdk.test.ts and gates.test.ts both on 19500, three files on 22000, and webrtc-integration.test.ts used 22000 twice within itself — so with test files serving in parallel the collision was not unlikely, it was scheduled. Found from the outside, in Nova, where Shade is vendored: across eight runs of an unchanged tree, two had failures and six were clean. The visible error was Failed to start server. Is port 24287 in use? EADDRINUSE and then a second, louder one: `teardownRig` ran on a rig that setup had never finished building and died on `rig.alice`, so the TypeError from cleanup is what you read first and the real cause scrolled past above it. `Bun.serve({ port: 0 })` lets the OS hand out a free port and `server.port` reads it back. There is no range to collide inside. teardownRig now tolerates a rig that was never built. Cleanup must never be the loudest thing in a failing test. A suite that fails a quarter of the time is worse than a suite that fails: it teaches everyone to re-run it, and then a real regression looks like the flake. Verified: 1198 pass / 0 fail here, and the same fix is going into Nova's vendored copy so the next sync does not bring the guessing back. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014489bKUtUEY1Zgs9xN9mt7 --- packages/shade-sdk/tests/broadcast.test.ts | 8 +++- packages/shade-sdk/tests/gates.test.ts | 8 +++- packages/shade-sdk/tests/profile.test.ts | 8 +++- packages/shade-sdk/tests/sdk.test.ts | 8 +++- .../tests/streams-integration.test.ts | 16 +++++-- packages/shade-sdk/tests/thumbnail.test.ts | 16 +++++-- .../shade-sdk/tests/webrtc-failover.test.ts | 42 +++++++++++++------ .../tests/webrtc-integration.test.ts | 24 ++++++++--- .../shade-sdk/tests/webrtc-throughput.test.ts | 24 ++++++++--- packages/shade-transfer/tests/resume.test.ts | 8 +++- .../shade-transfer/tests/ws-fallback.test.ts | 8 +++- .../tests/kt-transport.test.ts | 8 +++- 12 files changed, 132 insertions(+), 46 deletions(-) diff --git a/packages/shade-sdk/tests/broadcast.test.ts b/packages/shade-sdk/tests/broadcast.test.ts index 0105e8a..91049c6 100644 --- a/packages/shade-sdk/tests/broadcast.test.ts +++ b/packages/shade-sdk/tests/broadcast.test.ts @@ -14,8 +14,12 @@ async function startPrekeyServer(): Promise<{ url: string; stop: () => void }> { store: new MemoryPrekeyStore(), disableRateLimit: true, }); - const port = 19500 + Math.floor(Math.random() * 500); - const handle = Bun.serve({ port, fetch: server.fetch }); + // Port 0 lets the OS hand out a free one. This guessed inside a 500-wide + // range and hoped — and several files here guessed inside the SAME range, + // so with test files serving in parallel the collision arrived about a + // quarter of the time as EADDRINUSE. + const handle = Bun.serve({ port: 0, fetch: server.fetch }); + const port = handle.port; return { url: `http://localhost:${port}`, stop: () => handle.stop() }; } diff --git a/packages/shade-sdk/tests/gates.test.ts b/packages/shade-sdk/tests/gates.test.ts index 34ca154..e092313 100644 --- a/packages/shade-sdk/tests/gates.test.ts +++ b/packages/shade-sdk/tests/gates.test.ts @@ -12,8 +12,12 @@ async function startPrekeyServer(): Promise<{ url: string; stop: () => void }> { store: new MemoryPrekeyStore(), disableRateLimit: true, }); - const port = 19500 + Math.floor(Math.random() * 500); - const handle = Bun.serve({ port, fetch: server.fetch }); + // Port 0 lets the OS hand out a free one. This guessed inside a 500-wide + // range and hoped — and several files here guessed inside the SAME range, + // so with test files serving in parallel the collision arrived about a + // quarter of the time as EADDRINUSE. + const handle = Bun.serve({ port: 0, fetch: server.fetch }); + const port = handle.port; return { url: `http://localhost:${port}`, stop: () => handle.stop() }; } diff --git a/packages/shade-sdk/tests/profile.test.ts b/packages/shade-sdk/tests/profile.test.ts index 017751c..a4769a3 100644 --- a/packages/shade-sdk/tests/profile.test.ts +++ b/packages/shade-sdk/tests/profile.test.ts @@ -36,8 +36,12 @@ async function startServer(): Promise { blobStore: new MemoryBlobStore(), disableRateLimit: true, }); - const port = 19000 + Math.floor(Math.random() * 500); - const handle = Bun.serve({ port, fetch: app.fetch }); + // Port 0 lets the OS hand out a free one. This guessed inside a 500-wide + // range and hoped — and several files here guessed inside the SAME range, + // so with test files serving in parallel the collision arrived about a + // quarter of the time as EADDRINUSE. + const handle = Bun.serve({ port: 0, fetch: app.fetch }); + const port = handle.port; return { url: `http://localhost:${port}`, stop: () => handle.stop(true), diff --git a/packages/shade-sdk/tests/sdk.test.ts b/packages/shade-sdk/tests/sdk.test.ts index db4e20e..37ad6ee 100644 --- a/packages/shade-sdk/tests/sdk.test.ts +++ b/packages/shade-sdk/tests/sdk.test.ts @@ -25,8 +25,12 @@ async function startPrekeyServer(): Promise<{ disableRateLimit: true, events, }); - const port = 19500 + Math.floor(Math.random() * 500); - const handle = Bun.serve({ port, fetch: server.fetch }); + // Port 0 lets the OS hand out a free one. This guessed inside a 500-wide + // range and hoped — and several files here guessed inside the SAME range, + // so with test files serving in parallel the collision arrived about a + // quarter of the time as EADDRINUSE. + const handle = Bun.serve({ port: 0, fetch: server.fetch }); + const port = handle.port; return { url: `http://localhost:${port}`, stop: () => handle.stop(), diff --git a/packages/shade-sdk/tests/streams-integration.test.ts b/packages/shade-sdk/tests/streams-integration.test.ts index 52269da..c3bff57 100644 --- a/packages/shade-sdk/tests/streams-integration.test.ts +++ b/packages/shade-sdk/tests/streams-integration.test.ts @@ -26,8 +26,12 @@ async function startPrekeyServer(): Promise<{ url: string; stop: () => void }> { disableRateLimit: true, events, }); - const port = 21000 + Math.floor(Math.random() * 500); - const handle = Bun.serve({ port, fetch: server.fetch }); + // Port 0 lets the OS hand out a free one. This guessed inside a 500-wide + // range and hoped — and several files here guessed inside the SAME range, + // so with test files serving in parallel the collision arrived about a + // quarter of the time as EADDRINUSE. + const handle = Bun.serve({ port: 0, fetch: server.fetch }); + const port = handle.port; return { url: `http://localhost:${port}`, stop: () => handle.stop() }; } @@ -46,8 +50,12 @@ async function setupRig(): Promise { // Spin up Bob's HTTP transfer endpoint. const bobApp = await bob.transferRoute(); - const port = 21500 + Math.floor(Math.random() * 500); - const bobServer = Bun.serve({ port, fetch: bobApp.fetch }); + // Port 0 lets the OS hand out a free one. This guessed inside a 500-wide + // range and hoped — and several files here guessed inside the SAME range, + // so with test files serving in parallel the collision arrived about a + // quarter of the time as EADDRINUSE. + const bobServer = Bun.serve({ port: 0, fetch: bobApp.fetch }); + const port = bobServer.port; const bobBaseUrl = `http://localhost:${port}`; // Wire up Alice's outgoing transfer routing. diff --git a/packages/shade-sdk/tests/thumbnail.test.ts b/packages/shade-sdk/tests/thumbnail.test.ts index 50661da..b396407 100644 --- a/packages/shade-sdk/tests/thumbnail.test.ts +++ b/packages/shade-sdk/tests/thumbnail.test.ts @@ -33,8 +33,12 @@ async function startPrekeyServer(): Promise<{ url: string; stop: () => void }> { disableRateLimit: true, events, }); - const port = 22000 + Math.floor(Math.random() * 500); - const handle = Bun.serve({ port, fetch: server.fetch }); + // Port 0 lets the OS hand out a free one. This guessed inside a 500-wide + // range and hoped — and several files here guessed inside the SAME range, + // so with test files serving in parallel the collision arrived about a + // quarter of the time as EADDRINUSE. + const handle = Bun.serve({ port: 0, fetch: server.fetch }); + const port = handle.port; return { url: `http://localhost:${port}`, stop: () => handle.stop() }; } @@ -49,8 +53,12 @@ async function setupRig(): Promise { }, }); const bobApp = await bob.transferRoute(); - const port = 22500 + Math.floor(Math.random() * 500); - const bobServer = Bun.serve({ port, fetch: bobApp.fetch }); + // Port 0 lets the OS hand out a free one. This guessed inside a 500-wide + // range and hoped — and several files here guessed inside the SAME range, + // so with test files serving in parallel the collision arrived about a + // quarter of the time as EADDRINUSE. + const bobServer = Bun.serve({ port: 0, fetch: bobApp.fetch }); + const port = bobServer.port; const bobBaseUrl = `http://localhost:${port}`; alice.configureTransfers({ diff --git a/packages/shade-sdk/tests/webrtc-failover.test.ts b/packages/shade-sdk/tests/webrtc-failover.test.ts index e3c117f..a3bfc1f 100644 --- a/packages/shade-sdk/tests/webrtc-failover.test.ts +++ b/packages/shade-sdk/tests/webrtc-failover.test.ts @@ -103,8 +103,12 @@ async function startPrekeyServer(): Promise<{ url: string; stop: () => void }> { disableRateLimit: true, events, }); - const port = 23000 + Math.floor(Math.random() * 500); - const handle = Bun.serve({ port, fetch: server.fetch }); + // Port 0 lets the OS hand out a free one. This guessed inside a 500-wide + // range and hoped — and several files here guessed inside the SAME range, + // so with test files serving in parallel the collision arrived about a + // quarter of the time as EADDRINUSE. + const handle = Bun.serve({ port: 0, fetch: server.fetch }); + const port = handle.port; return { url: `http://localhost:${port}`, stop: () => handle.stop() }; } @@ -127,13 +131,21 @@ async function setupRig(connectTimeoutMs: number): Promise { bob.configureWebRTC({ factory: broken, connectTimeoutMs }); const bobApp = await bob.transferRoute(); - const bobPort = 23500 + Math.floor(Math.random() * 500); - const bobServer = Bun.serve({ port: bobPort, fetch: bobApp.fetch }); + // Port 0 lets the OS hand out a free one. This guessed inside a 500-wide + // range and hoped — and several files here guessed inside the SAME range, + // so with test files serving in parallel the collision arrived about a + // quarter of the time as EADDRINUSE. + const bobServer = Bun.serve({ port: 0, fetch: bobApp.fetch }); + const bobPort = bobServer.port; baseUrls.set('bob', `http://localhost:${bobPort}`); const aliceApp = await alice.transferRoute(); - const alicePort = 24000 + Math.floor(Math.random() * 500); - const aliceServer = Bun.serve({ port: alicePort, fetch: aliceApp.fetch }); + // Port 0 lets the OS hand out a free one. This guessed inside a 500-wide + // range and hoped — and several files here guessed inside the SAME range, + // so with test files serving in parallel the collision arrived about a + // quarter of the time as EADDRINUSE. + const aliceServer = Bun.serve({ port: 0, fetch: aliceApp.fetch }); + const alicePort = aliceServer.port; baseUrls.set('alice', `http://localhost:${alicePort}`); return { @@ -145,12 +157,18 @@ async function setupRig(connectTimeoutMs: number): Promise { }; } -async function teardownRig(rig: Rig): Promise { - await rig.alice.shutdown(); - await rig.bob.shutdown(); - rig.bobServerStop(); - rig.aliceServerStop(); - rig.prekeyStop(); +async function teardownRig(rig: Rig | undefined): Promise { + // Tolerates a rig that never finished being built. When setupRig threw — + // EADDRINUSE, before ports stopped being guessed — this ran anyway and died + // on `rig.alice`, so one failure became two and the second one, a TypeError + // in teardown, is what you saw first. Cleanup must never be the loudest + // thing in a failing test. + if (!rig) return; + await rig.alice?.shutdown().catch(() => {}); + await rig.bob?.shutdown().catch(() => {}); + try { rig.bobServerStop?.(); } catch { /* already gone */ } + try { rig.aliceServerStop?.(); } catch { /* already gone */ } + try { rig.prekeyStop?.(); } catch { /* already gone */ } } function hex(b: Uint8Array): string { diff --git a/packages/shade-sdk/tests/webrtc-integration.test.ts b/packages/shade-sdk/tests/webrtc-integration.test.ts index aebe0b6..f5a1bad 100644 --- a/packages/shade-sdk/tests/webrtc-integration.test.ts +++ b/packages/shade-sdk/tests/webrtc-integration.test.ts @@ -40,8 +40,12 @@ async function startPrekeyServer(): Promise<{ url: string; stop: () => void }> { disableRateLimit: true, events, }); - const port = 22000 + Math.floor(Math.random() * 500); - const handle = Bun.serve({ port, fetch: server.fetch }); + // Port 0 lets the OS hand out a free one. This guessed inside a 500-wide + // range and hoped — and several files here guessed inside the SAME range, + // so with test files serving in parallel the collision arrived about a + // quarter of the time as EADDRINUSE. + const handle = Bun.serve({ port: 0, fetch: server.fetch }); + const port = handle.port; return { url: `http://localhost:${port}`, stop: () => handle.stop() }; } @@ -70,13 +74,21 @@ async function setupRig(): Promise { bob.configureWebRTC({ factory, connectTimeoutMs: 10_000 }); const bobApp = await bob.transferRoute(); - const bobPort = 22500 + Math.floor(Math.random() * 500); - const bobServer = Bun.serve({ port: bobPort, fetch: bobApp.fetch }); + // Port 0 lets the OS hand out a free one. This guessed inside a 500-wide + // range and hoped — and several files here guessed inside the SAME range, + // so with test files serving in parallel the collision arrived about a + // quarter of the time as EADDRINUSE. + const bobServer = Bun.serve({ port: 0, fetch: bobApp.fetch }); + const bobPort = bobServer.port; const bobBaseUrl = `http://localhost:${bobPort}`; const aliceApp = await alice.transferRoute(); - const alicePort = 22000 + Math.floor(Math.random() * 500); - const aliceServer = Bun.serve({ port: alicePort, fetch: aliceApp.fetch }); + // Port 0 lets the OS hand out a free one. This guessed inside a 500-wide + // range and hoped — and several files here guessed inside the SAME range, + // so with test files serving in parallel the collision arrived about a + // quarter of the time as EADDRINUSE. + const aliceServer = Bun.serve({ port: 0, fetch: aliceApp.fetch }); + const alicePort = aliceServer.port; const aliceBaseUrl = `http://localhost:${alicePort}`; baseUrls.set('alice', aliceBaseUrl); diff --git a/packages/shade-sdk/tests/webrtc-throughput.test.ts b/packages/shade-sdk/tests/webrtc-throughput.test.ts index cab8dcd..bb9b023 100644 --- a/packages/shade-sdk/tests/webrtc-throughput.test.ts +++ b/packages/shade-sdk/tests/webrtc-throughput.test.ts @@ -44,8 +44,12 @@ async function startPrekeyServer(): Promise<{ url: string; stop: () => void }> { disableRateLimit: true, events, }); - const port = 24500 + Math.floor(Math.random() * 500); - const handle = Bun.serve({ port, fetch: server.fetch }); + // Port 0 lets the OS hand out a free one. This guessed inside a 500-wide + // range and hoped — and several files here guessed inside the SAME range, + // so with test files serving in parallel the collision arrived about a + // quarter of the time as EADDRINUSE. + const handle = Bun.serve({ port: 0, fetch: server.fetch }); + const port = handle.port; return { url: `http://localhost:${port}`, stop: () => handle.stop() }; } @@ -70,13 +74,21 @@ async function setupRig(opts: { withWebRTC: boolean }): Promise { } const bobApp = await bob.transferRoute(); - const bobPort = 25000 + Math.floor(Math.random() * 500); - const bobServer = Bun.serve({ port: bobPort, fetch: bobApp.fetch }); + // Port 0 lets the OS hand out a free one. This guessed inside a 500-wide + // range and hoped — and several files here guessed inside the SAME range, + // so with test files serving in parallel the collision arrived about a + // quarter of the time as EADDRINUSE. + const bobServer = Bun.serve({ port: 0, fetch: bobApp.fetch }); + const bobPort = bobServer.port; baseUrls.set('bob', `http://localhost:${bobPort}`); const aliceApp = await alice.transferRoute(); - const alicePort = 25500 + Math.floor(Math.random() * 500); - const aliceServer = Bun.serve({ port: alicePort, fetch: aliceApp.fetch }); + // Port 0 lets the OS hand out a free one. This guessed inside a 500-wide + // range and hoped — and several files here guessed inside the SAME range, + // so with test files serving in parallel the collision arrived about a + // quarter of the time as EADDRINUSE. + const aliceServer = Bun.serve({ port: 0, fetch: aliceApp.fetch }); + const alicePort = aliceServer.port; baseUrls.set('alice', `http://localhost:${alicePort}`); return { diff --git a/packages/shade-transfer/tests/resume.test.ts b/packages/shade-transfer/tests/resume.test.ts index 76ac562..325ad13 100644 --- a/packages/shade-transfer/tests/resume.test.ts +++ b/packages/shade-transfer/tests/resume.test.ts @@ -41,8 +41,12 @@ describe('Resume protocol — kill-restart-verify', () => { myAddress: 'bob', }); const receiverApp = await createTransferRoutes(receiverEngine); - const port = 22000 + Math.floor(Math.random() * 500); - const server = Bun.serve({ port, fetch: receiverApp.fetch }); + // Port 0 lets the OS hand out a free one. This guessed inside a 500-wide + // range and hoped — and several files here guessed inside the SAME range, + // so with test files serving in parallel the collision arrived about a + // quarter of the time as EADDRINUSE. + const server = Bun.serve({ port: 0, fetch: receiverApp.fetch }); + const port = server.port; const baseUrl = `http://localhost:${port}`; // Receiver accepts incoming. diff --git a/packages/shade-transfer/tests/ws-fallback.test.ts b/packages/shade-transfer/tests/ws-fallback.test.ts index 3f0f153..befbca4 100644 --- a/packages/shade-transfer/tests/ws-fallback.test.ts +++ b/packages/shade-transfer/tests/ws-fallback.test.ts @@ -39,8 +39,12 @@ describe('WS opt-in transport with HTTP fallback', () => { myAddress: 'bob', }); const httpApp = await createTransferRoutes(receiverEngine); - const httpPort = 23000 + Math.floor(Math.random() * 500); - const httpServer = Bun.serve({ port: httpPort, fetch: httpApp.fetch }); + // Port 0 lets the OS hand out a free one. This guessed inside a 500-wide + // range and hoped — and several files here guessed inside the SAME range, + // so with test files serving in parallel the collision arrived about a + // quarter of the time as EADDRINUSE. + const httpServer = Bun.serve({ port: 0, fetch: httpApp.fetch }); + const httpPort = httpServer.port; const httpBaseUrl = `http://localhost:${httpPort}`; cleanups.push(() => httpServer.stop()); diff --git a/packages/shade-transport/tests/kt-transport.test.ts b/packages/shade-transport/tests/kt-transport.test.ts index 8ebfdac..4910596 100644 --- a/packages/shade-transport/tests/kt-transport.test.ts +++ b/packages/shade-transport/tests/kt-transport.test.ts @@ -31,8 +31,12 @@ describe('ShadeFetchTransport with KT verifier', () => { }, }); - const port = 20100 + Math.floor(Math.random() * 500); - const handle = Bun.serve({ port, fetch: app.fetch }); + // Port 0 lets the OS hand out a free one. This guessed inside a 500-wide + // range and hoped — and several files here guessed inside the SAME range, + // so with test files serving in parallel the collision arrived about a + // quarter of the time as EADDRINUSE. + const handle = Bun.serve({ port: 0, fetch: app.fetch }); + const port = handle.port; try { const baseUrl = `http://localhost:${port}`;