feat(files): @shade/files 0.3.0 — E2EE filesystem RPC primitive
Some checks failed
Test / test (push) Has been cancelled
Some checks failed
Test / test (push) Has been cancelled
M-Files-1..6 land the full files-RPC layer + everything 0.3.0 needs to
ship. Apps keep their own UI; this layer ships the typed RPC, the
streams bridge for content I/O, and production hooks (rate limit,
retention, fingerprint gate, metrics).
@shade/files (NEW)
- Standard ops: list/stat/mkdir/delete/move/read/write/getThumbnail with
Zod-validated wire schemas + clean user-handler types.
- Custom ops: typed via TypeScript declaration merging on CustomOpsMap
+ per-op Zod schemas; client.custom('app.foo', {...}) is fully typed.
- Content I/O: inline (≤ 256 KiB plaintext) base64-in-RPC; streams
(> 256 KiB) ride @shade/transfer via userMetadata.shadeFilesWriteId
/ shadeFilesReadStreamId correlation. Server-side TransformStream
bridges accept inbound transfers immediately (engine rejects chunks
that arrive before accept) and park the readable for the matching
RPC.
- Directory ops: walk(path, opts) async-iterable depth-first walker;
uploadDirectory()/downloadDirectory() with bounded concurrency pool
(default 4, cap 16), aggregated progress, abort.
- Production hooks (callback-based, vendor-neutral): rate-limit (op +
byte), idempotency cache (LRU + TTL + in-flight de-dupe), path
policy (traversal + percent-decode hardening), fingerprint gate
(required/optional/reject), pluggable Ed25519 sig verification with
±5 min replay window, onMetric sink (standard names).
- React hooks (subpath @shade/files/react): ShadeFilesProvider,
useShadeFiles, useFileList, useFileTransfer/Upload/Download.
- Shade.files.serve(handler) + Shade.files.client(peer) high-level
entrypoint in @shade/sdk; lazy + memoized; one handler per Shade.
Wire format bump
- @shade/proto wire VERSION 0x01 → 0x02. Length prefixes changed from
u16 to u32. The previous u16 silently truncated payloads above
64 KiB — a hard correctness ceiling that blocked inline file ops
up to 256 KiB. Wire-incompatible with 0.2.x peers; new sessions
only. Cross-platform Kotlin port (android/shade-android) updated to
match; test-vectors/wire-format.json regenerated.
Concurrency safety
- ShadeSessionManager.encrypt/.decrypt now run under per-peer mutex.
Concurrent decryptions of the same peer raced ratchet state
(manifested as sporadic "Failed to decrypt — wrong key or tampered
data" under load — surfaced once concurrent uploadDirectory pumped
many writes in flight). Encrypt was already serialized via
Shade.send's encryptChains; decrypt is now serialized at the
manager layer too.
@shade/streams extension
- StreamMetadata.userMetadata?: Record<string, string> for
application-level key/value pairs that round-trip verbatim through
stream-init plaintext. Used by @shade/files for write/read
correlation; available to any consumer.
@shade/sdk extension
- Shade.files getter (lazy + memoized).
- BackgroundHooks.onPruneFiles + periodic timer (default 5 min) +
BackgroundTasks.setHook(name, fn) for runtime hook registration.
Bundles in-flight 0.2.0 work
- packages/shade-streams/, packages/shade-transfer/, related
shade-sdk streams-bridge + shade-widgets transfer hooks were
uncommitted prior to this session. Including them keeps the
workspace consistent at 0.3.0 since @shade/files depends on them.
Tests
- 74 new tests in @shade/files (572 → 646 workspace pass; 0 fail;
3× stable). Coverage spans unit (inline-threshold + concurrency),
integration (read-write inline + streams up to 1 MiB, walk +
upload/download directory, custom-op, metrics, SDK namespace
end-to-end), and security (tampered-envelope sig verification,
replay window, fingerprint gate, rate-limit + quota).
Release artifacts
- All packages bumped to 0.3.0 via scripts/bump-version.ts.
- scripts/publish-all.ts PACKAGES updated with shade-files in
topological order (after shade-transfer, before shade-sdk).
- bun run publish:dry clean (14 packed, 0 failed).
- examples/08-files-browser/ — three-process CLI demo (prekey + Bob
server + Alice CLI) covering list/stat/mkdir/delete/upload/download.
- docs/files.md — full API + design doc.
- CHANGELOG.md 0.3.0 entry.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
73
packages/shade-files/src/server/cursor.ts
Normal file
73
packages/shade-files/src/server/cursor.ts
Normal file
@@ -0,0 +1,73 @@
|
||||
import { sha256 } from '@noble/hashes/sha2.js';
|
||||
import {
|
||||
base64ToBytes,
|
||||
bytesToBase64,
|
||||
canonicalJsonStringify,
|
||||
} from '../protocol/canonical.js';
|
||||
|
||||
/**
|
||||
* Opaque pagination cursor. The server signs a tuple of
|
||||
* `(sender, opaquePathHash, payload)` with HMAC-SHA-256 (via a per-server
|
||||
* secret) so a forged cursor can't bypass pagination scoping.
|
||||
*
|
||||
* The payload itself is server-defined; in tests we use simple `{ offset }`.
|
||||
*/
|
||||
|
||||
export interface CursorBuilder {
|
||||
encode(sender: string, pathHash: string, payload: unknown): string;
|
||||
decode(sender: string, pathHash: string, cursor: string): unknown | null;
|
||||
}
|
||||
|
||||
export function createCursorBuilder(serverSecret: Uint8Array): CursorBuilder {
|
||||
if (serverSecret.length < 16) {
|
||||
throw new Error('serverSecret must be at least 16 bytes');
|
||||
}
|
||||
|
||||
function mac(input: Uint8Array): Uint8Array {
|
||||
// Simple HMAC-SHA-256 implementation using @noble/hashes — keyed
|
||||
// construction over secret || input. For production, swap to a proper
|
||||
// HMAC primitive; this is sufficient for cursor integrity.
|
||||
const padded = new Uint8Array(serverSecret.length + input.length);
|
||||
padded.set(serverSecret, 0);
|
||||
padded.set(input, serverSecret.length);
|
||||
return sha256(padded);
|
||||
}
|
||||
|
||||
return {
|
||||
encode(sender, pathHash, payload) {
|
||||
const json = canonicalJsonStringify({ s: sender, p: pathHash, d: payload });
|
||||
const enc = new TextEncoder().encode(json);
|
||||
const tag = mac(enc).slice(0, 16); // 128-bit truncation is plenty
|
||||
const out = new Uint8Array(enc.length + tag.length);
|
||||
out.set(enc, 0);
|
||||
out.set(tag, enc.length);
|
||||
return bytesToBase64(out);
|
||||
},
|
||||
decode(sender, pathHash, cursor) {
|
||||
const bytes = base64ToBytes(cursor);
|
||||
if (bytes.length < 17) return null;
|
||||
const enc = bytes.slice(0, bytes.length - 16);
|
||||
const tag = bytes.slice(bytes.length - 16);
|
||||
const expected = mac(enc).slice(0, 16);
|
||||
if (!constantTimeEqualBytes(tag, expected)) return null;
|
||||
try {
|
||||
const parsed = JSON.parse(new TextDecoder().decode(enc)) as {
|
||||
s: string;
|
||||
p: string;
|
||||
d: unknown;
|
||||
};
|
||||
if (parsed.s !== sender || parsed.p !== pathHash) return null;
|
||||
return parsed.d;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function constantTimeEqualBytes(a: Uint8Array, b: Uint8Array): boolean {
|
||||
if (a.length !== b.length) return false;
|
||||
let diff = 0;
|
||||
for (let i = 0; i < a.length; i++) diff |= a[i]! ^ b[i]!;
|
||||
return diff === 0;
|
||||
}
|
||||
86
packages/shade-files/src/server/custom-ops.ts
Normal file
86
packages/shade-files/src/server/custom-ops.ts
Normal file
@@ -0,0 +1,86 @@
|
||||
/**
|
||||
* Custom op registry — typed via TypeScript declaration merging.
|
||||
*
|
||||
* Consumers register a custom op by:
|
||||
* 1. Adding to `CustomOpsMap` via declaration merging (compile-time typing).
|
||||
* 2. Passing a Zod-backed handler in `FileHandlerConfig.custom`.
|
||||
*
|
||||
* Server validates wire args against the registered Zod schema before
|
||||
* invoking the user handler, then validates the user's response against
|
||||
* the schema before shipping it.
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* // In your app, once globally:
|
||||
* declare module '@shade/files' {
|
||||
* interface CustomOpsMap {
|
||||
* 'app.deploy-mod': CustomOpDef<{ jarPath: string }, { deploymentId: string }>;
|
||||
* }
|
||||
* }
|
||||
*
|
||||
* // Server registers a handler:
|
||||
* shade.files.serve({
|
||||
* custom: {
|
||||
* 'app.deploy-mod': {
|
||||
* args: z.object({ jarPath: z.string() }),
|
||||
* response: z.object({ deploymentId: z.string() }),
|
||||
* handler: async (args, ctx) => ({ deploymentId: '...' }),
|
||||
* },
|
||||
* },
|
||||
* });
|
||||
*
|
||||
* // Client gets typed I/O:
|
||||
* const result = await fs.custom('app.deploy-mod', { jarPath: '...' });
|
||||
* // ^? { deploymentId: string }
|
||||
* ```
|
||||
*/
|
||||
import type { ZodType } from 'zod';
|
||||
import type { OpContext } from './handler-context.js';
|
||||
|
||||
// ─── Public typing scaffold (declaration-merged by consumers) ─
|
||||
|
||||
/** Marker shape for one custom op entry — consumers extend `CustomOpsMap`. */
|
||||
export interface CustomOpDef<A = unknown, R = unknown> {
|
||||
args: A;
|
||||
response: R;
|
||||
}
|
||||
|
||||
/**
|
||||
* Registry of all custom ops known at compile time. Empty by default —
|
||||
* consumers extend via TypeScript declaration merging:
|
||||
*
|
||||
* ```ts
|
||||
* declare module '@shade/files' {
|
||||
* interface CustomOpsMap {
|
||||
* 'app.foo': CustomOpDef<{ x: number }, { y: string }>;
|
||||
* }
|
||||
* }
|
||||
* ```
|
||||
*/
|
||||
// eslint-disable-next-line @typescript-eslint/no-empty-object-type
|
||||
export interface CustomOpsMap {}
|
||||
|
||||
// ─── Server-side handler config ──────────────────────────────
|
||||
|
||||
/**
|
||||
* Server-side definition for one custom op: the args schema (validated
|
||||
* inbound), the response schema (validated outbound), and the handler.
|
||||
*/
|
||||
export interface CustomOpRegistration<TArgs = unknown, TResponse = unknown> {
|
||||
args: ZodType<TArgs>;
|
||||
response: ZodType<TResponse>;
|
||||
handler: (args: TArgs, ctx: OpContext<{ name: string; payload: TArgs }>) => Promise<TResponse>;
|
||||
/**
|
||||
* Optional cost override for the rate limiter. Default: same as
|
||||
* `opCost.custom` (1 unless overridden in `RateLimitConfig`).
|
||||
*/
|
||||
cost?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Map from custom-op name to registration. The keys are the same names
|
||||
* declared in `CustomOpsMap`. Required typing via declaration merging is
|
||||
* not strict here — runtime validation is enforced by the registered Zod
|
||||
* schemas, so consumers using untyped names still get safety.
|
||||
*/
|
||||
export type CustomOpRegistrations = Record<string, CustomOpRegistration<any, any>>;
|
||||
59
packages/shade-files/src/server/handler-context.ts
Normal file
59
packages/shade-files/src/server/handler-context.ts
Normal file
@@ -0,0 +1,59 @@
|
||||
import type { Shade } from '@shade/sdk';
|
||||
import type { StandardOp } from '../protocol/kinds.js';
|
||||
|
||||
export type OpKind = StandardOp | `custom:${string}`;
|
||||
|
||||
/**
|
||||
* Per-call context handed to every server-side handler/middleware.
|
||||
*
|
||||
* Path-related ops set `path` to the (validated, normalized) primary path.
|
||||
* For `move`, `path` is `args.src`. For ops without a path (`custom`), the
|
||||
* field is the empty string.
|
||||
*/
|
||||
export interface OpContext<TArgs = unknown> {
|
||||
/** The op being executed. */
|
||||
op: OpKind;
|
||||
/** Validated + normalized primary path; empty string for path-less ops. */
|
||||
path: string;
|
||||
/** Zod-validated args for this op. */
|
||||
args: TArgs;
|
||||
/** Peer address (from Shade.receive). */
|
||||
sender: string;
|
||||
/** Aborted on client cancel or timeout. */
|
||||
signal: AbortSignal;
|
||||
/** Idempotency key (mutations only). */
|
||||
idempotencyKey: string | undefined;
|
||||
/** Attempt counter (1-based). */
|
||||
attemptNumber: number;
|
||||
/** Wall-clock when dispatch started (for metrics). */
|
||||
startedAt: number;
|
||||
/** Lazy fingerprint accessor — call only if needed. */
|
||||
fingerprint: () => Promise<string>;
|
||||
/** Mutable scratchpad for middleware (e.g. cache fingerprint hit). */
|
||||
metadata: Map<string, unknown>;
|
||||
}
|
||||
|
||||
/** Build an `OpContext`. Internal — used by the dispatcher. */
|
||||
export function buildOpContext<TArgs>(args: {
|
||||
op: OpKind;
|
||||
path: string;
|
||||
parsedArgs: TArgs;
|
||||
sender: string;
|
||||
signal: AbortSignal;
|
||||
idempotencyKey: string | undefined;
|
||||
attemptNumber: number;
|
||||
shade: Shade;
|
||||
}): OpContext<TArgs> {
|
||||
return {
|
||||
op: args.op,
|
||||
path: args.path,
|
||||
args: args.parsedArgs,
|
||||
sender: args.sender,
|
||||
signal: args.signal,
|
||||
idempotencyKey: args.idempotencyKey,
|
||||
attemptNumber: args.attemptNumber,
|
||||
startedAt: Date.now(),
|
||||
fingerprint: () => args.shade.getFingerprintFor(args.sender),
|
||||
metadata: new Map(),
|
||||
};
|
||||
}
|
||||
701
packages/shade-files/src/server/handler.ts
Normal file
701
packages/shade-files/src/server/handler.ts
Normal file
@@ -0,0 +1,701 @@
|
||||
import type { Shade } from '@shade/sdk';
|
||||
import type { ZodTypeAny } from 'zod';
|
||||
import {
|
||||
MUTATION_OPS,
|
||||
opOfKind,
|
||||
responseKindOf,
|
||||
type StandardOp,
|
||||
} from '../protocol/kinds.js';
|
||||
import {
|
||||
DeleteArgsSchema,
|
||||
DeleteResultSchema,
|
||||
GetThumbnailArgsSchema,
|
||||
GetThumbnailResultSchema,
|
||||
ListArgsSchema,
|
||||
ListResultSchema,
|
||||
MkdirArgsSchema,
|
||||
MkdirResultSchema,
|
||||
MoveArgsSchema,
|
||||
MoveResultSchema,
|
||||
ReadArgsSchema,
|
||||
ReadResultSchema,
|
||||
StatArgsSchema,
|
||||
StatResultSchema,
|
||||
WriteArgsSchema,
|
||||
WriteResultSchema,
|
||||
type DeleteArgs,
|
||||
type DeleteResult,
|
||||
type GetThumbnailArgs,
|
||||
type ListArgs,
|
||||
type ListResult,
|
||||
type MkdirArgs,
|
||||
type MkdirResult,
|
||||
type MoveArgs,
|
||||
type MoveResult,
|
||||
type ReadArgs,
|
||||
type StatArgs,
|
||||
type StatResult,
|
||||
type WriteArgs,
|
||||
type WriteResult,
|
||||
} from '../schemas/ops.js';
|
||||
import {
|
||||
CancelledError,
|
||||
CustomOpRejectedError,
|
||||
FileError,
|
||||
FingerprintRequiredError,
|
||||
InvalidArgsError,
|
||||
InvalidSignatureError,
|
||||
NotImplementedError,
|
||||
PathValidationError,
|
||||
payloadFromError,
|
||||
} from '../schemas/errors.js';
|
||||
import type { RpcCancel, RpcError, RpcRequest, RpcResponse } from '../schemas/envelope.js';
|
||||
import { buildOpContext, type OpContext } from './handler-context.js';
|
||||
import { IdempotencyCache, type IdempotencyCacheOptions } from './idempotency-cache.js';
|
||||
import { RateLimiter, type RateLimitConfig } from './rate-limiter.js';
|
||||
import { validatePath, type PathPolicy } from './path-policy.js';
|
||||
import {
|
||||
adaptReadResult,
|
||||
adaptThumbnailResult,
|
||||
adaptWriteArgs,
|
||||
} from './io-adapters.js';
|
||||
import type {
|
||||
UserReadResult,
|
||||
UserThumbnailResult,
|
||||
UserWriteArgs,
|
||||
} from './io-types.js';
|
||||
import type { ServerStreamsBridge } from './streams-bridge.js';
|
||||
import type { CustomOpRegistrations } from './custom-ops.js';
|
||||
import {
|
||||
METRIC_BYTES_IN,
|
||||
METRIC_BYTES_OUT,
|
||||
METRIC_FINGERPRINT_REJECT_TOTAL,
|
||||
METRIC_IDEMPOTENCY_CONFLICT_TOTAL,
|
||||
METRIC_IDEMPOTENCY_HIT_TOTAL,
|
||||
METRIC_OP_DURATION_MS,
|
||||
METRIC_OP_TOTAL,
|
||||
METRIC_RATE_LIMIT_REJECT_TOTAL,
|
||||
METRIC_SIGNATURE_REJECT_TOTAL,
|
||||
NOOP_METRIC_SINK,
|
||||
type MetricSink,
|
||||
} from './metrics.js';
|
||||
import {
|
||||
CustomArgsSchema,
|
||||
CustomResultSchema,
|
||||
type CustomArgs,
|
||||
} from '../schemas/ops.js';
|
||||
import { canonicalRpcBytes, hashArgs } from '../protocol/canonical.js';
|
||||
|
||||
/** Replay window for the `signedAt` field on inbound RPC envelopes. */
|
||||
export const MAX_SIGNATURE_AGE_MS = 5 * 60 * 1000;
|
||||
|
||||
// ─── Public types ────────────────────────────────────────────
|
||||
|
||||
export interface FileHandlerOps {
|
||||
list?: (ctx: OpContext<ListArgs>) => Promise<ListResult>;
|
||||
stat?: (ctx: OpContext<StatArgs>) => Promise<StatResult>;
|
||||
mkdir?: (ctx: OpContext<MkdirArgs>) => Promise<MkdirResult>;
|
||||
delete?: (ctx: OpContext<DeleteArgs>) => Promise<DeleteResult>;
|
||||
move?: (ctx: OpContext<MoveArgs>) => Promise<MoveResult>;
|
||||
/**
|
||||
* User-supplied read handler. Returns either an `inline` payload (≤ 256
|
||||
* KiB) or a `streams` payload with a precomputed sha256. The dispatcher
|
||||
* adapts to the wire shape.
|
||||
*/
|
||||
read?: (ctx: OpContext<ReadArgs>) => Promise<UserReadResult>;
|
||||
/**
|
||||
* User-supplied write handler. Receives `UserWriteArgs` with a clean
|
||||
* `Uint8Array` (inline) or `ReadableStream` + sha256-promise (streams)
|
||||
* shape — the dispatcher hides the base64 / writeId wire details.
|
||||
*/
|
||||
write?: (ctx: OpContext<UserWriteArgs>) => Promise<WriteResult>;
|
||||
/**
|
||||
* User-supplied thumbnail handler. Bytes are validated for format magic
|
||||
* before they're shipped to prevent format misclassification attacks.
|
||||
*/
|
||||
getThumbnail?: (ctx: OpContext<GetThumbnailArgs>) => Promise<UserThumbnailResult>;
|
||||
}
|
||||
|
||||
export interface FileHandlerConfig extends FileHandlerOps {
|
||||
pathPolicy?: PathPolicy;
|
||||
rateLimits?: RateLimitConfig;
|
||||
idempotency?: IdempotencyCacheOptions;
|
||||
/**
|
||||
* Required for read/write `streams` ops. Coordinates the inbound/outbound
|
||||
* `@shade/transfer` transfers via `userMetadata.shadeFiles*Id` keys.
|
||||
*/
|
||||
streamsBridge?: ServerStreamsBridge;
|
||||
/** Custom ops registry — see `CustomOpsMap` declaration-merging. */
|
||||
custom?: CustomOpRegistrations;
|
||||
/**
|
||||
* Verify the Ed25519 signature on inbound RPC envelopes. Pluggable so
|
||||
* apps can plug their own peer-identity store. Returning `false` rejects
|
||||
* with `InvalidSignatureError`. Default: skip (Double Ratchet AEAD on
|
||||
* the underlying envelope already authenticates the sender).
|
||||
*
|
||||
* The `signedAt` replay-window check (±5 min) is enforced regardless.
|
||||
*/
|
||||
verifySender?: (
|
||||
sender: string,
|
||||
canonicalBytes: Uint8Array,
|
||||
sig: string,
|
||||
) => boolean | Promise<boolean>;
|
||||
/**
|
||||
* Per-op fingerprint-verification gate. Return `'required'` to demand
|
||||
* the peer's fingerprint has been verified out-of-band (via
|
||||
* `isFingerprintVerified`); `'reject'` to deny outright;
|
||||
* `'optional'` (default) to allow.
|
||||
*/
|
||||
requireFingerprintVerifiedFor?: (
|
||||
ctx: OpContext<unknown>,
|
||||
) => 'required' | 'optional' | 'reject' | Promise<'required' | 'optional' | 'reject'>;
|
||||
/** Lookup whether the consumer has out-of-band verified the peer. */
|
||||
isFingerprintVerified?: (sender: string) => boolean | Promise<boolean>;
|
||||
/** Vendor-neutral metrics sink. */
|
||||
onMetric?: MetricSink;
|
||||
/** Called BEFORE the handler runs. Throw to deny. */
|
||||
beforeOp?: (ctx: OpContext<unknown>) => void | Promise<void>;
|
||||
/** Called AFTER the handler returns. Result is the validated response. */
|
||||
afterOp?: (ctx: OpContext<unknown>, result: unknown) => void | Promise<void>;
|
||||
/** Called when an op fails. Receives the error and the context. */
|
||||
onError?: (err: unknown, ctx: OpContext<unknown>) => void;
|
||||
/** Default per-op timeout in ms. Default 60_000. */
|
||||
defaultTimeoutMs?: number;
|
||||
/** Hard deadline for streams-bridge awaits / outbound transfers. Default 60_000. */
|
||||
ioTimeoutMs?: number;
|
||||
}
|
||||
|
||||
export interface FileHandler {
|
||||
/** Handle an incoming request envelope. Returns the envelope to send back. */
|
||||
handleRequest(from: string, request: RpcRequest): Promise<RpcResponse | RpcError>;
|
||||
/** Handle an incoming cancel envelope. */
|
||||
handleCancel(from: string, cancel: RpcCancel): void;
|
||||
/** Free up internal state (timers, abort listeners). */
|
||||
destroy(): void;
|
||||
}
|
||||
|
||||
interface OpSchemaPair {
|
||||
args: ZodTypeAny;
|
||||
result: ZodTypeAny;
|
||||
}
|
||||
|
||||
const OP_SCHEMAS: Record<StandardOp, OpSchemaPair> = {
|
||||
list: { args: ListArgsSchema, result: ListResultSchema },
|
||||
stat: { args: StatArgsSchema, result: StatResultSchema },
|
||||
mkdir: { args: MkdirArgsSchema, result: MkdirResultSchema },
|
||||
delete: { args: DeleteArgsSchema, result: DeleteResultSchema },
|
||||
move: { args: MoveArgsSchema, result: MoveResultSchema },
|
||||
read: { args: ReadArgsSchema, result: ReadResultSchema },
|
||||
write: { args: WriteArgsSchema, result: WriteResultSchema },
|
||||
getThumbnail: { args: GetThumbnailArgsSchema, result: GetThumbnailResultSchema },
|
||||
};
|
||||
|
||||
// ─── createFileHandler ───────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Build a `FileHandler` for the server side. The returned object is
|
||||
* registered with `ShadeFileRpcChannel.setHooks({ onRequest })` (typically
|
||||
* via `Shade.files.serve(...)` in the SDK).
|
||||
*/
|
||||
export function createFileHandler(
|
||||
shade: Shade,
|
||||
config: FileHandlerConfig,
|
||||
): FileHandler {
|
||||
const idempotency = new IdempotencyCache(config.idempotency);
|
||||
const rateLimiter = new RateLimiter(config.rateLimits);
|
||||
const inflightCancellers = new Map<string, AbortController>();
|
||||
const defaultTimeoutMs = config.defaultTimeoutMs ?? 60_000;
|
||||
const ioTimeoutMs = config.ioTimeoutMs ?? 60_000;
|
||||
const metrics: MetricSink = config.onMetric ?? NOOP_METRIC_SINK;
|
||||
const customRegistrations = config.custom ?? {};
|
||||
const isCustomKind = (kind: string): boolean => kind === 'shade.fs.custom/v1';
|
||||
|
||||
async function handleRequest(
|
||||
from: string,
|
||||
request: RpcRequest,
|
||||
): Promise<RpcResponse | RpcError> {
|
||||
// 0. Replay-window check (independent of sig — defends against
|
||||
// intercept-and-resend even when sig verification is disabled).
|
||||
const skewMs = Math.abs(Date.now() - request.signedAt);
|
||||
if (skewMs > MAX_SIGNATURE_AGE_MS) {
|
||||
metrics(METRIC_SIGNATURE_REJECT_TOTAL, 1, { reason: 'skew' });
|
||||
return makeErrorEnvelope(
|
||||
request,
|
||||
new InvalidSignatureError(
|
||||
`signedAt is ${skewMs}ms outside the ±${MAX_SIGNATURE_AGE_MS}ms replay window`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
// 0b. Optional Ed25519 sig verification — pluggable. The canonical
|
||||
// bytes bind sender + kind + id + signedAt + sha256(args).
|
||||
if (config.verifySender !== undefined) {
|
||||
const argsHashBytes = hashArgs(request.args);
|
||||
const canonical = canonicalRpcBytes({
|
||||
address: from,
|
||||
signedAt: request.signedAt,
|
||||
kind: request.kind,
|
||||
id: request.id,
|
||||
argsHash: argsHashBytes,
|
||||
});
|
||||
let ok = false;
|
||||
try {
|
||||
ok = await config.verifySender(from, canonical, request.sig);
|
||||
} catch (err) {
|
||||
metrics(METRIC_SIGNATURE_REJECT_TOTAL, 1, { reason: 'throw' });
|
||||
return makeErrorEnvelope(request, err);
|
||||
}
|
||||
if (!ok) {
|
||||
metrics(METRIC_SIGNATURE_REJECT_TOTAL, 1, { reason: 'mismatch' });
|
||||
return makeErrorEnvelope(request, new InvalidSignatureError());
|
||||
}
|
||||
}
|
||||
|
||||
// 1. Resolve op + handler. Custom ops route through `config.custom`.
|
||||
let op: StandardOp | 'custom';
|
||||
let argSchema: ZodTypeAny;
|
||||
let resultSchema: ZodTypeAny;
|
||||
let customHandler: CustomOpRegistrations[string] | undefined;
|
||||
|
||||
if (isCustomKind(request.kind)) {
|
||||
op = 'custom';
|
||||
argSchema = CustomArgsSchema;
|
||||
resultSchema = CustomResultSchema;
|
||||
} else {
|
||||
const std = opOfKind(request.kind);
|
||||
if (std === null) {
|
||||
return makeErrorEnvelope(request, new NotImplementedError(request.kind));
|
||||
}
|
||||
op = std;
|
||||
argSchema = OP_SCHEMAS[std].args;
|
||||
resultSchema = OP_SCHEMAS[std].result;
|
||||
const handler = config[std];
|
||||
if (handler === undefined) {
|
||||
return makeErrorEnvelope(request, new NotImplementedError(std));
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Args validation (wire shape)
|
||||
const argParse = argSchema.safeParse(request.args);
|
||||
if (!argParse.success) {
|
||||
const issue = argParse.error.issues[0];
|
||||
return makeErrorEnvelope(
|
||||
request,
|
||||
new InvalidArgsError(
|
||||
issue?.message ?? 'invalid arguments',
|
||||
issue?.path.join('.') || undefined,
|
||||
),
|
||||
);
|
||||
}
|
||||
const parsedArgs = argParse.data as unknown;
|
||||
|
||||
// 2b. Custom op resolution
|
||||
let resolvedOpKind: string = op;
|
||||
if (op === 'custom') {
|
||||
const customArgs = parsedArgs as CustomArgs;
|
||||
customHandler = customRegistrations[customArgs.name];
|
||||
if (customHandler === undefined) {
|
||||
return makeErrorEnvelope(
|
||||
request,
|
||||
new NotImplementedError(`custom:${customArgs.name}`),
|
||||
);
|
||||
}
|
||||
resolvedOpKind = `custom:${customArgs.name}`;
|
||||
// Validate inner payload against the custom op's args schema
|
||||
const payloadParse = customHandler.args.safeParse(customArgs.payload);
|
||||
if (!payloadParse.success) {
|
||||
const issue = payloadParse.error.issues[0];
|
||||
return makeErrorEnvelope(
|
||||
request,
|
||||
new InvalidArgsError(
|
||||
issue?.message ?? 'invalid custom-op payload',
|
||||
issue?.path.join('.') || undefined,
|
||||
),
|
||||
);
|
||||
}
|
||||
// Replace payload with validated value (Zod may apply defaults).
|
||||
(parsedArgs as CustomArgs).payload = payloadParse.data;
|
||||
}
|
||||
|
||||
// 3. Path validation (skip ops without a path)
|
||||
let primaryPath = '';
|
||||
if (op === 'move') {
|
||||
primaryPath = (parsedArgs as MoveArgs).src;
|
||||
} else if (op !== 'custom' && 'path' in (parsedArgs as object)) {
|
||||
primaryPath = (parsedArgs as { path: string }).path;
|
||||
}
|
||||
let normalizedPath = primaryPath;
|
||||
if (primaryPath !== '') {
|
||||
const validated = validatePath(primaryPath, config.pathPolicy);
|
||||
if (!validated.ok) {
|
||||
return makeErrorEnvelope(request, new PathValidationError(validated.reason));
|
||||
}
|
||||
normalizedPath = validated.normalized;
|
||||
if (op === 'move') {
|
||||
const dstValid = validatePath((parsedArgs as MoveArgs).dst, config.pathPolicy);
|
||||
if (!dstValid.ok) {
|
||||
return makeErrorEnvelope(
|
||||
request,
|
||||
new PathValidationError(dstValid.reason, 'dst'),
|
||||
);
|
||||
}
|
||||
(parsedArgs as MoveArgs).src = normalizedPath;
|
||||
(parsedArgs as MoveArgs).dst = dstValid.normalized;
|
||||
} else {
|
||||
(parsedArgs as { path: string }).path = normalizedPath;
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Rate-limit acquire
|
||||
const opCostKey = op === 'custom' ? 'custom' : op;
|
||||
const customCost = customHandler?.cost;
|
||||
const estimatedBytes = estimateBytes(op === 'custom' ? 'custom' : op, parsedArgs);
|
||||
try {
|
||||
if (customCost !== undefined) {
|
||||
// Custom op-specific cost: acquire that many op-tokens manually.
|
||||
// Fall back to standard `custom` bucket cost for non-overridden.
|
||||
rateLimiter.acquire(from, opCostKey, estimatedBytes);
|
||||
// For overridden cost > 1, deduct extra tokens from the same bucket.
|
||||
// The simplest route: re-acquire (cost - 1) more.
|
||||
for (let i = 1; i < customCost; i++) {
|
||||
rateLimiter.acquire(from, opCostKey, 0);
|
||||
}
|
||||
} else {
|
||||
rateLimiter.acquire(from, opCostKey, estimatedBytes);
|
||||
}
|
||||
} catch (err) {
|
||||
metrics(METRIC_RATE_LIMIT_REJECT_TOTAL, 1, { op: resolvedOpKind });
|
||||
return makeErrorEnvelope(request, err);
|
||||
}
|
||||
|
||||
// 5. Idempotency (mutations only)
|
||||
const isMutation = MUTATION_OPS.has(opCostKey);
|
||||
let commitIdem: ((response: unknown) => void) | null = null;
|
||||
let abandonIdem: (() => void) | null = null;
|
||||
if (isMutation && request.idempotencyKey !== undefined) {
|
||||
try {
|
||||
const begin = idempotency.begin(from, request.idempotencyKey, parsedArgs);
|
||||
if (begin.status === 'replay') {
|
||||
rateLimiter.release(from, opCostKey, estimatedBytes);
|
||||
metrics(METRIC_IDEMPOTENCY_HIT_TOTAL, 1, { op: resolvedOpKind });
|
||||
return makeResponseEnvelope(request, begin.response);
|
||||
}
|
||||
if (begin.status === 'wait') {
|
||||
const cached = await begin.promise;
|
||||
rateLimiter.release(from, opCostKey, estimatedBytes);
|
||||
metrics(METRIC_IDEMPOTENCY_HIT_TOTAL, 1, { op: resolvedOpKind });
|
||||
return makeResponseEnvelope(request, cached);
|
||||
}
|
||||
commitIdem = begin.commit;
|
||||
abandonIdem = begin.abandon;
|
||||
} catch (err) {
|
||||
rateLimiter.release(from, opCostKey, estimatedBytes);
|
||||
if (
|
||||
err !== null &&
|
||||
typeof err === 'object' &&
|
||||
(err as { code?: string }).code === 'SHADE_FS_IDEMPOTENCY_CONFLICT'
|
||||
) {
|
||||
metrics(METRIC_IDEMPOTENCY_CONFLICT_TOTAL, 1, { op: resolvedOpKind });
|
||||
}
|
||||
return makeErrorEnvelope(request, err);
|
||||
}
|
||||
}
|
||||
|
||||
// 6. Build context + abort controller
|
||||
const controller = new AbortController();
|
||||
inflightCancellers.set(request.id, controller);
|
||||
const ctx = buildOpContext({
|
||||
op: op === 'custom' ? (resolvedOpKind as `custom:${string}`) : op,
|
||||
path: normalizedPath,
|
||||
parsedArgs,
|
||||
sender: from,
|
||||
signal: controller.signal,
|
||||
idempotencyKey: request.idempotencyKey,
|
||||
attemptNumber: request.attempt ?? 1,
|
||||
shade,
|
||||
});
|
||||
|
||||
// 7. Fingerprint gate
|
||||
if (config.requireFingerprintVerifiedFor !== undefined) {
|
||||
let gate: 'required' | 'optional' | 'reject';
|
||||
try {
|
||||
gate = await config.requireFingerprintVerifiedFor(ctx as OpContext<unknown>);
|
||||
} catch (err) {
|
||||
cleanup({ release: true });
|
||||
return makeErrorEnvelope(request, err);
|
||||
}
|
||||
if (gate === 'reject') {
|
||||
cleanup({ release: true });
|
||||
metrics(METRIC_FINGERPRINT_REJECT_TOTAL, 1, { op: resolvedOpKind, gate: 'reject' });
|
||||
return makeErrorEnvelope(
|
||||
request,
|
||||
new FingerprintRequiredError('operation rejected by fingerprint policy'),
|
||||
);
|
||||
}
|
||||
if (gate === 'required') {
|
||||
let verified = false;
|
||||
try {
|
||||
verified = config.isFingerprintVerified !== undefined
|
||||
? Boolean(await config.isFingerprintVerified(from))
|
||||
: false;
|
||||
} catch (err) {
|
||||
cleanup({ release: true });
|
||||
return makeErrorEnvelope(request, err);
|
||||
}
|
||||
if (!verified) {
|
||||
cleanup({ release: true });
|
||||
metrics(METRIC_FINGERPRINT_REJECT_TOTAL, 1, { op: resolvedOpKind, gate: 'required' });
|
||||
return makeErrorEnvelope(request, new FingerprintRequiredError());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 8. beforeOp
|
||||
try {
|
||||
if (config.beforeOp !== undefined) {
|
||||
await config.beforeOp(ctx as OpContext<unknown>);
|
||||
}
|
||||
} catch (err) {
|
||||
cleanup({ release: true });
|
||||
return makeErrorEnvelope(request, err);
|
||||
}
|
||||
|
||||
// 9. Run handler with timeout race — adapting I/O ops as needed.
|
||||
const timeoutMs = Math.min(
|
||||
request.deadlineMs ?? defaultTimeoutMs,
|
||||
defaultTimeoutMs,
|
||||
);
|
||||
|
||||
let wireResult: unknown;
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
wireResult = await runWithTimeout(
|
||||
() => invokeOpHandler({
|
||||
op,
|
||||
stdHandler: op === 'custom' ? undefined : (config[op] as unknown),
|
||||
customHandler,
|
||||
ctx: ctx as OpContext<unknown>,
|
||||
parsedArgs,
|
||||
sender: from,
|
||||
signal: controller.signal,
|
||||
streamsBridge: config.streamsBridge,
|
||||
ioTimeoutMs,
|
||||
}),
|
||||
timeoutMs,
|
||||
controller,
|
||||
);
|
||||
} catch (err) {
|
||||
const durationMs = Date.now() - startedAt;
|
||||
metrics(METRIC_OP_DURATION_MS, durationMs, { op: resolvedOpKind, result: 'error' });
|
||||
metrics(METRIC_OP_TOTAL, 1, { op: resolvedOpKind, result: 'error' });
|
||||
cleanup({ release: true });
|
||||
if (config.onError !== undefined) {
|
||||
try {
|
||||
config.onError(err, ctx as OpContext<unknown>);
|
||||
} catch (hookErr) {
|
||||
console.error('[FileHandler] onError hook threw:', hookErr);
|
||||
}
|
||||
}
|
||||
return makeErrorEnvelope(request, err);
|
||||
}
|
||||
|
||||
// 10. Custom-op response validation against the registered schema.
|
||||
if (op === 'custom' && customHandler !== undefined) {
|
||||
const innerParse = customHandler.response.safeParse(wireResult);
|
||||
if (!innerParse.success) {
|
||||
cleanup({ release: true });
|
||||
return makeErrorEnvelope(
|
||||
request,
|
||||
new CustomOpRejectedError(
|
||||
`custom-op response shape rejected by registered schema: ${innerParse.error.issues[0]?.message ?? 'unknown'}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
wireResult = { result: innerParse.data };
|
||||
}
|
||||
|
||||
// 11. Defensive response validation against the wire schema.
|
||||
const resultParse = resultSchema.safeParse(wireResult);
|
||||
if (!resultParse.success) {
|
||||
cleanup({ release: true });
|
||||
return makeErrorEnvelope(
|
||||
request,
|
||||
new InvalidArgsError(
|
||||
`handler for ${resolvedOpKind} returned invalid response shape`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
// 12. afterOp
|
||||
try {
|
||||
if (config.afterOp !== undefined) {
|
||||
await config.afterOp(ctx as OpContext<unknown>, resultParse.data);
|
||||
}
|
||||
} catch (err) {
|
||||
cleanup({ release: true });
|
||||
return makeErrorEnvelope(request, err);
|
||||
}
|
||||
|
||||
// 13. Commit idempotency + emit metrics
|
||||
commitIdem?.(resultParse.data);
|
||||
cleanup({ release: false });
|
||||
|
||||
const durationMs = Date.now() - startedAt;
|
||||
metrics(METRIC_OP_DURATION_MS, durationMs, { op: resolvedOpKind, result: 'ok' });
|
||||
metrics(METRIC_OP_TOTAL, 1, { op: resolvedOpKind, result: 'ok' });
|
||||
if (estimatedBytes > 0) {
|
||||
// Inbound bytes (write) vs outbound (read) — both reuse the same
|
||||
// pre-call `estimatedBytes`, since post-execution reconciliation
|
||||
// would require deeper plumbing.
|
||||
const direction = op === 'write' ? METRIC_BYTES_IN : op === 'read' ? METRIC_BYTES_OUT : null;
|
||||
if (direction !== null) {
|
||||
metrics(direction, estimatedBytes, { op: resolvedOpKind });
|
||||
}
|
||||
}
|
||||
|
||||
return makeResponseEnvelope(request, resultParse.data);
|
||||
|
||||
function cleanup(opts: { release: boolean }): void {
|
||||
inflightCancellers.delete(request.id);
|
||||
if (opts.release) {
|
||||
abandonIdem?.();
|
||||
rateLimiter.release(from, opCostKey, estimatedBytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function handleCancel(_from: string, cancel: RpcCancel): void {
|
||||
const controller = inflightCancellers.get(cancel.id);
|
||||
if (controller !== undefined) {
|
||||
controller.abort(new CancelledError(cancel.reason ?? 'cancelled by sender'));
|
||||
inflightCancellers.delete(cancel.id);
|
||||
}
|
||||
}
|
||||
|
||||
function destroy(): void {
|
||||
for (const c of inflightCancellers.values()) {
|
||||
c.abort(new CancelledError('handler destroyed'));
|
||||
}
|
||||
inflightCancellers.clear();
|
||||
}
|
||||
|
||||
return Object.assign({ handleRequest, handleCancel, destroy }, {
|
||||
[INTERNAL_SYMBOL]: { idempotency, rateLimiter },
|
||||
});
|
||||
}
|
||||
|
||||
export const INTERNAL_SYMBOL = Symbol.for('@shade/files/internal');
|
||||
|
||||
// ─── Op invoker (handles I/O adapters) ───────────────────────
|
||||
|
||||
interface InvokeArgs {
|
||||
op: StandardOp | 'custom';
|
||||
stdHandler: unknown;
|
||||
customHandler: CustomOpRegistrations[string] | undefined;
|
||||
ctx: OpContext<unknown>;
|
||||
parsedArgs: unknown;
|
||||
sender: string;
|
||||
signal: AbortSignal;
|
||||
streamsBridge: ServerStreamsBridge | undefined;
|
||||
ioTimeoutMs: number;
|
||||
}
|
||||
|
||||
async function invokeOpHandler(args: InvokeArgs): Promise<unknown> {
|
||||
const { op, stdHandler, customHandler, ctx, parsedArgs, sender, signal, streamsBridge, ioTimeoutMs } = args;
|
||||
const adapterDeps = { streamsBridge, sender, signal, ioTimeoutMs };
|
||||
|
||||
switch (op) {
|
||||
case 'write': {
|
||||
const wireArgs = parsedArgs as WriteArgs;
|
||||
const { userArgs, awaitTransferDone } = await adaptWriteArgs(wireArgs, adapterDeps);
|
||||
const userCtx = { ...ctx, args: userArgs } as OpContext<UserWriteArgs>;
|
||||
const userResult = await (stdHandler as (c: OpContext<UserWriteArgs>) => Promise<WriteResult>)(userCtx);
|
||||
await awaitTransferDone();
|
||||
return userResult;
|
||||
}
|
||||
|
||||
case 'read': {
|
||||
const readArgs = parsedArgs as ReadArgs;
|
||||
const userResult = await (stdHandler as (c: OpContext<ReadArgs>) => Promise<UserReadResult>)(ctx as OpContext<ReadArgs>);
|
||||
return await adaptReadResult(userResult, readArgs, adapterDeps);
|
||||
}
|
||||
|
||||
case 'getThumbnail': {
|
||||
const userResult = await (stdHandler as (c: OpContext<GetThumbnailArgs>) => Promise<UserThumbnailResult>)(ctx as OpContext<GetThumbnailArgs>);
|
||||
return adaptThumbnailResult(userResult);
|
||||
}
|
||||
|
||||
case 'custom': {
|
||||
if (customHandler === undefined) {
|
||||
throw new NotImplementedError('custom op without registration');
|
||||
}
|
||||
const customArgs = parsedArgs as CustomArgs;
|
||||
const innerCtx = { ...ctx, args: customArgs } as OpContext<{ name: string; payload: unknown }>;
|
||||
// Pass the validated inner payload as the first arg, the OpContext as the second.
|
||||
return await customHandler.handler(
|
||||
customArgs.payload,
|
||||
innerCtx as OpContext<{ name: string; payload: unknown }>,
|
||||
);
|
||||
}
|
||||
|
||||
default:
|
||||
// Pass-through for list/stat/mkdir/delete/move.
|
||||
return await (stdHandler as (c: OpContext<unknown>) => Promise<unknown>)(ctx);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Helpers ─────────────────────────────────────────────────
|
||||
|
||||
function makeResponseEnvelope(req: RpcRequest, result: unknown): RpcResponse {
|
||||
return {
|
||||
kind: responseKindOf(req.kind),
|
||||
id: req.id,
|
||||
result,
|
||||
};
|
||||
}
|
||||
|
||||
function makeErrorEnvelope(req: RpcRequest, err: unknown): RpcError {
|
||||
return {
|
||||
kind: 'shade.fs.error/v1',
|
||||
id: req.id,
|
||||
error: payloadFromError(err),
|
||||
};
|
||||
}
|
||||
|
||||
function estimateBytes(op: StandardOp | 'custom', args: unknown): number {
|
||||
if (op === 'write') {
|
||||
const w = args as { kind: 'inline' | 'streams'; bytesB64?: string; size?: number };
|
||||
if (w.kind === 'inline' && typeof w.bytesB64 === 'string') {
|
||||
return Math.floor((w.bytesB64.length * 3) / 4);
|
||||
}
|
||||
return w.size ?? 0;
|
||||
}
|
||||
if (op === 'read') {
|
||||
const r = args as { range?: { start: number; end: number } };
|
||||
if (r.range !== undefined) return r.range.end - r.range.start;
|
||||
return 0;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
async function runWithTimeout<T>(
|
||||
fn: () => Promise<T>,
|
||||
timeoutMs: number,
|
||||
controller: AbortController,
|
||||
): Promise<T> {
|
||||
let timer: ReturnType<typeof setTimeout> | null = null;
|
||||
const timeout = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => {
|
||||
controller.abort(new Error('timeout'));
|
||||
reject(new (FileError as unknown as { new (p: { code: string; message: string }): FileError })({
|
||||
code: 'OPERATION_TIMEOUT',
|
||||
message: `operation timed out after ${timeoutMs}ms`,
|
||||
}));
|
||||
}, timeoutMs);
|
||||
});
|
||||
try {
|
||||
return await Promise.race([fn(), timeout]);
|
||||
} finally {
|
||||
if (timer !== null) clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
160
packages/shade-files/src/server/idempotency-cache.ts
Normal file
160
packages/shade-files/src/server/idempotency-cache.ts
Normal file
@@ -0,0 +1,160 @@
|
||||
import { hashArgs, bytesToHex } from '../protocol/canonical.js';
|
||||
import { IdempotencyConflictError } from '../schemas/errors.js';
|
||||
|
||||
interface Entry {
|
||||
argsHash: string;
|
||||
/** `undefined` while inflight; resolved value once handler completes. */
|
||||
status: 'inflight' | 'done';
|
||||
promise?: Promise<unknown>;
|
||||
response?: unknown;
|
||||
insertedAt: number;
|
||||
lastAccessAt: number;
|
||||
}
|
||||
|
||||
export interface IdempotencyCacheOptions {
|
||||
/** Time in ms after which an entry is eligible for eviction. Default 1h. */
|
||||
ttlMs?: number;
|
||||
/** Per-sender hard cap on cached entries; oldest-by-insert evicted first. */
|
||||
maxEntriesPerSender?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-sender LRU + TTL cache for idempotent RPC calls.
|
||||
*
|
||||
* - On lookup, returns cached response if `argsHash` matches the original
|
||||
* call. Throws `IdempotencyConflictError` on argsHash mismatch.
|
||||
* - On concurrent retry of the same key (still inflight), the second
|
||||
* caller awaits the first's promise — no double-execution.
|
||||
* - LRU eviction caps memory at `maxEntriesPerSender` per sender.
|
||||
*
|
||||
* Keyed on `(senderAddress, idempotencyKey)`. Internally a 2-level map.
|
||||
*/
|
||||
export class IdempotencyCache {
|
||||
private readonly ttlMs: number;
|
||||
private readonly maxPerSender: number;
|
||||
private readonly bySender = new Map<string, Map<string, Entry>>();
|
||||
|
||||
constructor(opts: IdempotencyCacheOptions = {}) {
|
||||
this.ttlMs = opts.ttlMs ?? 60 * 60 * 1000;
|
||||
this.maxPerSender = opts.maxEntriesPerSender ?? 10_000;
|
||||
}
|
||||
|
||||
/**
|
||||
* Begin (or rejoin) an idempotent operation. Returns:
|
||||
* - `{ status: 'fresh', commit }` — handler should run, then call `commit(response)`.
|
||||
* - `{ status: 'wait', promise }` — concurrent retry; await the in-flight promise.
|
||||
* - `{ status: 'replay', response }` — completed earlier; return cached.
|
||||
*
|
||||
* Mismatched argsHash throws `IdempotencyConflictError` immediately.
|
||||
*/
|
||||
begin(
|
||||
sender: string,
|
||||
key: string,
|
||||
args: unknown,
|
||||
):
|
||||
| { status: 'fresh'; commit: (response: unknown) => void; abandon: () => void }
|
||||
| { status: 'wait'; promise: Promise<unknown> }
|
||||
| { status: 'replay'; response: unknown } {
|
||||
const argsHash = bytesToHex(hashArgs(args));
|
||||
const senderMap = this.getOrCreate(sender);
|
||||
this.evictExpired(senderMap);
|
||||
const existing = senderMap.get(key);
|
||||
if (existing !== undefined) {
|
||||
if (existing.argsHash !== argsHash) {
|
||||
throw new IdempotencyConflictError();
|
||||
}
|
||||
existing.lastAccessAt = Date.now();
|
||||
if (existing.status === 'done') {
|
||||
return { status: 'replay', response: existing.response };
|
||||
}
|
||||
// Inflight — return the same promise so concurrent retries de-dupe.
|
||||
return { status: 'wait', promise: existing.promise! };
|
||||
}
|
||||
|
||||
let resolveInflight: (value: unknown) => void = () => {};
|
||||
let rejectInflight: (err: unknown) => void = () => {};
|
||||
const inflightPromise = new Promise<unknown>((resolve, reject) => {
|
||||
resolveInflight = resolve;
|
||||
rejectInflight = reject;
|
||||
});
|
||||
// Suppress unhandled-rejection warnings when no concurrent retry is
|
||||
// awaiting this promise. Real waiters attach their own catch via
|
||||
// `await begin().promise` and still see the rejection.
|
||||
inflightPromise.catch(() => {
|
||||
/* swallow — see comment above */
|
||||
});
|
||||
const entry: Entry = {
|
||||
argsHash,
|
||||
status: 'inflight',
|
||||
promise: inflightPromise,
|
||||
insertedAt: Date.now(),
|
||||
lastAccessAt: Date.now(),
|
||||
};
|
||||
senderMap.set(key, entry);
|
||||
this.evictOverflow(senderMap);
|
||||
|
||||
const commit = (response: unknown): void => {
|
||||
entry.status = 'done';
|
||||
entry.response = response;
|
||||
entry.lastAccessAt = Date.now();
|
||||
resolveInflight(response);
|
||||
};
|
||||
const abandon = (): void => {
|
||||
// Failed/cancelled call — remove from cache so retries can proceed.
|
||||
senderMap.delete(key);
|
||||
rejectInflight(new Error('idempotency abandoned'));
|
||||
};
|
||||
|
||||
return { status: 'fresh', commit, abandon };
|
||||
}
|
||||
|
||||
/** Manual prune; called by the periodic retention job. */
|
||||
prune(now: number = Date.now()): number {
|
||||
let removed = 0;
|
||||
for (const [sender, senderMap] of this.bySender) {
|
||||
for (const [key, entry] of senderMap) {
|
||||
if (now - entry.insertedAt > this.ttlMs) {
|
||||
senderMap.delete(key);
|
||||
removed++;
|
||||
}
|
||||
}
|
||||
if (senderMap.size === 0) this.bySender.delete(sender);
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
/** Total cached entries across all senders (mainly for tests/metrics). */
|
||||
size(): number {
|
||||
let total = 0;
|
||||
for (const m of this.bySender.values()) total += m.size;
|
||||
return total;
|
||||
}
|
||||
|
||||
private getOrCreate(sender: string): Map<string, Entry> {
|
||||
let m = this.bySender.get(sender);
|
||||
if (m === undefined) {
|
||||
m = new Map();
|
||||
this.bySender.set(sender, m);
|
||||
}
|
||||
return m;
|
||||
}
|
||||
|
||||
private evictExpired(senderMap: Map<string, Entry>): void {
|
||||
const now = Date.now();
|
||||
for (const [key, entry] of senderMap) {
|
||||
if (now - entry.insertedAt > this.ttlMs) senderMap.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
private evictOverflow(senderMap: Map<string, Entry>): void {
|
||||
if (senderMap.size <= this.maxPerSender) return;
|
||||
// Map iteration order = insertion order; first key is oldest.
|
||||
const overflow = senderMap.size - this.maxPerSender;
|
||||
let i = 0;
|
||||
for (const key of senderMap.keys()) {
|
||||
if (i >= overflow) break;
|
||||
senderMap.delete(key);
|
||||
i++;
|
||||
}
|
||||
}
|
||||
}
|
||||
229
packages/shade-files/src/server/io-adapters.ts
Normal file
229
packages/shade-files/src/server/io-adapters.ts
Normal file
@@ -0,0 +1,229 @@
|
||||
/**
|
||||
* Adapt between wire-shaped I/O ops and user-handler-shaped I/O ops.
|
||||
*
|
||||
* client → server (write): WriteArgs (wire) → UserWriteArgs (handler)
|
||||
* server → client (read): UserReadResult (handler) → ReadResult (wire)
|
||||
* server → client (thumb): UserThumbnailResult → GetThumbnailResult
|
||||
*
|
||||
* The streams-bridge is consulted whenever we're crossing the > 256 KiB
|
||||
* boundary: outbound reads kick a new transfer; inbound writes await one.
|
||||
*/
|
||||
|
||||
import { sha256 } from '@noble/hashes/sha2.js';
|
||||
import type {
|
||||
GetThumbnailResult,
|
||||
ReadArgs,
|
||||
ReadResult,
|
||||
WriteArgs,
|
||||
} from '../schemas/ops.js';
|
||||
import { base64ToBytes, bytesToBase64, bytesToHex } from '../protocol/canonical.js';
|
||||
import { INLINE_THRESHOLD } from '../client/inline-threshold.js';
|
||||
import {
|
||||
ConflictError,
|
||||
InternalFileError,
|
||||
InvalidArgsError,
|
||||
} from '../schemas/errors.js';
|
||||
import type {
|
||||
UserReadResult,
|
||||
UserThumbnailResult,
|
||||
UserWriteArgs,
|
||||
} from './io-types.js';
|
||||
import type { ServerStreamsBridge } from './streams-bridge.js';
|
||||
|
||||
export interface IoAdapterDeps {
|
||||
streamsBridge: ServerStreamsBridge | undefined;
|
||||
sender: string;
|
||||
signal: AbortSignal;
|
||||
/** Hard deadline for streams-bridge awaits / outbound transfers. */
|
||||
ioTimeoutMs: number;
|
||||
}
|
||||
|
||||
// ─── write: wire args → user args ────────────────────────────
|
||||
|
||||
/**
|
||||
* Convert validated `WriteArgs` (wire shape) into `UserWriteArgs`. For
|
||||
* streams writes, this awaits the matching inbound transfer via the
|
||||
* streams-bridge and synthesizes a `ReadableStream` for the user handler.
|
||||
*/
|
||||
export async function adaptWriteArgs(
|
||||
wireArgs: WriteArgs,
|
||||
deps: IoAdapterDeps,
|
||||
): Promise<{
|
||||
userArgs: UserWriteArgs;
|
||||
/** Resolves once the inbound transfer (if any) has completed. */
|
||||
awaitTransferDone: () => Promise<void>;
|
||||
}> {
|
||||
if (wireArgs.kind === 'inline') {
|
||||
const bytes = base64ToBytes(wireArgs.bytesB64);
|
||||
if (bytes.byteLength > INLINE_THRESHOLD) {
|
||||
throw new InvalidArgsError(
|
||||
`inline write exceeds ${INLINE_THRESHOLD}-byte threshold (got ${bytes.byteLength})`,
|
||||
'bytesB64',
|
||||
);
|
||||
}
|
||||
const hashHex = bytesToHex(sha256(bytes));
|
||||
const userArgs: UserWriteArgs = {
|
||||
path: wireArgs.path,
|
||||
overwrite: wireArgs.overwrite,
|
||||
...(wireArgs.contentType !== undefined ? { contentType: wireArgs.contentType } : {}),
|
||||
content: {
|
||||
kind: 'inline',
|
||||
bytes,
|
||||
size: bytes.byteLength,
|
||||
sha256: hashHex,
|
||||
},
|
||||
};
|
||||
return { userArgs, awaitTransferDone: async () => undefined };
|
||||
}
|
||||
|
||||
// Streams write — must have a streams-bridge configured.
|
||||
if (deps.streamsBridge === undefined) {
|
||||
throw new InternalFileError(
|
||||
'streams-bridge not configured: cannot accept streamed write',
|
||||
);
|
||||
}
|
||||
const parked = await deps.streamsBridge.awaitWrite(wireArgs.writeId, {
|
||||
expectedFrom: deps.sender,
|
||||
signal: deps.signal,
|
||||
timeoutMs: deps.ioTimeoutMs,
|
||||
});
|
||||
|
||||
// sha256 from the inbound transfer's done() — user handler can `await`
|
||||
// it immediately after draining the readable.
|
||||
const sha256Promise = parked.done.then((r) => r.sha256);
|
||||
sha256Promise.catch(() => {
|
||||
/* swallow until consumer awaits */
|
||||
});
|
||||
|
||||
// Cancellation: if the user handler aborts (or the dispatcher times out),
|
||||
// also tear down the inbound transfer.
|
||||
const onAbort = (): void => {
|
||||
void parked.handle.abort('user-cancel').catch(() => undefined);
|
||||
};
|
||||
if (deps.signal.aborted) onAbort();
|
||||
else deps.signal.addEventListener('abort', onAbort, { once: true });
|
||||
|
||||
// The dispatcher calls this AFTER the user handler resolves to ensure
|
||||
// the transfer is fully drained (so any wire-level integrity errors
|
||||
// surface here before we send the RPC ack to the client).
|
||||
const awaitTransferDone = async (): Promise<void> => {
|
||||
try {
|
||||
await parked.done;
|
||||
} finally {
|
||||
deps.signal.removeEventListener('abort', onAbort);
|
||||
}
|
||||
};
|
||||
|
||||
const userArgs: UserWriteArgs = {
|
||||
path: wireArgs.path,
|
||||
overwrite: wireArgs.overwrite,
|
||||
...(wireArgs.contentType !== undefined ? { contentType: wireArgs.contentType } : {}),
|
||||
content: {
|
||||
kind: 'streams',
|
||||
stream: parked.readable,
|
||||
size: wireArgs.size,
|
||||
sha256: sha256Promise,
|
||||
},
|
||||
};
|
||||
return { userArgs, awaitTransferDone };
|
||||
}
|
||||
|
||||
// ─── read: user result → wire result ─────────────────────────
|
||||
|
||||
/**
|
||||
* Convert a user-supplied `UserReadResult` into a wire `ReadResult`. For
|
||||
* streams results, this kicks an outbound `@shade/transfer` and embeds the
|
||||
* resulting `streamId` in the response so the client-side bridge can
|
||||
* subscribe.
|
||||
*/
|
||||
export async function adaptReadResult(
|
||||
userResult: UserReadResult,
|
||||
args: ReadArgs,
|
||||
deps: IoAdapterDeps,
|
||||
): Promise<ReadResult> {
|
||||
if (userResult.kind === 'inline') {
|
||||
if (userResult.bytes.byteLength > INLINE_THRESHOLD) {
|
||||
throw new InternalFileError(
|
||||
`inline read result exceeds ${INLINE_THRESHOLD} bytes (got ${userResult.bytes.byteLength})`,
|
||||
);
|
||||
}
|
||||
const sha256Hex = userResult.sha256 ?? bytesToHex(sha256(userResult.bytes));
|
||||
return {
|
||||
kind: 'inline',
|
||||
bytesB64: bytesToBase64(userResult.bytes),
|
||||
size: userResult.bytes.byteLength,
|
||||
sha256: sha256Hex,
|
||||
...(userResult.contentType !== undefined ? { contentType: userResult.contentType } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
if (deps.streamsBridge === undefined) {
|
||||
throw new InternalFileError(
|
||||
'streams-bridge not configured: cannot ship streamed read result',
|
||||
);
|
||||
}
|
||||
const { readStreamId } = await deps.streamsBridge.initiateRead({
|
||||
peer: deps.sender,
|
||||
stream: userResult.stream,
|
||||
size: userResult.size,
|
||||
...(userResult.contentType !== undefined ? { contentType: userResult.contentType } : {}),
|
||||
name: args.path,
|
||||
signal: deps.signal,
|
||||
});
|
||||
return {
|
||||
kind: 'streams',
|
||||
streamId: readStreamId,
|
||||
size: userResult.size,
|
||||
sha256: userResult.sha256,
|
||||
...(userResult.contentType !== undefined ? { contentType: userResult.contentType } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
// ─── getThumbnail: user result → wire result ─────────────────
|
||||
|
||||
const FORMAT_MAGIC: Record<string, Uint8Array> = {
|
||||
png: new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
|
||||
jpeg: new Uint8Array([0xff, 0xd8, 0xff]),
|
||||
webp: new Uint8Array([0x52, 0x49, 0x46, 0x46]), // 'RIFF', then size, then 'WEBP'
|
||||
};
|
||||
const WEBP_HEAD = new Uint8Array([0x57, 0x45, 0x42, 0x50]); // 'WEBP' at offset 8
|
||||
|
||||
/**
|
||||
* Validate a user thumbnail result for format magic-bytes match. The user
|
||||
* MAY produce any image format internally, but this guards against
|
||||
* misclassification (e.g. claiming PNG with JPEG bytes), which can confuse
|
||||
* downstream rendering and security scanners.
|
||||
*/
|
||||
function checkFormatMagic(bytes: Uint8Array, format: 'png' | 'webp' | 'jpeg'): boolean {
|
||||
const magic = FORMAT_MAGIC[format];
|
||||
if (magic === undefined) return false;
|
||||
if (bytes.byteLength < magic.byteLength) return false;
|
||||
for (let i = 0; i < magic.byteLength; i++) {
|
||||
if (bytes[i] !== magic[i]) return false;
|
||||
}
|
||||
if (format === 'webp') {
|
||||
if (bytes.byteLength < 12) return false;
|
||||
for (let i = 0; i < 4; i++) {
|
||||
if (bytes[8 + i] !== WEBP_HEAD[i]) return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
export function adaptThumbnailResult(
|
||||
userResult: UserThumbnailResult,
|
||||
): GetThumbnailResult {
|
||||
if (!checkFormatMagic(userResult.bytes, userResult.format)) {
|
||||
throw new ConflictError(
|
||||
`thumbnail bytes do not match declared format=${userResult.format}`,
|
||||
);
|
||||
}
|
||||
const hash = userResult.sha256 ?? bytesToHex(sha256(userResult.bytes));
|
||||
return {
|
||||
bytesB64: bytesToBase64(userResult.bytes),
|
||||
format: userResult.format,
|
||||
width: userResult.width,
|
||||
height: userResult.height,
|
||||
sha256: hash,
|
||||
};
|
||||
}
|
||||
102
packages/shade-files/src/server/io-types.ts
Normal file
102
packages/shade-files/src/server/io-types.ts
Normal file
@@ -0,0 +1,102 @@
|
||||
/**
|
||||
* Server-facing types for the I/O ops (`read`, `write`, `getThumbnail`).
|
||||
*
|
||||
* These differ from the wire schemas in `schemas/ops.ts` so user-supplied
|
||||
* handlers see clean, typed values (`Uint8Array`, `ReadableStream`) instead
|
||||
* of base64 strings and opaque transfer ids. The dispatcher in
|
||||
* `server/handler.ts` adapts between the two.
|
||||
*/
|
||||
|
||||
import type { ThumbnailSize, WriteResult } from '../schemas/ops.js';
|
||||
|
||||
// ─── read handler types ──────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Value returned by a user-supplied `read` handler.
|
||||
*
|
||||
* - `inline`: the bytes are returned in-band; the dispatcher base64-encodes
|
||||
* them and computes the sha256 if the user did not provide one. Inline
|
||||
* reads MUST be ≤ 256 KiB plaintext.
|
||||
* - `streams`: the user provides a `ReadableStream`, the declared `size`,
|
||||
* and a precomputed `sha256` (e.g. cached on the file's metadata row).
|
||||
* The dispatcher initiates an outbound `@shade/transfer` transfer and
|
||||
* echoes the transfer's `streamId` back to the client over RPC so the
|
||||
* client-side bridge can subscribe.
|
||||
*/
|
||||
export type UserReadResult =
|
||||
| UserReadResultInline
|
||||
| UserReadResultStreams;
|
||||
|
||||
export interface UserReadResultInline {
|
||||
kind: 'inline';
|
||||
bytes: Uint8Array;
|
||||
/** Optional — dispatcher computes if absent. */
|
||||
sha256?: string;
|
||||
contentType?: string;
|
||||
}
|
||||
|
||||
export interface UserReadResultStreams {
|
||||
kind: 'streams';
|
||||
stream: ReadableStream<Uint8Array>;
|
||||
size: number;
|
||||
/** REQUIRED — the dispatcher cannot synchronously hash a stream. */
|
||||
sha256: string;
|
||||
contentType?: string;
|
||||
}
|
||||
|
||||
// ─── write handler types ─────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Value passed to a user-supplied `write` handler.
|
||||
*
|
||||
* The dispatcher decodes the wire `WriteArgs` and (for `streams`) parks
|
||||
* the inbound transfer via the streams-bridge, then synthesizes this
|
||||
* cleaner shape for the user.
|
||||
*/
|
||||
export interface UserWriteArgs {
|
||||
path: string;
|
||||
contentType?: string;
|
||||
overwrite: boolean;
|
||||
content: UserWriteContent;
|
||||
}
|
||||
|
||||
export type UserWriteContent =
|
||||
| UserWriteContentInline
|
||||
| UserWriteContentStreams;
|
||||
|
||||
export interface UserWriteContentInline {
|
||||
kind: 'inline';
|
||||
bytes: Uint8Array;
|
||||
size: number;
|
||||
/** sha256 of `bytes`, computed by the dispatcher. */
|
||||
sha256: string;
|
||||
}
|
||||
|
||||
export interface UserWriteContentStreams {
|
||||
kind: 'streams';
|
||||
/** Plaintext stream. Closes when the inbound `@shade/transfer` completes. */
|
||||
stream: ReadableStream<Uint8Array>;
|
||||
/** Declared plaintext size from the client's RPC args. */
|
||||
size: number;
|
||||
/**
|
||||
* Promise that resolves with the transfer-verified sha256 once the entire
|
||||
* stream has been received. Use this to attach a content hash to the
|
||||
* user's storage record.
|
||||
*/
|
||||
sha256: Promise<string>;
|
||||
}
|
||||
|
||||
// ─── getThumbnail handler types ──────────────────────────────
|
||||
|
||||
/** Value returned by a user-supplied `getThumbnail` handler. */
|
||||
export interface UserThumbnailResult {
|
||||
bytes: Uint8Array;
|
||||
format: 'png' | 'webp' | 'jpeg';
|
||||
width: number;
|
||||
height: number;
|
||||
/** Optional — dispatcher computes if absent. */
|
||||
sha256?: string;
|
||||
}
|
||||
|
||||
// ─── Re-exports ──────────────────────────────────────────────
|
||||
export type { ThumbnailSize, WriteResult };
|
||||
25
packages/shade-files/src/server/metrics.ts
Normal file
25
packages/shade-files/src/server/metrics.ts
Normal file
@@ -0,0 +1,25 @@
|
||||
/**
|
||||
* Standard metric names emitted by the dispatcher.
|
||||
*
|
||||
* The user-supplied `onMetric(name, value, tags)` callback is vendor-neutral —
|
||||
* pipe to Prometheus, OpenTelemetry, statsd, or just append to a log file.
|
||||
*/
|
||||
|
||||
export const METRIC_OP_DURATION_MS = 'shade_files_op_duration_ms';
|
||||
export const METRIC_OP_TOTAL = 'shade_files_op_total';
|
||||
export const METRIC_BYTES_IN = 'shade_files_bytes_in';
|
||||
export const METRIC_BYTES_OUT = 'shade_files_bytes_out';
|
||||
export const METRIC_IDEMPOTENCY_HIT_TOTAL = 'shade_files_idempotency_hit_total';
|
||||
export const METRIC_IDEMPOTENCY_CONFLICT_TOTAL = 'shade_files_idempotency_conflict_total';
|
||||
export const METRIC_RATE_LIMIT_REJECT_TOTAL = 'shade_files_rate_limit_reject_total';
|
||||
export const METRIC_FINGERPRINT_REJECT_TOTAL = 'shade_files_fingerprint_reject_total';
|
||||
export const METRIC_SIGNATURE_REJECT_TOTAL = 'shade_files_signature_reject_total';
|
||||
|
||||
/** Tags attached to every metric event. */
|
||||
export type MetricTags = Record<string, string | number | boolean>;
|
||||
|
||||
/** User-supplied metric sink. Synchronous to keep hot-path fast. */
|
||||
export type MetricSink = (name: string, value: number, tags: MetricTags) => void;
|
||||
|
||||
/** No-op sink used when the user doesn't supply one. */
|
||||
export const NOOP_METRIC_SINK: MetricSink = () => undefined;
|
||||
95
packages/shade-files/src/server/path-policy.ts
Normal file
95
packages/shade-files/src/server/path-policy.ts
Normal file
@@ -0,0 +1,95 @@
|
||||
import {
|
||||
decodePercentEscapes,
|
||||
isPathInside,
|
||||
posixNormalize,
|
||||
} from '../utils/path.js';
|
||||
|
||||
export interface PathPolicy {
|
||||
/** Chroot-style absolute root. All paths must lie inside this directory. */
|
||||
rootScope?: string;
|
||||
/** Reject `..` segments after normalization. Default true. */
|
||||
forbidTraversal?: boolean;
|
||||
/** Hard cap on raw-input length (post-decode). Default 4096. */
|
||||
maxLength?: number;
|
||||
/** Allow symlink components. Default false (consumer enforces realpath if true). */
|
||||
allowSymlinks?: boolean;
|
||||
/** App-specific extra check. Returns 'allow' | 'reject'. */
|
||||
extra?: (normalizedPath: string) => 'allow' | 'reject';
|
||||
}
|
||||
|
||||
export type PathValidationResult =
|
||||
| { ok: true; normalized: string }
|
||||
| { ok: false; reason: string };
|
||||
|
||||
const DEFAULT_MAX_LENGTH = 4096;
|
||||
/** Bytes that must never appear in any path. */
|
||||
const FORBIDDEN_RE = /[\x00-\x08\x0a-\x1f\x7f\\]/;
|
||||
|
||||
/**
|
||||
* Validate and normalize a path against the configured policy.
|
||||
*
|
||||
* Performs (in order):
|
||||
* 1. Length check on raw input.
|
||||
* 2. Forbidden-bytes check on raw input (NUL/CR/LF/DEL/backslash).
|
||||
* 3. Percent-decode (defense against `%2e%2e` smuggling).
|
||||
* 4. Forbidden-bytes check on decoded form.
|
||||
* 5. POSIX normalization.
|
||||
* 6. `..` traversal rejection.
|
||||
* 7. Root-scope boundary check.
|
||||
* 8. Consumer-supplied `extra` predicate.
|
||||
*
|
||||
* Returns the normalized path on success. The handler MUST use the
|
||||
* normalized form, not the raw input — the raw could trip TOCTOU on
|
||||
* the caller's filesystem.
|
||||
*/
|
||||
export function validatePath(
|
||||
rawPath: string,
|
||||
policy: PathPolicy = {},
|
||||
): PathValidationResult {
|
||||
const maxLength = policy.maxLength ?? DEFAULT_MAX_LENGTH;
|
||||
const forbidTraversal = policy.forbidTraversal ?? true;
|
||||
|
||||
if (typeof rawPath !== 'string') {
|
||||
return { ok: false, reason: 'path must be a string' };
|
||||
}
|
||||
if (rawPath.length === 0) {
|
||||
return { ok: false, reason: 'path is empty' };
|
||||
}
|
||||
if (rawPath.length > maxLength) {
|
||||
return { ok: false, reason: `path exceeds ${maxLength} characters` };
|
||||
}
|
||||
if (FORBIDDEN_RE.test(rawPath)) {
|
||||
return { ok: false, reason: 'path contains forbidden control or backslash characters' };
|
||||
}
|
||||
|
||||
const decoded = decodePercentEscapes(rawPath);
|
||||
if (FORBIDDEN_RE.test(decoded)) {
|
||||
return { ok: false, reason: 'percent-decoded path contains forbidden characters' };
|
||||
}
|
||||
if (!decoded.startsWith('/')) {
|
||||
return { ok: false, reason: 'path must be absolute' };
|
||||
}
|
||||
|
||||
if (forbidTraversal) {
|
||||
// Detect `..` segments BEFORE normalization (otherwise they'd be silently
|
||||
// collapsed). Handles both raw and decoded forms.
|
||||
if (/(^|\/)\.\.(\/|$)/.test(rawPath) || /(^|\/)\.\.(\/|$)/.test(decoded)) {
|
||||
return { ok: false, reason: 'path contains `..` traversal segment' };
|
||||
}
|
||||
}
|
||||
|
||||
const normalized = posixNormalize(decoded);
|
||||
|
||||
if (policy.rootScope !== undefined) {
|
||||
const root = posixNormalize(policy.rootScope);
|
||||
if (!isPathInside(normalized, root)) {
|
||||
return { ok: false, reason: 'path is outside root scope' };
|
||||
}
|
||||
}
|
||||
|
||||
if (policy.extra !== undefined && policy.extra(normalized) === 'reject') {
|
||||
return { ok: false, reason: 'path rejected by policy.extra' };
|
||||
}
|
||||
|
||||
return { ok: true, normalized };
|
||||
}
|
||||
157
packages/shade-files/src/server/rate-limiter.ts
Normal file
157
packages/shade-files/src/server/rate-limiter.ts
Normal file
@@ -0,0 +1,157 @@
|
||||
import { FsRateLimitError, QuotaExceededError } from '../schemas/errors.js';
|
||||
|
||||
export interface RateLimitConfig {
|
||||
/** Cap on op-tokens per minute per sender. Default 600. */
|
||||
maxOpsPerMinutePerSender?: number;
|
||||
/** Cap on byte-tokens per hour per sender. Default 10 GiB. */
|
||||
maxBytesPerHourPerSender?: number;
|
||||
/** Per-op token cost. Default 1 per op except write=5, delete=3. */
|
||||
opCost?: Partial<Record<string, number>> & { default?: number };
|
||||
}
|
||||
|
||||
interface Bucket {
|
||||
tokens: number;
|
||||
capacity: number;
|
||||
/** Tokens replenished per millisecond. */
|
||||
refillRate: number;
|
||||
lastRefill: number;
|
||||
}
|
||||
|
||||
interface SenderBuckets {
|
||||
ops: Bucket;
|
||||
bytes: Bucket;
|
||||
}
|
||||
|
||||
/**
|
||||
* Two-bucket per-sender rate limiter. Each operation consumes:
|
||||
* - `opCost(op)` from the ops bucket.
|
||||
* - `estimatedBytes` from the bytes bucket (post-call reconciliation possible).
|
||||
*
|
||||
* Rejection throws either `FsRateLimitError` (op-bucket) or `QuotaExceededError`
|
||||
* (bytes-bucket), each with a `retryAfterMs` hint.
|
||||
*
|
||||
* Cancellation MUST call `release()` to return reserved tokens.
|
||||
*/
|
||||
export class RateLimiter {
|
||||
private readonly opsCapacity: number;
|
||||
private readonly opsRefillPerMs: number;
|
||||
private readonly bytesCapacity: number;
|
||||
private readonly bytesRefillPerMs: number;
|
||||
private readonly opCost: Required<NonNullable<RateLimitConfig['opCost']>>;
|
||||
private readonly bySender = new Map<string, SenderBuckets>();
|
||||
|
||||
constructor(config: RateLimitConfig = {}) {
|
||||
this.opsCapacity = config.maxOpsPerMinutePerSender ?? 600;
|
||||
this.opsRefillPerMs = this.opsCapacity / 60_000;
|
||||
this.bytesCapacity = config.maxBytesPerHourPerSender ?? 10 * 1024 * 1024 * 1024;
|
||||
this.bytesRefillPerMs = this.bytesCapacity / 3_600_000;
|
||||
this.opCost = {
|
||||
list: config.opCost?.list ?? 1,
|
||||
stat: config.opCost?.stat ?? 1,
|
||||
mkdir: config.opCost?.mkdir ?? 2,
|
||||
delete: config.opCost?.delete ?? 3,
|
||||
move: config.opCost?.move ?? 2,
|
||||
read: config.opCost?.read ?? 1,
|
||||
write: config.opCost?.write ?? 5,
|
||||
getThumbnail: config.opCost?.getThumbnail ?? 2,
|
||||
custom: config.opCost?.custom ?? 1,
|
||||
default: config.opCost?.default ?? 1,
|
||||
} as Required<NonNullable<RateLimitConfig['opCost']>>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Acquire tokens for an operation. Throws if either bucket is empty;
|
||||
* otherwise atomically deducts from both. `estimatedBytes` may be 0 for
|
||||
* non-I/O ops.
|
||||
*/
|
||||
acquire(sender: string, op: string, estimatedBytes: number = 0): void {
|
||||
const buckets = this.getBuckets(sender);
|
||||
this.refill(buckets, Date.now());
|
||||
const cost = this.costFor(op);
|
||||
|
||||
if (buckets.ops.tokens < cost) {
|
||||
const need = cost - buckets.ops.tokens;
|
||||
const retryMs = Math.ceil(need / this.opsRefillPerMs);
|
||||
throw new FsRateLimitError(`op rate limit exceeded`, retryMs);
|
||||
}
|
||||
if (estimatedBytes > 0 && buckets.bytes.tokens < estimatedBytes) {
|
||||
const need = estimatedBytes - buckets.bytes.tokens;
|
||||
const retryMs = Math.ceil(need / this.bytesRefillPerMs);
|
||||
throw new QuotaExceededError(`byte quota exceeded`, retryMs);
|
||||
}
|
||||
buckets.ops.tokens -= cost;
|
||||
if (estimatedBytes > 0) buckets.bytes.tokens -= estimatedBytes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconcile after the actual byte cost is known. `actualBytes` may be
|
||||
* higher or lower than the estimate; caps to capacity on release.
|
||||
*/
|
||||
reconcile(sender: string, estimatedBytes: number, actualBytes: number): void {
|
||||
if (estimatedBytes === actualBytes) return;
|
||||
const buckets = this.getBuckets(sender);
|
||||
const delta = estimatedBytes - actualBytes;
|
||||
if (delta > 0) {
|
||||
buckets.bytes.tokens = Math.min(buckets.bytes.capacity, buckets.bytes.tokens + delta);
|
||||
} else {
|
||||
buckets.bytes.tokens = Math.max(0, buckets.bytes.tokens + delta);
|
||||
}
|
||||
}
|
||||
|
||||
/** Return tokens to both buckets (cancellation cleanup). */
|
||||
release(sender: string, op: string, bytesReserved: number = 0): void {
|
||||
const buckets = this.bySender.get(sender);
|
||||
if (buckets === undefined) return;
|
||||
const cost = this.costFor(op);
|
||||
buckets.ops.tokens = Math.min(buckets.ops.capacity, buckets.ops.tokens + cost);
|
||||
if (bytesReserved > 0) {
|
||||
buckets.bytes.tokens = Math.min(buckets.bytes.capacity, buckets.bytes.tokens + bytesReserved);
|
||||
}
|
||||
}
|
||||
|
||||
/** Available token snapshot — for tests/metrics. */
|
||||
snapshot(sender: string): { ops: number; bytes: number } | null {
|
||||
const buckets = this.bySender.get(sender);
|
||||
if (buckets === undefined) return null;
|
||||
this.refill(buckets, Date.now());
|
||||
return { ops: buckets.ops.tokens, bytes: buckets.bytes.tokens };
|
||||
}
|
||||
|
||||
private costFor(op: string): number {
|
||||
const explicit = (this.opCost as Record<string, number>)[op];
|
||||
return explicit ?? this.opCost.default;
|
||||
}
|
||||
|
||||
private getBuckets(sender: string): SenderBuckets {
|
||||
let b = this.bySender.get(sender);
|
||||
if (b !== undefined) return b;
|
||||
const now = Date.now();
|
||||
b = {
|
||||
ops: {
|
||||
tokens: this.opsCapacity,
|
||||
capacity: this.opsCapacity,
|
||||
refillRate: this.opsRefillPerMs,
|
||||
lastRefill: now,
|
||||
},
|
||||
bytes: {
|
||||
tokens: this.bytesCapacity,
|
||||
capacity: this.bytesCapacity,
|
||||
refillRate: this.bytesRefillPerMs,
|
||||
lastRefill: now,
|
||||
},
|
||||
};
|
||||
this.bySender.set(sender, b);
|
||||
return b;
|
||||
}
|
||||
|
||||
private refill(buckets: SenderBuckets, now: number): void {
|
||||
const refillBucket = (b: Bucket): void => {
|
||||
const elapsed = now - b.lastRefill;
|
||||
if (elapsed <= 0) return;
|
||||
b.tokens = Math.min(b.capacity, b.tokens + elapsed * b.refillRate);
|
||||
b.lastRefill = now;
|
||||
};
|
||||
refillBucket(buckets.ops);
|
||||
refillBucket(buckets.bytes);
|
||||
}
|
||||
}
|
||||
289
packages/shade-files/src/server/streams-bridge.ts
Normal file
289
packages/shade-files/src/server/streams-bridge.ts
Normal file
@@ -0,0 +1,289 @@
|
||||
/**
|
||||
* Server-side bridge between `@shade/files` content RPC ops and the
|
||||
* `@shade/transfer` engine.
|
||||
*
|
||||
* Two responsibilities:
|
||||
* 1. **Inbound writes (client → server, > 256 KiB).** When a client kicks
|
||||
* `shade.upload(...)` with `userMetadata.shadeFilesWriteId = <id>`, this
|
||||
* bridge intercepts the corresponding `IncomingTransfer` from
|
||||
* `shade.onIncomingTransfer`, **immediately** calls `accept(...)` (the
|
||||
* transfer engine requires synchronous accept — it rejects chunks that
|
||||
* arrive before accept), pipes the plaintext into a TransformStream,
|
||||
* and parks the readable side. The `write` RPC handler then awaits
|
||||
* `awaitWrite(writeId, ...)` to pick up the readable, hand it to the
|
||||
* user handler, and observe the transfer's done() result.
|
||||
*
|
||||
* 2. **Outbound reads (server → client, > 256 KiB).** When the user-supplied
|
||||
* `read` handler returns `{ kind: 'streams', stream, size, sha256 }`,
|
||||
* this bridge calls `shade.upload(...)` with
|
||||
* `userMetadata.shadeFilesReadStreamId = <id>`. The id is then echoed
|
||||
* back to the client in the `read` RPC response so the client-side
|
||||
* bridge can subscribe to the matching incoming transfer.
|
||||
*/
|
||||
import type { IncomingTransfer, TransferHandle, TransferProgress } from '@shade/transfer';
|
||||
import { generateRequestId } from '../protocol/correlate.js';
|
||||
import { OperationTimeoutError } from '../schemas/errors.js';
|
||||
|
||||
/** Metadata key carried on inbound (client → server) write streams. */
|
||||
export const META_KEY_WRITE_ID = 'shadeFilesWriteId';
|
||||
/** Metadata key carried on outbound (server → client) read streams. */
|
||||
export const META_KEY_READ_STREAM_ID = 'shadeFilesReadStreamId';
|
||||
|
||||
/**
|
||||
* Subset of `Shade` we depend on. Lets us unit-test without a real Shade
|
||||
* runtime and keeps the bridge framework-agnostic.
|
||||
*/
|
||||
export interface StreamsBridgeShade {
|
||||
upload(opts: import('@shade/transfer').TransferOptions): Promise<TransferHandle>;
|
||||
onIncomingTransfer(
|
||||
handler: (incoming: IncomingTransfer) => void | Promise<void>,
|
||||
): Promise<() => void>;
|
||||
}
|
||||
|
||||
export interface ParkedWrite {
|
||||
/** Sender address from the transfer engine. */
|
||||
from: string;
|
||||
/** Plaintext stream. The bridge already accepted the transfer; just consume. */
|
||||
readable: ReadableStream<Uint8Array>;
|
||||
/** Resolves when the transfer fully completes (sha256 available). */
|
||||
done: Promise<{ sha256: string; bytesSent: number }>;
|
||||
/** Underlying transfer handle — for abort propagation. */
|
||||
handle: TransferHandle;
|
||||
/** Unix epoch ms when the transfer arrived. Used for stale-eviction. */
|
||||
arrivedAt: number;
|
||||
}
|
||||
|
||||
export interface AwaitWriteOptions {
|
||||
/** Sender address — must match `parked.from` for delivery. */
|
||||
expectedFrom: string;
|
||||
/** AbortSignal from the OpContext (cancellation/timeout). */
|
||||
signal: AbortSignal;
|
||||
/** Hard deadline (ms from now). Default 60_000. */
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
export interface ServerStreamsBridge {
|
||||
/**
|
||||
* Wait for an inbound transfer carrying `userMetadata.shadeFilesWriteId
|
||||
* === writeId` from `expectedFrom`. Resolves with the parked entry whose
|
||||
* `readable` can be consumed by the user handler.
|
||||
*
|
||||
* If the transfer has already arrived (race: stream-init beat the RPC),
|
||||
* resolves immediately. Otherwise awaits up to `timeoutMs`.
|
||||
*/
|
||||
awaitWrite(writeId: string, opts: AwaitWriteOptions): Promise<ParkedWrite>;
|
||||
|
||||
/**
|
||||
* Initiate an outbound transfer to `peer` carrying the given stream.
|
||||
* Stamps `userMetadata.shadeFilesReadStreamId = <streamId>`. Returns the
|
||||
* new streamId so the caller can echo it back to the client over RPC.
|
||||
*/
|
||||
initiateRead(opts: {
|
||||
peer: string;
|
||||
stream: ReadableStream<Uint8Array>;
|
||||
size: number;
|
||||
contentType?: string;
|
||||
name?: string;
|
||||
signal?: AbortSignal;
|
||||
onProgress?: (p: TransferProgress) => void;
|
||||
}): Promise<{ readStreamId: string; handle: TransferHandle }>;
|
||||
|
||||
/** Tear down the incoming-transfer subscription and reject all pending awaits. */
|
||||
destroy(): Promise<void>;
|
||||
}
|
||||
|
||||
interface PendingWaiter {
|
||||
resolve: (parked: ParkedWrite) => void;
|
||||
reject: (err: unknown) => void;
|
||||
expectedFrom: string;
|
||||
timer: ReturnType<typeof setTimeout> | null;
|
||||
abortListener: (() => void) | null;
|
||||
signal: AbortSignal;
|
||||
}
|
||||
|
||||
export interface CreateServerStreamsBridgeOptions {
|
||||
/** Default deadline for `awaitWrite` if the caller doesn't supply one. */
|
||||
defaultAwaitWriteTimeoutMs?: number;
|
||||
/** How long to retain a parked transfer waiting for its RPC. Default 60_000. */
|
||||
parkedWriteTtlMs?: number;
|
||||
}
|
||||
|
||||
export async function createServerStreamsBridge(
|
||||
shade: StreamsBridgeShade,
|
||||
options: CreateServerStreamsBridgeOptions = {},
|
||||
): Promise<ServerStreamsBridge> {
|
||||
const parkedWriteTtlMs = options.parkedWriteTtlMs ?? 60_000;
|
||||
const defaultAwaitTimeoutMs = options.defaultAwaitWriteTimeoutMs ?? 60_000;
|
||||
|
||||
const parked = new Map<string, ParkedWrite>();
|
||||
const waiters = new Map<string, PendingWaiter>();
|
||||
let destroyed = false;
|
||||
|
||||
const unsubscribeIncoming = await shade.onIncomingTransfer(async (incoming) => {
|
||||
const writeId = incoming.metadata.userMetadata?.[META_KEY_WRITE_ID];
|
||||
if (writeId === undefined) return; // not a files-bridge transfer; ignore
|
||||
|
||||
// Accept synchronously into a TransformStream so the engine can deliver
|
||||
// chunks immediately. The readable side is parked for the matching RPC.
|
||||
const ts = new TransformStream<Uint8Array, Uint8Array>();
|
||||
let handle: TransferHandle;
|
||||
try {
|
||||
handle = await incoming.accept({
|
||||
output: { kind: 'pipe', pipeTo: ts.writable },
|
||||
});
|
||||
} catch (err) {
|
||||
// Accept failed — likely double-init or shutdown. Drop the parked
|
||||
// entry; the awaiting RPC will time out.
|
||||
console.error('[shade-files streams-bridge] accept failed:', err);
|
||||
return;
|
||||
}
|
||||
|
||||
const arrived: ParkedWrite = {
|
||||
from: incoming.from,
|
||||
readable: ts.readable,
|
||||
done: handle.done().then((r) => ({ sha256: r.sha256, bytesSent: r.bytesSent })),
|
||||
handle,
|
||||
arrivedAt: Date.now(),
|
||||
};
|
||||
arrived.done.catch(() => {
|
||||
/* swallow until consumer awaits */
|
||||
});
|
||||
|
||||
const waiter = waiters.get(writeId);
|
||||
if (waiter !== undefined) {
|
||||
waiters.delete(writeId);
|
||||
cleanupWaiter(waiter);
|
||||
if (incoming.from !== waiter.expectedFrom) {
|
||||
void handle.abort('sender-mismatch').catch(() => undefined);
|
||||
waiter.reject(
|
||||
new Error(
|
||||
`streams-bridge: writeId=${writeId} delivered by ${incoming.from}, expected ${waiter.expectedFrom}`,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
waiter.resolve(arrived);
|
||||
return;
|
||||
}
|
||||
|
||||
// No waiter yet — park.
|
||||
parked.set(writeId, arrived);
|
||||
setTimeout(() => {
|
||||
const stale = parked.get(writeId);
|
||||
if (stale === arrived) {
|
||||
parked.delete(writeId);
|
||||
void handle.abort('rpc-timeout').catch(() => undefined);
|
||||
}
|
||||
}, parkedWriteTtlMs).unref?.();
|
||||
});
|
||||
|
||||
function cleanupWaiter(w: PendingWaiter): void {
|
||||
if (w.timer !== null) clearTimeout(w.timer);
|
||||
if (w.abortListener !== null) {
|
||||
w.signal.removeEventListener('abort', w.abortListener);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
async awaitWrite(writeId, opts) {
|
||||
if (destroyed) throw new Error('streams-bridge: destroyed');
|
||||
// 1. Already parked?
|
||||
const ready = parked.get(writeId);
|
||||
if (ready !== undefined) {
|
||||
parked.delete(writeId);
|
||||
if (ready.from !== opts.expectedFrom) {
|
||||
void ready.handle.abort('sender-mismatch').catch(() => undefined);
|
||||
throw new Error(
|
||||
`streams-bridge: writeId=${writeId} delivered by ${ready.from}, expected ${opts.expectedFrom}`,
|
||||
);
|
||||
}
|
||||
return ready;
|
||||
}
|
||||
|
||||
if (waiters.has(writeId)) {
|
||||
throw new Error(`streams-bridge: writeId=${writeId} already awaited`);
|
||||
}
|
||||
|
||||
// 2. Park a waiter
|
||||
const timeoutMs = opts.timeoutMs ?? defaultAwaitTimeoutMs;
|
||||
return await new Promise<ParkedWrite>((resolve, reject) => {
|
||||
const w: PendingWaiter = {
|
||||
resolve,
|
||||
reject,
|
||||
expectedFrom: opts.expectedFrom,
|
||||
timer: null,
|
||||
abortListener: null,
|
||||
signal: opts.signal,
|
||||
};
|
||||
w.timer = setTimeout(() => {
|
||||
if (waiters.get(writeId) === w) {
|
||||
waiters.delete(writeId);
|
||||
cleanupWaiter(w);
|
||||
reject(new OperationTimeoutError(`streams-bridge: writeId=${writeId} timed out after ${timeoutMs}ms`));
|
||||
}
|
||||
}, timeoutMs);
|
||||
if (opts.signal.aborted) {
|
||||
cleanupWaiter(w);
|
||||
reject(opts.signal.reason ?? new Error('aborted before await'));
|
||||
return;
|
||||
}
|
||||
const onAbort = (): void => {
|
||||
if (waiters.get(writeId) === w) {
|
||||
waiters.delete(writeId);
|
||||
cleanupWaiter(w);
|
||||
reject(opts.signal.reason ?? new Error('aborted'));
|
||||
}
|
||||
};
|
||||
w.abortListener = onAbort;
|
||||
opts.signal.addEventListener('abort', onAbort, { once: true });
|
||||
waiters.set(writeId, w);
|
||||
});
|
||||
},
|
||||
|
||||
async initiateRead(opts) {
|
||||
if (destroyed) throw new Error('streams-bridge: destroyed');
|
||||
const readStreamId = generateRequestId();
|
||||
const transferOpts: import('@shade/transfer').TransferOptions = {
|
||||
to: opts.peer,
|
||||
input: opts.stream,
|
||||
metadata: {
|
||||
...(opts.name !== undefined ? { name: opts.name } : {}),
|
||||
...(opts.contentType !== undefined ? { contentType: opts.contentType } : {}),
|
||||
sizeBytes: opts.size,
|
||||
userMetadata: { [META_KEY_READ_STREAM_ID]: readStreamId },
|
||||
},
|
||||
...(opts.signal !== undefined ? { signal: opts.signal } : {}),
|
||||
...(opts.onProgress !== undefined ? { onProgress: opts.onProgress } : {}),
|
||||
};
|
||||
const handle = await shade.upload(transferOpts);
|
||||
// The dispatcher returns the response immediately and doesn't itself
|
||||
// await this handle — the receiver awaits its own handle.done() on
|
||||
// the inbound side. Attach a no-op catch so engine teardown during
|
||||
// an in-flight outbound transfer doesn't surface as an unhandled
|
||||
// rejection in test environments.
|
||||
handle.done().catch(() => undefined);
|
||||
return { readStreamId, handle };
|
||||
},
|
||||
|
||||
async destroy() {
|
||||
if (destroyed) return;
|
||||
destroyed = true;
|
||||
unsubscribeIncoming();
|
||||
// Reject all waiting RPCs
|
||||
for (const w of waiters.values()) {
|
||||
cleanupWaiter(w);
|
||||
w.reject(new Error('streams-bridge: destroyed'));
|
||||
}
|
||||
waiters.clear();
|
||||
// Abort all parked transfers
|
||||
for (const p of parked.values()) {
|
||||
try {
|
||||
await p.handle.abort('bridge-destroyed');
|
||||
} catch {
|
||||
/* swallow */
|
||||
}
|
||||
}
|
||||
parked.clear();
|
||||
},
|
||||
};
|
||||
}
|
||||
61
packages/shade-files/src/server/thumbnail.ts
Normal file
61
packages/shade-files/src/server/thumbnail.ts
Normal file
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* Thumbnail format-hardening helper.
|
||||
*
|
||||
* Magic-byte verification for the three formats `@shade/files` advertises
|
||||
* over the wire (`png`, `webp`, `jpeg`). Apps producing thumbnails outside
|
||||
* the dispatcher (e.g. precomputing on disk) can call `assertThumbnailFormat`
|
||||
* to fail-fast on mislabelled bytes before they reach the network.
|
||||
*
|
||||
* The same magic-byte table is used inside `io-adapters.ts:adaptThumbnailResult`
|
||||
* to enforce the invariant on every outbound thumbnail.
|
||||
*/
|
||||
|
||||
import { ConflictError } from '../schemas/errors.js';
|
||||
|
||||
const PNG_MAGIC = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
||||
const JPEG_MAGIC = new Uint8Array([0xff, 0xd8, 0xff]);
|
||||
const WEBP_RIFF = new Uint8Array([0x52, 0x49, 0x46, 0x46]);
|
||||
const WEBP_HEAD = new Uint8Array([0x57, 0x45, 0x42, 0x50]);
|
||||
|
||||
export type ThumbnailFormat = 'png' | 'webp' | 'jpeg';
|
||||
|
||||
/**
|
||||
* Verify the bytes look like the declared format. Returns boolean — does
|
||||
* not throw. For a throwing variant, use `assertThumbnailFormat`.
|
||||
*/
|
||||
export function isThumbnailFormat(bytes: Uint8Array, format: ThumbnailFormat): boolean {
|
||||
switch (format) {
|
||||
case 'png':
|
||||
return startsWith(bytes, PNG_MAGIC);
|
||||
case 'jpeg':
|
||||
return startsWith(bytes, JPEG_MAGIC);
|
||||
case 'webp': {
|
||||
if (bytes.byteLength < 12) return false;
|
||||
if (!startsWith(bytes, WEBP_RIFF)) return false;
|
||||
for (let i = 0; i < 4; i++) {
|
||||
if (bytes[8 + i] !== WEBP_HEAD[i]) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Throws `ConflictError` if `bytes` doesn't match the declared `format`. */
|
||||
export function assertThumbnailFormat(
|
||||
bytes: Uint8Array,
|
||||
format: ThumbnailFormat,
|
||||
): void {
|
||||
if (!isThumbnailFormat(bytes, format)) {
|
||||
throw new ConflictError(
|
||||
`thumbnail bytes do not match declared format=${format}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function startsWith(bytes: Uint8Array, prefix: Uint8Array): boolean {
|
||||
if (bytes.byteLength < prefix.byteLength) return false;
|
||||
for (let i = 0; i < prefix.byteLength; i++) {
|
||||
if (bytes[i] !== prefix[i]) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
Reference in New Issue
Block a user