Shade/packages/shade-files/tests/integration/upload-directory.test.ts

import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { sha256 } from '@noble/hashes/sha2.js';
import {
  ConflictError,
  NotFoundError,
  uploadDirectory,
  createMemoryDirectory,
  type FileEntry,
  type BulkTransferEvent,
} from '../../src/index.js';
import { setupFileRig, type FileTestRig } from './helpers/rig.js';

interface StoredFile {
  bytes: Uint8Array;
  contentType?: string;
  sha256: string;
}

function bytesToHex(b: Uint8Array): string {
  return Array.from(b, (x) => x.toString(16).padStart(2, '0')).join('');
}

async function streamToBytes(s: ReadableStream<Uint8Array>): Promise<Uint8Array> {
  const reader = s.getReader();
  const parts: Uint8Array[] = [];
  let total = 0;
  while (true) {
    const { value, done } = await reader.read();
    if (done) break;
    if (value === undefined) continue;
    parts.push(value);
    total += value.byteLength;
  }
  reader.releaseLock();
  const out = new Uint8Array(total);
  let offset = 0;
  for (const p of parts) {
    out.set(p, offset);
    offset += p.byteLength;
  }
  return out;
}

describe('uploadDirectory — bulk upload to remote', () => {
  let rig: FileTestRig;
  const blobs = new Map<string, StoredFile>();
  const dirs = new Set<string>(['/']);

  beforeAll(async () => {
    blobs.clear();
    dirs.clear();
    dirs.add('/');

    rig = await setupFileRig({
      mkdir: async (ctx) => {
        const path = ctx.path;
        if (dirs.has(path)) {
          if (!ctx.args.recursive) throw new ConflictError('exists');
          // Idempotent for recursive
        }
        // Recursive: add ancestors
        if (ctx.args.recursive) {
          const segments = path.split('/').filter(Boolean);
          let acc = '';
          for (const seg of segments) {
            acc += '/' + seg;
            dirs.add(acc);
          }
        } else {
          dirs.add(path);
        }
        return {
          entry: {
            name: path.split('/').filter(Boolean).pop() ?? '',
            kind: 'dir',
            size: 0,
            mtime: Date.now(),
            metadata: {},
          },
        };
      },
      write: async (ctx) => {
        const args = ctx.args;
        let bytes: Uint8Array;
        let storedSha: string;
        if (args.content.kind === 'inline') {
          bytes = args.content.bytes;
          storedSha = args.content.sha256;
        } else {
          bytes = await streamToBytes(args.content.stream);
          storedSha = await args.content.sha256;
        }
        if (blobs.has(args.path) && !args.overwrite) {
          throw new ConflictError(`${args.path} exists`);
        }
        blobs.set(args.path, {
          bytes,
          ...(args.contentType !== undefined ? { contentType: args.contentType } : {}),
          sha256: storedSha,
        });
        const entry: FileEntry = {
          name: args.path.split('/').filter(Boolean).pop() ?? '',
          kind: 'file',
          size: bytes.byteLength,
          mtime: Date.now(),
          ...(args.contentType !== undefined ? { contentType: args.contentType } : {}),
          metadata: { sha256: storedSha },
        };
        return { entry };
      },
    });
  });

  afterAll(async () => {
    await rig.teardown();
  });

  test('uploads a small tree with mixed inline + streams', async () => {
    const local = createMemoryDirectory('local');
    const sub = local.addDir('sub');
    const small = new Uint8Array(100); for (let i = 0; i < small.length; i++) small[i] = i & 0xff;
    const big = new Uint8Array(400 * 1024); crypto.getRandomValues(big);
    local.addFile('hello.txt', new TextEncoder().encode('hello world'), 'text/plain');
    local.addFile('small.bin', small);
    sub.addFile('big.bin', big, 'application/octet-stream');

    const handle = uploadDirectory(rig.fs, local, '/upload-target');

    const events: BulkTransferEvent[] = [];
    (async () => {
      for await (const ev of handle.events) events.push(ev);
    })().catch(() => undefined);

    const result = await handle.done();
    expect(result.filesDone).toBe(3);
    expect(result.bytesDone).toBe(11 + 100 + 400 * 1024);

    // Verify remote tree
    expect(blobs.has('/upload-target/hello.txt')).toBe(true);
    expect(blobs.has('/upload-target/small.bin')).toBe(true);
    expect(blobs.has('/upload-target/sub/big.bin')).toBe(true);
    expect(dirs.has('/upload-target/sub')).toBe(true);

    // Sha256 paritet for streamed file
    expect(blobs.get('/upload-target/sub/big.bin')!.sha256).toBe(bytesToHex(sha256(big)));
    expect(blobs.get('/upload-target/hello.txt')!.contentType).toBe('text/plain');

    // Wait a tick for events to flush
    await new Promise((r) => setTimeout(r, 30));
    const planEvent = events.find((e) => e.type === 'plan');
    expect(planEvent).toBeDefined();
    if (planEvent && planEvent.type === 'plan') {
      expect(planEvent.totalFiles).toBe(3);
      expect(planEvent.totalBytes).toBe(11 + 100 + 400 * 1024);
    }
    const completes = events.filter((e) => e.type === 'complete');
    expect(completes.length).toBe(1);
  });

  test('aggregated progress is monotonically non-decreasing', async () => {
    const local = createMemoryDirectory('local');
    for (let i = 0; i < 10; i++) {
      local.addFile(`f${i}.bin`, new Uint8Array(50));
    }
    const handle = uploadDirectory(rig.fs, local, '/progress');
    const progresses: { filesDone: number; bytesDone: number }[] = [];
    (async () => {
      for await (const ev of handle.events) {
        if (ev.type === 'progress') {
          progresses.push({ filesDone: ev.filesDone, bytesDone: ev.bytesDone });
        }
      }
    })().catch(() => undefined);
    await handle.done();
    await new Promise((r) => setTimeout(r, 30));
    for (let i = 1; i < progresses.length; i++) {
      expect(progresses[i]!.filesDone).toBeGreaterThanOrEqual(progresses[i - 1]!.filesDone);
      expect(progresses[i]!.bytesDone).toBeGreaterThanOrEqual(progresses[i - 1]!.bytesDone);
    }
    expect(progresses[progresses.length - 1]!.filesDone).toBe(10);
  });

  test('fail-fast: first error aborts the bulk', async () => {
    const local = createMemoryDirectory('local');
    for (let i = 0; i < 5; i++) {
      local.addFile(`x${i}.bin`, new Uint8Array(10));
    }
    // Pre-create a conflicting file at /conflict/x0.bin so the first write fails.
    blobs.set('/conflict/x0.bin', { bytes: new Uint8Array(0), sha256: 'x' });

    const handle = uploadDirectory(rig.fs, local, '/conflict', { concurrency: 1 });
    await expect(handle.done()).rejects.toThrow();
  });

  test('continueOnError: completes despite per-file errors', async () => {
    const local = createMemoryDirectory('local');
    for (let i = 0; i < 5; i++) {
      local.addFile(`y${i}.bin`, new Uint8Array(10));
    }
    blobs.set('/cont/y2.bin', { bytes: new Uint8Array(0), sha256: 'x' });

    const handle = uploadDirectory(rig.fs, local, '/cont', {
      concurrency: 1,
      continueOnError: true,
    });
    const errors: string[] = [];
    (async () => {
      for await (const ev of handle.events) {
        if (ev.type === 'file-error') errors.push(ev.path);
      }
    })().catch(() => undefined);
    const result = await handle.done();
    await new Promise((r) => setTimeout(r, 30));
    expect(errors).toEqual(['y2.bin']);
    expect(result.filesDone).toBe(4);
  });

  test('concurrency cap respected', async () => {
    const local = createMemoryDirectory('local');
    for (let i = 0; i < 30; i++) {
      local.addFile(`z${i}.bin`, new Uint8Array(10));
    }
    // Concurrency above MAX (16) should be clamped.
    const handle = uploadDirectory(rig.fs, local, '/cap', { concurrency: 100 });
    const result = await handle.done();
    expect(result.filesDone).toBe(30);
  });

  test('aborts mid-flight via handle.abort()', async () => {
    const local = createMemoryDirectory('local');
    for (let i = 0; i < 50; i++) {
      local.addFile(`q${i}.bin`, new Uint8Array(50 * 1024)); // 50 KiB each
    }
    const handle = uploadDirectory(rig.fs, local, '/abort');
    setTimeout(() => void handle.abort('test-cancel'), 20);
    await expect(handle.done()).rejects.toThrow();
  });
});