studio/denoise.js

// The trained denoiser (the `--restore` pre-pass), run in the browser.
//
// Inkvec's denoiser is a 19.7-million-parameter U-Net exported to ONNX — `restorer.onnx`,
// the same file the command line's `restore-model` build feeds to ONNX Runtime. Running it
// here is therefore not a port: it is the same graph in a second runtime, ONNX Runtime Web,
// which is the same engine again with WebGPU and WebAssembly kernels in place of the native
// ones. Nothing about the network, its weights or the arithmetic around it is reimplemented
// in JavaScript. Measured against native ONNX Runtime on a 512-px JPEG, the WebAssembly
// kernels agree to 4.2e-7, and after the 8-bit quantisation the tracer reads, one channel of
// one pixel in 786,432 differs by one level.
//
// What is reimplemented here is nothing at all, and that is deliberate: the padding, the
// compositing, the crop, the quantisation and the extreme-snapping all live in
// `inkvec-restore` and reach this file through the WebAssembly tracer's `Intake` object
// (`denoiser_input` / `take_denoiser_output`). This module's whole job is to turn one
// Float32Array into another.
//
// Two things are fetched before the denoiser can run, and kept in the browser's Cache Storage
// afterwards: ONNX Runtime Web's WebAssembly (~26 MB; its two small JavaScript files ride the
// HTTP cache, jsDelivr marks them immutable) and the weights (~76 MB). `prefetch` fetches
// both ahead of need -- Inkvec Studio Lite starts it as soon as its engine has arrived -- and
// `load` fetches whatever is still missing. One download at a time per file however many ask,
// and one that breaks off resumes where it stopped (an HTTP range) when it is asked again.
// The image is not among them: it never leaves the machine, the same as the tracer.

// Pinned, both of them. A runtime that silently moved under the page would be a second,
// untested set of kernels for a network whose agreement with the native one is a measured
// claim; and `wasmPaths` must point at the dist folder of the very same version, or ONNX
// Runtime loads its JavaScript from one release and its WebAssembly from another.
const ORT_VERSION = "1.30.0";
const ORT_BASE = `https://cdn.jsdelivr.net/npm/onnxruntime-web@${ORT_VERSION}/dist/`;
const ORT_ENTRY = `${ORT_BASE}ort.webgpu.min.mjs`;
// What the WebGPU build loads at run time (1.30 builds its GPU and CPU kernels with
// asyncify): the WebAssembly, handed over as bytes from the cache (`env.wasm.wasmBinary`),
// and its loader, which only the HTTP cache keeps.
const ORT_WASM = `${ORT_BASE}ort-wasm-simd-threaded.asyncify.wasm`;
const ORT_LOADER = `${ORT_BASE}ort-wasm-simd-threaded.asyncify.mjs`;
// Its size, for progress: jsDelivr compresses it, so its Content-Length is not this.
const ORT_WASM_BYTES = 26781914;
const ORT_CACHE = `inkvec-ort-${ORT_VERSION}`;

// Where a verified copy of the weights lives between visits. Bump the suffix if the model
// the page asks for ever changes, so an old entry cannot be served for a new URL.
const CACHE = "inkvec-denoiser-v1";

// The export's own names for its input and output (`export_restorer_onnx.py`).
const INPUT = "image";
const OUTPUT = "restored";

let ortPromise = null;
let sessionPromise = null;
let backend = null;
// Where the weights came from, so a session that has to be rebuilt can read them back out
// of the cache instead of holding 80 MB alive for a fallback that usually never happens.
let source = null;

/** The execution provider the loaded session actually runs on, or null before it loads. */
export function activeBackend() {
  return backend;
}

async function ort() {
  if (!ortPromise) {
    ortPromise = (async () => {
      const [mod, binary] = await Promise.all([import(/* @vite-ignore */ ORT_ENTRY), runtimeBinary()]);
      const ort = mod.default ?? mod;
      // The WebGPU build still loads WebAssembly: the GPU kernels live in a wasm module and
      // everything WebGPU has no kernel for falls back to CPU inside the same session. Its
      // bytes come from the cache; its loader from the same folder, the same version.
      ort.env.wasm.wasmPaths = ORT_BASE;
      if (binary) ort.env.wasm.wasmBinary = binary;
      // Threads need `SharedArrayBuffer`, which needs the cross-origin isolation the Space
      // asks for in its README. Where it is not granted this is one thread, and the CPU
      // fallback is correspondingly slower — the same trade the tracer makes next door.
      ort.env.wasm.numThreads = self.crossOriginIsolated
        ? Math.max(1, Math.min(navigator.hardwareConcurrency || 4, 8))
        : 1;
      ort.env.logLevel = "error";
      return ort;
    })();
    ortPromise.catch(() => { ortPromise = null; });
  }
  return ortPromise;
}

const hex = (buf) =>
  [...new Uint8Array(buf)].map((b) => b.toString(16).padStart(2, "0")).join("");

async function openCache(name) {
  try {
    return await caches.open(name);
  } catch (e) {
    // Private windows and pages with site data blocked have no Cache Storage. Downloading
    // every time is worse than caching, and better than not working.
    return null;
  }
}

/** Bytes of one file already fetched in this page, by URL, while its download is incomplete. */
const partial = new Map();
/** Downloads in flight, by URL: a second caller joins the first instead of fetching again. */
const inflight = new Map();
let aborter = null;

/**
 * One file, streamed with progress. If an earlier attempt in this page broke off, the bytes it
 * got are kept and only the rest is asked for (`Range`); a server that answers with the whole
 * file instead simply starts it over.
 */
async function download(url, { onBytes, priority, knownBytes, signal }) {
  const have = partial.get(url) ?? { chunks: [], received: 0 };
  const headers = have.received > 0 ? { Range: `bytes=${have.received}-` } : undefined;
  // `no-store`: the bytes go into Cache Storage below, and a second copy in the HTTP cache
  // would only double what is written to disk while the page is starting.
  const res = await fetch(url, { headers, priority, signal, cache: "no-store" });
  if (!(res.ok || res.status === 206)) throw new Error(`HTTP ${res.status} for ${url.split("?")[0]}`);
  if (res.status !== 206) {
    have.chunks = [];
    have.received = 0;
  }
  partial.set(url, have);
  const length = Number(res.headers.get("content-length")) || 0;
  const encoded = Boolean(res.headers.get("content-encoding"));
  const total = knownBytes || (!encoded && length ? have.received + length : 0);
  onBytes(have.received, total);
  const reader = res.body.getReader();
  for (;;) {
    const { done, value } = await reader.read();
    if (done) break;
    have.chunks.push(value);
    have.received += value.length;
    onBytes(have.received, total);
  }
  const bytes = new Uint8Array(have.received);
  let at = 0;
  for (const c of have.chunks) { bytes.set(c, at); at += c.length; }
  partial.delete(url);
  return bytes;
}

/**
 * What is still to be fetched of `files` ([{ url, cache, sha256?, knownBytes? }]), fetched
 * together with one combined progress report, verified where a hash is known, and stored.
 * Returns each file's bytes (from the cache or the network), in order.
 */
async function ensure(files, { onProgress, priority = "auto", read = true } = {}) {
  const found = await Promise.all(
    files.map(async (f) => {
      const cache = await openCache(f.cache);
      const hit = cache && (await cache.match(f.url));
      return { ...f, cache, hit };
    }),
  );
  const missing = found.filter((f) => !f.hit);
  if (!missing.length) {
    onProgress?.({ received: 0, total: 0, cached: true });
    return read ? Promise.all(found.map(async (f) => new Uint8Array(await f.hit.arrayBuffer()))) : [];
  }
  const got = new Map();
  const totals = new Map();
  const report = () => {
    let received = 0;
    let total = 0;
    for (const f of missing) {
      received += got.get(f.url) ?? 0;
      total += totals.get(f.url) || f.knownBytes || 0;
    }
    onProgress?.({ received, total: Math.max(total, received), cached: false });
  };
  if (!aborter) aborter = new AbortController();
  const signal = aborter.signal;
  const fetched = await Promise.all(
    missing.map((f) => {
      let job = inflight.get(f.url);
      if (!job) {
        job = (async () => {
          const bytes = await download(f.url, {
            priority,
            knownBytes: f.knownBytes,
            signal,
            onBytes: (n, t) => {
              got.set(f.url, n);
              if (t) totals.set(f.url, t);
              report();
            },
          });
          if (f.sha256) {
            const digest = hex(await crypto.subtle.digest("SHA-256", bytes));
            if (digest !== f.sha256) {
              throw new Error(`the denoiser weights failed verification: SHA-256 ${digest} is not ${f.sha256}`);
            }
          }
          if (f.cache) {
            // A failure here is a full disk or a quota, not a reason to refuse to denoise.
            try {
              const headers = { "content-type": "application/octet-stream", "content-length": String(bytes.length) };
              await f.cache.put(f.url, new Response(bytes, { headers }));
            } catch (e) { /* the next visit downloads again */ }
          }
          return bytes;
        })();
        inflight.set(f.url, job);
        job.then(() => inflight.delete(f.url), () => inflight.delete(f.url));
      } else {
        // Joined a download another caller started: its progress is reported there, and
        // here the file counts as arriving all at once.
        totals.set(f.url, f.knownBytes || 0);
        job.then((b) => { got.set(f.url, b.length); totals.set(f.url, b.length); report(); }, () => {});
      }
      return job;
    }),
  ).finally(() => {
    if (!inflight.size) aborter = null;
  });
  if (!read) return [];
  const out = [];
  let k = 0;
  for (const f of found) out.push(f.hit ? new Uint8Array(await f.hit.arrayBuffer()) : fetched[k++]);
  return out;
}

const runtimeFile = () => ({ url: ORT_WASM, cache: ORT_CACHE, knownBytes: ORT_WASM_BYTES });
const weightsFile = (url, sha256) => ({ url, cache: CACHE, sha256 });

/** ONNX Runtime Web's WebAssembly, from the cache, or fetched and kept there. */
async function runtimeBinary() {
  const [bytes] = await ensure([runtimeFile()]);
  return bytes;
}

/**
 * Fetch everything the denoiser needs and keep it, without starting it: the weights
 * (verified against `sha256`) and ONNX Runtime Web. `priority` is the fetch priority hint
 * ("low" for a download nobody is waiting for yet). Resolves to `{ cached }`: whether
 * everything was already here. Nothing is read back out of the cache to answer that.
 */
export async function prefetch({ url, sha256, onProgress, priority = "low" } = {}) {
  let cached = true;
  await ensure([weightsFile(url, sha256), runtimeFile()], {
    priority,
    read: false,
    onProgress: (p) => {
      if (!p.cached) cached = false;
      onProgress?.(p);
    },
  });
  // The small loaders as well, so a later visit's first run needs nothing from the network.
  if (!cached) {
    await Promise.all(
      [ORT_ENTRY, ORT_LOADER].map((u) => fetch(u, { priority }).then((r) => r.arrayBuffer(), () => null)),
    );
  }
  return { cached };
}

/** Stop any download in flight; what arrived is kept, and the next request resumes it. */
export function abort() {
  aborter?.abort();
  aborter = null;
}

/**
 * The weights, from the cache if they are there and from Hugging Face if they are not,
 * verified either way against the SHA-256 the tracer reports.
 *
 * Verified either way on purpose: `inkvec_restore::pull_onnx_weights` re-checks a file it
 * finds already present rather than trusting it, because a half-written or tampered copy
 * would otherwise fail much later, deep inside the runtime, with an unrelated message. The
 * hash costs about a tenth of a second against the minute the download costs once.
 */
async function weights(url, sha256, onProgress) {
  // The runtime is fetched alongside, so the progress reported is for everything missing.
  const [bytes] = await ensure([weightsFile(url, sha256), runtimeFile()], { onProgress, priority: "high" });
  const digest = hex(await crypto.subtle.digest("SHA-256", bytes));
  if (digest !== sha256) {
    const cache = await openCache(CACHE);
    if (cache) await cache.delete(url);
    throw new Error(`the denoiser weights failed verification: SHA-256 ${digest} is not ${sha256}`);
  }
  return bytes;
}

async function build(ortMod, model, eps) {
  return ortMod.InferenceSession.create(model, {
    executionProviders: eps,
    graphOptimizationLevel: "all",
  });
}

/**
 * Whether WebGPU here is real hardware, which is not the same question as whether
 * `navigator.gpu` exists.
 *
 * A browser with no usable GPU normally answers `requestAdapter()` with null, and that is the
 * end of it. It can also hand out a **software adapter** — WebGPU implemented on the CPU,
 * SwiftShader in Chrome, lavapipe or llvmpipe through Mesa. Running a 19.7-million-parameter
 * convolutional network through one is the worst of both worlds: CPU arithmetic with a GPU
 * abstraction in the way, where ONNX Runtime's own WebAssembly kernels are SIMD and, on an
 * isolated page, threaded. Measured in a headless Chromium whose only adapter was SwiftShader
 * (`vendor: "google", architecture: "swiftshader"`): the 512-px pass that the WebAssembly
 * kernels finish in about eight seconds there had still not returned ten minutes in.
 *
 * `isFallbackAdapter` is the spec's own name for this and is checked first, but Chrome does
 * not set it for SwiftShader, so the adapter's reported architecture is checked too.
 */
const SOFTWARE_ADAPTERS = /swiftshader|lavapipe|llvmpipe|software|warp/i;

async function realGpu() {
  if (!navigator.gpu) return false;
  try {
    const adapter = await navigator.gpu.requestAdapter({ powerPreference: "high-performance" });
    if (!adapter) return false;
    const info = adapter.info ?? {};
    const named = `${info.architecture ?? ""} ${info.device ?? ""} ${info.description ?? ""}`;
    if (adapter.isFallbackAdapter || SOFTWARE_ADAPTERS.test(named)) {
      const what = named.trim() || "fallback";
      console.info(`inkvec: WebGPU here is software (${what}); using the CPU kernels`);
      return false;
    }
    return true;
  } catch (e) {
    return false;
  }
}

/**
 * Load the denoiser, once per page. `url` and `sha256` come from the tracer
 * (`denoiser_model_url()` / `denoiser_model_sha256()`) so that the page cannot end up
 * checking a different model than the one the build was made against.
 *
 * Returns `{ backend }`. WebGPU is asked for a session before it is believed: an adapter can
 * exist and still refuse this graph, and the only honest way to report which kernels ran is
 * to have watched one of them start.
 */
export async function load({ url, sha256, onProgress, onStage } = {}) {
  if (!sessionPromise) {
    source = { url, sha256 };
    sessionPromise = (async () => {
      // The weights first: that fetches whatever is missing of both files, with progress,
      // so the runtime after it comes out of the cache.
      onStage?.("weights");
      const model = await weights(url, sha256, onProgress);
      onStage?.("runtime");
      const ortMod = await ort();
      onStage?.("session");
      if (await realGpu()) {
        try {
          const s = await build(ortMod, model, ["webgpu"]);
          backend = "webgpu";
          return s;
        } catch (e) {
          console.warn("inkvec: WebGPU would not take the denoiser, using the CPU kernels:", e);
        }
      }
      const s = await build(ortMod, model, ["wasm"]);
      backend = "wasm";
      return s;
    })();
    sessionPromise.catch(() => { sessionPromise = null; backend = null; });
  }
  await sessionPromise;
  return { backend };
}

/**
 * Run the network. `input` is what `Intake.denoiser_input()` returned and `[width, height]`
 * is `Intake.denoiser_input_size()`; the result goes straight back to
 * `Intake.take_denoiser_output()`.
 *
 * A WebGPU session that fails mid-run — a lost device, a buffer it cannot allocate for a
 * large image — is rebuilt on the CPU kernels once and the run retried, because the
 * alternative is telling someone their image is untraceable when it is only too big for
 * their GPU.
 */
export async function run(input, width, height) {
  const session = await sessionPromise;
  if (!session) throw new Error("the denoiser is not loaded");
  const ortMod = await ort();
  const dims = [1, 3, height, width];

  const once = async (s) => {
    const out = await s.run({ [INPUT]: new ortMod.Tensor("float32", input, dims) });
    const t = out[OUTPUT];
    if (!t) throw new Error(`the denoiser returned no "${OUTPUT}" tensor`);
    return t.data;
  };

  try {
    return await once(session);
  } catch (e) {
    if (backend !== "webgpu" || !source) throw e;
    console.warn("inkvec: the WebGPU denoiser run failed, retrying on the CPU kernels:", e);
    // The weights are in the cache by now, so this reads them back rather than downloading
    // them again.
    const model = await weights(source.url, source.sha256);
    sessionPromise = build(ortMod, model, ["wasm"]);
    backend = "wasm";
    return once(await sessionPromise);
  }
}