Ember mirror
Your camera as a dark mirror. Each camera frame, a compute pass compares every cell of the picture with the last and sheds particle embers where an edge moved, coloured by what shed them. Bloom adds glow. Without a camera, drifting smoke sheds instead. A video feed copies each frame into a texture the scene owns, on the web and on native.
Files, in reading order
- scene.tsCreates the scene: its resources, its settings and what each frame records.
- embers.tsSheds embers from the feed's moving edges.
- mirror.tsDraws a dim reflection behind the embers over the feed's rectangle of the world, from the origin to its size in texels: the camera's newest frame in place while one streams, otherwise the feed, into which it paints drifting smoke.
- ui.ts
Kit modules baker, bindings, bloom, cameras, clock, finish, owner, particles, passes, pipelines, targets, tone, video. Also @typegpu/color, @typegpu/noise, typegpu.
scene.ts
import { bindings, bloom, cameras, clock, finish, owner, passes, pipelines, targets, video, type Frame, type Surface,} from '@typegpu/kit';import type { TgpuRoot } from 'typegpu';import type { Settings } from '../settings.ts';import type { Camera } from '../ui.ts';import { createEmbers } from './embers.ts';import { createMirror } from './mirror.ts';/** The feed's long and short sides in texels. The feed is also the world the embers rise through. */const LONG = 640;const SHORT = 360;const STEP_SECONDS = 1 / 120;/** Smoke repaints at a camera's pace, so it sheds as a camera would. */const SMOKE_HZ = 30;/** Seconds without a camera frame before the smoke returns. */const CAMERA_GONE_SECONDS = 1;const LOOK = { exposure: 1.1, glowIntensity: 0.3, vignette: 0.45 };const BLOOM = { threshold: 0.7 };const SAMPLED = { format: 'rgba16float', sampled: true } as const;const defaults = { /** Change in brightness that counts as motion. Lower sheds from gentler movement. */ sensitivity: 0.08, /** Chance a moving edge sheds an ember each camera frame. */ rate: 0.35, /** Mixes ember colour from fire to the colour of what shed it. */ tint: 0.12, /** Brightness of the reflection behind the embers. */ reflection: 0.12, paused: false,};/** Your camera as a dark mirror whose moving edges shed embers. */export async function createScene(root: TgpuRoot, surface: Surface) { const values = { ...defaults }; const own = owner(); // A portrait screen gets a portrait feed, so a phone camera isn't cropped to a landscape band const portrait = surface.height > surface.width; const size = portrait ? ([SHORT, LONG] as const) : ([LONG, SHORT] as const); const [feedWidth, feedHeight] = size; const feed = own(video.create(root, { size })); // The embers compare frames in the small feed; the reflection samples the camera in place, where // the feed's crop lies const live = own(video.live(root, { crop: { size } })); const shared = bindings.share(root, { camera: live.part }); const sim = clock.fixedStep({ step: STEP_SECONDS }); const smoke = clock.cadence({ hz: SMOKE_HZ }); // Shows the middle of the feed, as large as covers the screen const view = cameras.flat(root, { size: surface, fit: { width: feedWidth, height: feedHeight, mode: 'cover' }, center: [feedWidth / 2, feedHeight / 2], }); const fire = own(createEmbers(root, { camera: view, step: STEP_SECONDS, feed, size })); const { embers } = fire; const mirror = createMirror(root, { feed, live, view, format: SAMPLED.format }); const screen = own(targets.create(root, surface, { hdr: SAMPLED })); const scene = passes.create(root, screen, { hdr: passes.color }, { shared }).pipe(view.inject()); const sparks = scene.add(embers.drawing); const hdr = () => screen.textures.hdr; const glow = own(bloom.create(root, { source: hdr, settings: BLOOM })); const final = own( finish.create(root, { format: surface.format, source: hdr, glow, settings: LOOK }), ); let lensed = false; let seen = feed.frames; let sinceCamera = Infinity; let pictures = 0; let shed = 0; const settings: Settings<typeof values> = { values, knobs: { sensitivity: { min: 0.02, max: 0.3, step: 0.01 }, rate: { label: 'shedding', min: 0, max: 1, step: 0.05 }, tint: { label: 'camera colour', min: 0, max: 1, step: 0.05 }, reflection: { min: 0, max: 0.5, step: 0.01 }, paused: {}, }, write(key) { if (key === undefined || key === 'paused') sim.paused = values.paused; if (key === undefined || key === 'reflection') { mirror.framing.patch({ brightness: values.reflection }); } if (key === undefined || key === 'sensitivity' || key === 'rate' || key === 'tint') { const { sensitivity, rate, tint } = values; fire.settings.patch({ sensitivity, rate, tint }); } }, readouts: { /** Counts camera frames the feed has written. */ 'camera frames': () => feed.frames, 'pictures shed from': () => shed, }, }; const camera: Camera = { attach(source) { feed.attach(source); live.attach(source); }, get frames() { return feed.frames; }, }; await Promise.all([ feed.initAsync(), mirror.initAsync(), fire.initAsync(), pipelines.compile([embers, scene, glow, final]), ]); return { settings, ui: { camera }, frame(frame: Frame) { view.update(frame.dt); screen.fit(); feed.update(frame); live.update(frame); if (feed.frames !== seen) { // The camera came back, so the smoke's last picture must not count as its first's past if (sinceCamera >= CAMERA_GONE_SECONDS) fire.reset(); seen = feed.frames; sinceCamera = 0; pictures += 1; } else { sinceCamera += frame.dt; if (sinceCamera >= CAMERA_GONE_SECONDS && smoke.due(frame.dt)) { mirror.paint(frame.encoder, frame.time); pictures += 1; } } // While a camera streams, the reflection samples it at full resolution const streaming = sinceCamera < CAMERA_GONE_SECONDS && feed.frames > 0; if (streaming !== lensed) { lensed = streaming; mirror.framing.patch({ live: streaming ? 1 : 0 }); } const steps = sim.advance(frame.dt); const compute = frame.encoder.beginComputePass(); if (pictures !== shed && !values.paused) { fire.shed(compute, sim.time); shed = pictures; } for (let i = 0; i < steps; i++) embers.step(compute); embers.spawn(compute); embers.compact(compute, sim.ahead); compute.end(); const pass = scene.begin(frame); mirror.draw(pass); sparks.draw(pass); pass.end(); glow.record(frame); final.record(frame, surface.view(frame)); }, destroy: own.destroy, };}embers.ts
import { type CameraBase, type Feed, Particle, particles, tone } from '@typegpu/kit';import { srgbToLinear } from '@typegpu/color';import { perlin2d, randf } from '@typegpu/noise';import { d, std, tgpu, type TgpuComputePass, type TgpuRoot } from 'typegpu';/** Feed texels per side of a cell that can shed an ember. */const CELL = 4;const WORKGROUP = 64;const CAPACITY = 16_384;/** Upward pull in feed texels per second squared: embers rise like sparks over a fire. */const LIFT = 70;const DRAG = 1.1;const SPEED = { least: 18, most: 60, across: 22 } as const;const LIFE_SECONDS = { least: 0.9, most: 2.6 } as const;const SIZE_TEXELS = 2.6;/** Sideways push of the air the embers drift through, in texels per second squared. */const SWIRL = 160;const SWIRL_SCALE = 0.012;const HOT = d.vec3f(6, 2.6, 0.6);const COOL = d.vec3f(2.4, 0.4, 0.04);export const Shedding = d.struct({ /** Change in brightness, from 0 to 1, that counts as motion. */ sensitivity: d.f32, /** Chance a moving edge cell sheds an ember per camera frame. */ rate: d.f32, /** Mixes each ember's colour from fire, 0, to the colour of what shed it, 1. */ tint: d.f32, /** Seconds of simulation, which move the air. */ seconds: d.f32, /** 0 on the first frame after a reset, which only records brightness. */ primed: d.u32,});/** * Sheds embers from the feed's moving edges. Brightness is kept per cell between camera frames, * and a cell sheds when its brightness changes where the picture has an edge. */export function createEmbers( root: TgpuRoot, options: { readonly camera: CameraBase; readonly step: number; readonly feed: Feed; readonly size: readonly [number, number]; },) { const { camera, step, feed, size } = options; const grid = d.vec2u(size[0] / CELL, size[1] / CELL); const cells = grid.x * grid.y; const height = size[1]; const reading = feed.texture.createView(d.texture2d(d.f32)); const brightness = root.createMutable(d.arrayOf(d.f32, cells)); const settings = root.createUniform(Shedding); const drift = (ember: Particle, dt: number) => { 'use gpu'; const next = Particle(ember); const air = d.vec2f(next.position.x, next.position.y) * SWIRL_SCALE; const push = perlin2d.sample(air + d.vec2f(0, settings.$.seconds * 0.4)); next.velocity.x += push * SWIRL * dt; return next; }; const embers = particles.create(root, camera, { capacity: CAPACITY, step, dimensions: 2, update: drift, shade: particles.glowingDisc, settings: { gravity: [0, LIFT, 0], drag: DRAG }, }); const lumaAt = (texel: d.v2i) => { 'use gpu'; const at = std.clamp(texel, d.vec2i(), d.vec2i(size[0] - 1, size[1] - 1)); return tone.luminance(std.textureLoad(reading.$, at, 0).rgb); }; const shed = tgpu.computeFn({ in: { gid: d.builtin.globalInvocationId }, workgroupSize: [WORKGROUP], })(({ gid }) => { 'use gpu'; if (gid.x >= cells) return; const cell = d.vec2i(d.i32(gid.x % grid.x), d.i32(gid.x / grid.x)); const centre = cell * CELL + d.i32(CELL / 2); const here = (lumaAt(centre) + lumaAt(centre - 1)) * 0.5; const across = lumaAt(centre + d.vec2i(CELL, 0)) - lumaAt(centre - d.vec2i(CELL, 0)); const down = lumaAt(centre + d.vec2i(0, CELL)) - lumaAt(centre - d.vec2i(0, CELL)); const edge = std.length(d.vec2f(across, down)); const before = brightness.$[gid.x]; brightness.$[gid.x] = here; if (settings.$.primed === 0) return; const sensitivity = settings.$.sensitivity; const moved = std.smoothstep(sensitivity * 0.5, sensitivity * 2, std.abs(here - before)); const outlined = std.smoothstep(0.04, 0.25, edge); randf.seed3(d.vec3f(d.vec2f(cell), settings.$.seconds)); if (randf.sample() >= moved * outlined * settings.$.rate) return; const texel = d.vec2f(centre) + (randf.inUnitCircle() * CELL) / 2; const seen = srgbToLinear(std.textureLoad(reading.$, d.vec2i(texel), 0).rgb); const heat = randf.sample(); const fire = std.mix(COOL, HOT, heat); const shade = seen * (tone.luminance(fire) / std.max(tone.luminance(seen), 0.05)); const speed = SPEED.least + (SPEED.most - SPEED.least) * randf.sample(); embers.emit( Particle({ position: d.vec3f(texel.x, height - texel.y, 0), age: 0, velocity: d.vec3f((randf.sample() * 2 - 1) * SPEED.across, speed, 0), life: LIFE_SECONDS.least + (LIFE_SECONDS.most - LIFE_SECONDS.least) * randf.sample(), color: d.vec4f(std.mix(fire, shade, settings.$.tint), 1), size: SIZE_TEXELS * (0.6 + heat * 0.8), seed: randf.sample(), }), ); }); const shedder = root.pipe(embers.inject()).createComputePipeline({ compute: shed }); const groups = Math.ceil(cells / WORKGROUP); let primed = false; return { embers, settings, initAsync: () => shedder.initAsync(), /** Compares a new picture with the last and sheds embers where it moved. */ shed(pass: TgpuComputePass, seconds: number) { settings.patch({ seconds, primed: primed ? 1 : 0 }); primed = true; pass.setPipeline(shedder); pass.dispatchWorkgroups(groups); }, /** Forgets the last picture, so the next only records brightness. */ reset() { primed = false; }, destroy() { embers.destroy(); brightness.buffer.destroy(); }, };}mirror.ts
import { baker, cameras, tone, video, type Feed } from '@typegpu/kit';import { srgbToLinear } from '@typegpu/color';import { perlin3d } from '@typegpu/noise';import { d, std, tgpu, type TgpuCommandEncoder, type TgpuRenderCommands, type TgpuRoot,} from 'typegpu';/** The cold glass the reflection sits behind. */const GLASS = d.vec3f(0.45, 0.62, 1);/** How much of the camera's own colour shows through the glass's tint. */const COLOR = 0.3;export const Framing = d.struct({ /** Linear brightness of the reflection. */ brightness: d.f32, /** 1 while a camera streams, which the reflection then samples at full resolution. */ live: d.u32,});/** * Draws a dim reflection behind the embers over the feed's rectangle of the world, from the origin * to its size in texels: the camera's newest frame in place while one streams, otherwise the feed, * into which it paints drifting smoke. */export function createMirror( root: TgpuRoot, options: { readonly feed: Feed; readonly live: video.Live; readonly view: cameras.CameraBase; readonly format: GPUTextureFormat; },) { const { feed, live, view, format } = options; const [width, height] = feed.texture.props.size; const framing = root.createUniform(Framing, { brightness: 0, live: 0, }); /** Seconds the smoke has drifted. */ const drift = root.createUniform(d.f32); const reading = feed.texture.createView(d.texture2d(d.f32)); const sampler = root.createSampler({ magFilter: 'linear', minFilter: 'linear' }); /** The feed's corners in a strip, with feed coordinates from its top-left corner. */ const corner = tgpu.vertexFn({ in: { i: d.builtin.vertexIndex }, out: { position: d.builtin.position, uv: d.vec2f }, })(({ i }) => { 'use gpu'; const uv = d.vec2f(d.f32(i & 1), d.f32(i >>> 1)); const at = d.vec2f(uv.x * width, (1 - uv.y) * height); return { position: cameras.current.$.viewProjection * d.vec4f(at, 0, 1), uv }; }); const reflect = tgpu.fragmentFn({ in: { uv: d.vec2f }, out: d.vec4f })(({ uv }) => { 'use gpu'; let encoded = std.textureSampleLevel(reading.$, sampler.$, uv, 0).rgb; if (framing.$.live === 1) { encoded = video.sample(uv).rgb; } const seen = srgbToLinear(encoded); const grey = tone.luminance(seen); const glass = std.mix(GLASS * grey, seen, COLOR); return d.vec4f(glass * framing.$.brightness, 1); }); const reflecting = root .pipe(live.inject()) .pipe(view.inject()) .createRenderPipeline({ vertex: corner, fragment: reflect, targets: { format }, primitive: { topology: 'triangle-strip' }, }); /** Smoke shapes slow enough to read as figures, with edges that move. */ const smoke = baker.create(root, feed.texture, { texel: (uv) => { 'use gpu'; const at = d.vec3f(uv.x * (width / height) * 2.2, uv.y * 2.2, drift.$ * 0.22); const body = perlin3d.sample(at) + perlin3d.sample(at * 2.1 + 7) * 0.45; const shape = std.smoothstep(0.02, 0.1, body); const light = 0.06 + shape * (0.55 + 0.3 * (1 - uv.y)); return d.vec4f(d.vec3f(light), 1); }, }); return { framing, initAsync: () => Promise.all([reflecting.initAsync(), smoke.initAsync()]), /** Draws the reflection into the caller's pass. */ draw(pass: TgpuRenderCommands) { pass.setPipeline(reflecting); pass.draw(4); }, /** Paints smoke at `seconds` into the feed's texture, in place of a camera frame. */ paint(encoder: TgpuCommandEncoder, seconds: number) { drift.write(seconds); smoke.record(encoder); }, };}ui.ts
import { camera } from '../ui.ts';export const ui = { camera: camera('use camera') };