wip

assets/js/app.ts

@@ -9,6 +9,8 @@
 import './ujs';
 import './when-ready';
 
+import './vision/chroma-subsample';
+
 // When developing CSS, include the relevant CSS you're working on here
 // in order to enable HMR (live reload) on it.
 // Would typically be either the theme file, or any additional file

assets/js/vision/chroma-subsample.ts

@@ -0,0 +1,286 @@
+import { loadImageCroppedPowerOfTwo } from './load';
+import {
+  createNonMippedLinearTexture,
+  createNonMippedLinearTextureFbo,
+  createOffscreenCanvasRenderingContext,
+  createProgramFromSources,
+  Framebuffer,
+  getUniformLocation,
+  Texture,
+} from './webgl';
+
+const vert = `#version 300 es
+
+void main() {
+  float x = float((gl_VertexID & 1) << 2);
+  float y = float((gl_VertexID & 2) << 1);
+  gl_Position = vec4(x - 1.0, -1.0 * (y - 1.0), 0.0, 1.0);
+}
+`;
+
+const mapFragment = `#version 300 es
+
+precision highp float;
+
+uniform highp sampler2D image;
+uniform highp uvec2 windowResolution;
+uniform highp ivec2 dynamicOffset;
+
+out vec3 outCov;
+
+vec2 linearRGBToPbPr(vec3 rgb) {
+  float rPrime = rgb.r;
+  float gPrime = rgb.g;
+  float bPrime = rgb.b;
+
+  float pb = - (0.168736 * rPrime) - (0.331264 * gPrime) + (0.5      * bPrime);
+  float pr =   (0.5      * rPrime) - (0.418688 * gPrime) - (0.081312 * bPrime);
+
+  return vec2(pb, pr);
+}
+
+float coefficientOfVariation(vec4 values) {
+  float mean   = dot(values, vec4(1.0)) / 4.0;
+  float stddev = length(values - mean) / sqrt(4.0);
+
+  if (abs(mean) > 1e-4) {
+    return abs(stddev / mean);
+  } else {
+    return 0.0;
+  }
+}
+
+vec4 fetchWithOffset(ivec2 coord, ivec2 size) {
+  int x = min(coord.x + dynamicOffset.x, size.x - 1);
+  int y = min(coord.y + dynamicOffset.y, size.y - 1);
+
+  return texelFetch(image, ivec2(x, y), 0);
+}
+
+void main() {
+  ivec2 windowCoord = ivec2(vec2(gl_FragCoord.x, float(windowResolution.y) - gl_FragCoord.y) - 0.5);
+  ivec2 size = ivec2(textureSize(image, 0));
+
+  vec4 nw = fetchWithOffset(windowCoord * 2 + ivec2(0, 0), size);
+  vec4 ne = fetchWithOffset(windowCoord * 2 + ivec2(0, 1), size);
+  vec4 sw = fetchWithOffset(windowCoord * 2 + ivec2(1, 0), size);
+  vec4 se = fetchWithOffset(windowCoord * 2 + ivec2(1, 1), size);
+
+  vec2 chromaNw = linearRGBToPbPr(nw.rgb);
+  vec2 chromaNe = linearRGBToPbPr(ne.rgb);
+  vec2 chromaSw = linearRGBToPbPr(sw.rgb);
+  vec2 chromaSe = linearRGBToPbPr(se.rgb);
+
+  float covPb = coefficientOfVariation(vec4(chromaNw.x, chromaNe.x, chromaSw.x, chromaSe.x));
+  float covPr = coefficientOfVariation(vec4(chromaNw.y, chromaNe.y, chromaSw.y, chromaSe.y));
+
+  bvec4 nonOpaqueAlpha = lessThan(vec4(nw.a, ne.a, sw.a, se.a), vec4(1.0));
+  float alphaAny = dot(vec4(nonOpaqueAlpha), vec4(1.0));
+
+  outCov = vec3(covPb, covPr, alphaAny);
+}
+`;
+
+const reduceFragment = `#version 300 es
+
+precision highp float;
+
+uniform highp sampler2D image;
+uniform highp uvec2 windowResolution;
+
+#define REDUCE_HORIZONTAL 0
+#define REDUCE_VERTICAL 1
+uniform uint reduceDimension;
+
+out vec3 outCov;
+
+void main() {
+  ivec2 windowCoord = ivec2(vec2(gl_FragCoord.x, float(windowResolution.y) - gl_FragCoord.y) - 0.5);
+  if (windowCoord[reduceDimension] > 0) {
+    discard;
+  }
+
+  vec3 result = vec3(0.0);
+  ivec2 offset = ivec2(0);
+  int n = textureSize(image, 0)[reduceDimension];
+
+  for (int i = 0; i < n; i++) {
+    offset[reduceDimension] += 1;
+    result += texelFetch(image, windowCoord + offset, 0).xyz;
+  }
+
+  outCov = result;
+}
+`;
+const reduceHorizontal = 0;
+const reduceVertical = 1;
+
+function getDynamicOffset(round: number): [GLint, GLint] {
+  switch (round) {
+    case 0:
+      return [0, 0];
+    case 1:
+      return [1, 0];
+    case 2:
+      return [0, 1];
+    default:
+      return [1, 1];
+  }
+}
+
+function coefficientOfVariation(values: number[]): number {
+  const mean = values.reduce((a, n) => a + n, 0) / values.length;
+  const stddev = Math.sqrt(values.reduce((a, n) => a + (n - mean) * (n - mean), 0) / values.length);
+
+  if (Math.abs(mean) > 1e-4) {
+    return Math.abs(stddev / mean);
+  }
+
+  return 0;
+}
+
+type CovCovPb = number;
+type CovCovPr = number;
+type SumAlpha = number;
+
+async function detectChromaSubsampling(imageUrl: string): Promise<[CovCovPb, CovCovPr, SumAlpha]> {
+  const bitmap = await loadImageCroppedPowerOfTwo(imageUrl);
+  const gl = createOffscreenCanvasRenderingContext();
+
+  const mapProgram = createProgramFromSources(gl, vert, mapFragment);
+  const mapImage = getUniformLocation(gl, mapProgram, 'image');
+  const mapWindowResolution = getUniformLocation(gl, mapProgram, 'windowResolution');
+  const mapDynamicOffset = getUniformLocation(gl, mapProgram, 'dynamicOffset');
+
+  const reduceProgram = createProgramFromSources(gl, vert, reduceFragment);
+  const reduceImage = getUniformLocation(gl, reduceProgram, 'image');
+  const reduceWindowResolution = getUniformLocation(gl, reduceProgram, 'windowResolution');
+  const reduceReduceDimension = getUniformLocation(gl, reduceProgram, 'reduceDimension');
+
+  const sourceFormat = {
+    internalFormat: gl.RGBA,
+    format: gl.RGBA,
+    type: gl.UNSIGNED_BYTE,
+  };
+
+  const targetFormat = {
+    internalFormat: gl.RGBA32F,
+    format: gl.RGBA,
+    type: gl.FLOAT,
+  };
+
+  const sourceTexture = createNonMippedLinearTexture(gl, {
+    width: bitmap.width,
+    height: bitmap.height,
+    pixels: bitmap.data,
+    ...sourceFormat,
+  });
+
+  const mapFbo = createNonMippedLinearTextureFbo(gl, {
+    width: Math.max(1, bitmap.width >> 1),
+    height: Math.max(1, bitmap.height >> 1),
+    ...targetFormat,
+  });
+
+  const reduceHorizontalFbo = createNonMippedLinearTextureFbo(gl, {
+    width: 1,
+    height: Math.max(1, bitmap.height >> 1),
+    ...targetFormat,
+  });
+
+  const reduceVerticalFbo = createNonMippedLinearTextureFbo(gl, {
+    width: 1,
+    height: 1,
+    ...targetFormat,
+  });
+
+  function configure(
+    program: WebGLProgram,
+    srcLocation: WebGLUniformLocation,
+    srcTex: Texture,
+    dstResLocation: WebGLUniformLocation,
+    dstFramebuffer: Framebuffer,
+  ) {
+    // Set up program
+    gl.useProgram(program);
+
+    // Bind FBO for offscreen rendering
+    gl.bindFramebuffer(gl.FRAMEBUFFER, dstFramebuffer.object);
+
+    // Configure sampler
+    gl.uniform1i(srcLocation, 0);
+    gl.activeTexture(gl.TEXTURE0);
+    gl.bindTexture(gl.TEXTURE_2D, srcTex.object);
+
+    // Configure resolution
+    gl.uniform2ui(dstResLocation, dstFramebuffer.texture.width, dstFramebuffer.texture.height);
+
+    // Set viewport and scissor
+    gl.viewport(0, 0, dstFramebuffer.texture.width, dstFramebuffer.texture.height);
+    gl.disable(gl.SCISSOR_TEST);
+    gl.disable(gl.DEPTH_TEST);
+
+    // Discard existing contents
+    gl.clearColor(0, 0, 0, 0);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+  }
+
+  function generateSumCoeffs(round: number): [number, number, number] {
+    // Map pixels into the given format
+    configure(mapProgram, mapImage, sourceTexture, mapWindowResolution, mapFbo);
+    gl.uniform2i(mapDynamicOffset, ...getDynamicOffset(round));
+    gl.drawArrays(gl.TRIANGLES, 0, 3);
+
+    // Horizontal reduction
+    configure(reduceProgram, reduceImage, mapFbo.texture, reduceWindowResolution, reduceHorizontalFbo);
+    gl.uniform1ui(reduceReduceDimension, reduceHorizontal);
+    gl.drawArrays(gl.TRIANGLES, 0, 3);
+
+    // Vertical reduction
+    configure(reduceProgram, reduceImage, reduceHorizontalFbo.texture, reduceWindowResolution, reduceVerticalFbo);
+    gl.uniform1ui(reduceReduceDimension, reduceVertical);
+    gl.drawArrays(gl.TRIANGLES, 0, 3);
+
+    // Output
+    const sumCoeffs = new Float32Array(4);
+    gl.readPixels(0, 0, 1, 1, targetFormat.format, targetFormat.type, sumCoeffs);
+
+    return [sumCoeffs[0], sumCoeffs[1], sumCoeffs[2]];
+  }
+
+  const allCovPb: number[] = [];
+  const allCovPr: number[] = [];
+  let sumAlpha: number = 0;
+
+  for (let i = 0; i < 4; i++) {
+    const [covPb, covPr, alpha] = generateSumCoeffs(i);
+    allCovPb.push(covPb);
+    allCovPr.push(covPr);
+    sumAlpha += alpha;
+  }
+
+  const covCovPb = coefficientOfVariation(allCovPb);
+  const covCovPr = coefficientOfVariation(allCovPr);
+
+  return [covCovPb, covCovPr, sumAlpha];
+}
+
+export type SubsampleClassification = 'probablyNotSubsampled' | 'probablySubsampled' | 'hasTransparency';
+
+export async function classifyChromaSubsampling(imageUrl: string): Promise<SubsampleClassification> {
+  const [covCovPb, covCovPr, sumAlpha] = await detectChromaSubsampling(imageUrl);
+
+  if (sumAlpha > 0) {
+    // Regardless of whether it was subsampled, this image has transparency
+    // and so classifications about its quality are no longer relevant
+    return 'hasTransparency';
+  }
+
+  if (covCovPb * covCovPr > 1e-3) {
+    return 'probablySubsampled';
+  }
+
+  return 'probablyNotSubsampled';
+}
+
+(window as any).detectChromaSubsampling = detectChromaSubsampling;

assets/js/vision/load.ts

@@ -0,0 +1,36 @@
+export interface ResultImage {
+  width: number;
+  height: number;
+  data: ImageBitmap;
+}
+
+function nextLowestDimension(dimension: number): number {
+  return Math.min(Math.pow(2, Math.floor(Math.log2(dimension))), 4096);
+}
+
+export async function loadImageCroppedPowerOfTwo(url: string): Promise<ResultImage> {
+  const image = document.createElement('img');
+  const body = document.body;
+
+  await new Promise<void>((resolve, reject) => {
+    image.onload = () => resolve();
+    image.onerror = () => reject();
+    image.crossOrigin = '';
+    image.style.width = '1px';
+    image.style.height = '1px';
+    image.src = url;
+
+    body.insertAdjacentElement('beforeend', image);
+  });
+
+  const cropWidth = nextLowestDimension(image.naturalWidth);
+  const cropHeight = nextLowestDimension(image.naturalHeight);
+  const data = await createImageBitmap(image, 0, 0, cropWidth, cropHeight);
+  body.removeChild(image);
+
+  return {
+    width: cropWidth,
+    height: cropHeight,
+    data,
+  };
+}

assets/js/vision/webgl.ts

@@ -0,0 +1,125 @@
+export function compileShader(gl: WebGL2RenderingContext, shaderSource: string, shaderType: GLenum) {
+  const shader = gl.createShader(shaderType);
+  if (!shader) {
+    throw new Error(`failed to create shader of type ${shaderType}`);
+  }
+
+  gl.shaderSource(shader, shaderSource);
+  gl.compileShader(shader);
+
+  if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
+    throw new Error(`Shader compilation failed: ${gl.getShaderInfoLog(shader)}`);
+  }
+
+  return shader;
+}
+
+export function createProgramFromSources(
+  gl: WebGL2RenderingContext,
+  vertexShaderSource: string,
+  fragmentShaderSource: string,
+) {
+  const vertexShader = compileShader(gl, vertexShaderSource, gl.VERTEX_SHADER);
+  const fragmentShader = compileShader(gl, fragmentShaderSource, gl.FRAGMENT_SHADER);
+
+  const program = gl.createProgram();
+  if (!program) {
+    throw new Error('failed to create vertex + fragment program');
+  }
+
+  gl.attachShader(program, vertexShader);
+  gl.attachShader(program, fragmentShader);
+  gl.linkProgram(program);
+
+  if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
+    throw new Error(`Program link failed: ${gl.getProgramInfoLog(program)}`);
+  }
+
+  return program;
+}
+
+export function createOffscreenCanvasRenderingContext(): WebGL2RenderingContext {
+  const canvas = document.createElement('canvas');
+  const gl = canvas.getContext('webgl2');
+
+  if (!gl) {
+    throw new Error('failed to create WebGL2 context');
+  }
+
+  if (!gl.getExtension('EXT_color_buffer_float')) {
+    throw new Error('failed to enable EXT_color_buffer_float extension');
+  }
+
+  return gl;
+}
+
+export interface TextureParameters {
+  width: number;
+  height: number;
+  internalFormat: GLenum;
+  format: GLenum;
+  type: GLenum;
+  pixels?: ImageBitmap;
+}
+
+export interface Texture extends TextureParameters {
+  object: WebGLTexture;
+}
+
+export interface Framebuffer {
+  object: WebGLFramebuffer;
+  texture: Texture;
+}
+
+export function createNonMippedLinearTexture(gl: WebGL2RenderingContext, params: TextureParameters): Texture {
+  const texture = gl.createTexture();
+  if (!texture) {
+    throw new Error('failed to create texture');
+  }
+
+  const level = 0;
+  const internalFormat = params.internalFormat;
+  const border = 0;
+  const format = params.format;
+  const type = params.type;
+  const data = params.pixels;
+
+  gl.bindTexture(gl.TEXTURE_2D, texture);
+  if (data) {
+    gl.texImage2D(gl.TEXTURE_2D, level, internalFormat, format, type, data);
+  } else {
+    gl.texImage2D(gl.TEXTURE_2D, level, internalFormat, params.width, params.height, border, format, type, null);
+  }
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+
+  return { object: texture, ...params };
+}
+
+export function createNonMippedLinearTextureFbo(gl: WebGL2RenderingContext, params: TextureParameters): Framebuffer {
+  const texture = createNonMippedLinearTexture(gl, params);
+  const fbo = gl.createFramebuffer();
+  if (!fbo) {
+    throw new Error('failed to create framebuffer object');
+  }
+
+  gl.bindFramebuffer(gl.FRAMEBUFFER, fbo);
+  gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture.object, 0);
+
+  return { object: fbo, texture };
+}
+
+export function getUniformLocation(
+  gl: WebGL2RenderingContext,
+  program: WebGLProgram,
+  name: string,
+): WebGLUniformLocation {
+  const location = gl.getUniformLocation(program, name);
+  if (!location) {
+    throw new Error('failed to get uniform location');
+  }
+
+  return location;
+}