From 295ddd510335d43898fb668bc9f58bd211593653 Mon Sep 17 00:00:00 2001 From: Liam Date: Sun, 25 Aug 2024 15:41:06 -0400 Subject: [PATCH] wip --- assets/js/app.ts | 2 + assets/js/vision/chroma-subsample.ts | 286 +++++++++++++++++++++++++++ assets/js/vision/load.ts | 36 ++++ assets/js/vision/webgl.ts | 125 ++++++++++++ 4 files changed, 449 insertions(+) create mode 100644 assets/js/vision/chroma-subsample.ts create mode 100644 assets/js/vision/load.ts create mode 100644 assets/js/vision/webgl.ts diff --git a/assets/js/app.ts b/assets/js/app.ts index 4f23d655..6ad885e8 100644 --- a/assets/js/app.ts +++ b/assets/js/app.ts @@ -9,6 +9,8 @@ import './ujs'; import './when-ready'; +import './vision/chroma-subsample'; + // When developing CSS, include the relevant CSS you're working on here // in order to enable HMR (live reload) on it. // Would typically be either the theme file, or any additional file diff --git a/assets/js/vision/chroma-subsample.ts b/assets/js/vision/chroma-subsample.ts new file mode 100644 index 00000000..8ed7dca2 --- /dev/null +++ b/assets/js/vision/chroma-subsample.ts @@ -0,0 +1,286 @@ +import { loadImageCroppedPowerOfTwo } from './load'; +import { + createNonMippedLinearTexture, + createNonMippedLinearTextureFbo, + createOffscreenCanvasRenderingContext, + createProgramFromSources, + Framebuffer, + getUniformLocation, + Texture, +} from './webgl'; + +const vert = `#version 300 es + +void main() { + float x = float((gl_VertexID & 1) << 2); + float y = float((gl_VertexID & 2) << 1); + gl_Position = vec4(x - 1.0, -1.0 * (y - 1.0), 0.0, 1.0); +} +`; + +const mapFragment = `#version 300 es + +precision highp float; + +uniform highp sampler2D image; +uniform highp uvec2 windowResolution; +uniform highp ivec2 dynamicOffset; + +out vec3 outCov; + +vec2 linearRGBToPbPr(vec3 rgb) { + float rPrime = rgb.r; + float gPrime = rgb.g; + float bPrime = rgb.b; + + float pb = - (0.168736 * rPrime) - (0.331264 * gPrime) + (0.5 * bPrime); + float pr = (0.5 * rPrime) - (0.418688 * gPrime) - (0.081312 * bPrime); + + return vec2(pb, pr); +} + +float coefficientOfVariation(vec4 values) { + float mean = dot(values, vec4(1.0)) / 4.0; + float stddev = length(values - mean) / sqrt(4.0); + + if (abs(mean) > 1e-4) { + return abs(stddev / mean); + } else { + return 0.0; + } +} + +vec4 fetchWithOffset(ivec2 coord, ivec2 size) { + int x = min(coord.x + dynamicOffset.x, size.x - 1); + int y = min(coord.y + dynamicOffset.y, size.y - 1); + + return texelFetch(image, ivec2(x, y), 0); +} + +void main() { + ivec2 windowCoord = ivec2(vec2(gl_FragCoord.x, float(windowResolution.y) - gl_FragCoord.y) - 0.5); + ivec2 size = ivec2(textureSize(image, 0)); + + vec4 nw = fetchWithOffset(windowCoord * 2 + ivec2(0, 0), size); + vec4 ne = fetchWithOffset(windowCoord * 2 + ivec2(0, 1), size); + vec4 sw = fetchWithOffset(windowCoord * 2 + ivec2(1, 0), size); + vec4 se = fetchWithOffset(windowCoord * 2 + ivec2(1, 1), size); + + vec2 chromaNw = linearRGBToPbPr(nw.rgb); + vec2 chromaNe = linearRGBToPbPr(ne.rgb); + vec2 chromaSw = linearRGBToPbPr(sw.rgb); + vec2 chromaSe = linearRGBToPbPr(se.rgb); + + float covPb = coefficientOfVariation(vec4(chromaNw.x, chromaNe.x, chromaSw.x, chromaSe.x)); + float covPr = coefficientOfVariation(vec4(chromaNw.y, chromaNe.y, chromaSw.y, chromaSe.y)); + + bvec4 nonOpaqueAlpha = lessThan(vec4(nw.a, ne.a, sw.a, se.a), vec4(1.0)); + float alphaAny = dot(vec4(nonOpaqueAlpha), vec4(1.0)); + + outCov = vec3(covPb, covPr, alphaAny); +} +`; + +const reduceFragment = `#version 300 es + +precision highp float; + +uniform highp sampler2D image; +uniform highp uvec2 windowResolution; + +#define REDUCE_HORIZONTAL 0 +#define REDUCE_VERTICAL 1 +uniform uint reduceDimension; + +out vec3 outCov; + +void main() { + ivec2 windowCoord = ivec2(vec2(gl_FragCoord.x, float(windowResolution.y) - gl_FragCoord.y) - 0.5); + if (windowCoord[reduceDimension] > 0) { + discard; + } + + vec3 result = vec3(0.0); + ivec2 offset = ivec2(0); + int n = textureSize(image, 0)[reduceDimension]; + + for (int i = 0; i < n; i++) { + offset[reduceDimension] += 1; + result += texelFetch(image, windowCoord + offset, 0).xyz; + } + + outCov = result; +} +`; +const reduceHorizontal = 0; +const reduceVertical = 1; + +function getDynamicOffset(round: number): [GLint, GLint] { + switch (round) { + case 0: + return [0, 0]; + case 1: + return [1, 0]; + case 2: + return [0, 1]; + default: + return [1, 1]; + } +} + +function coefficientOfVariation(values: number[]): number { + const mean = values.reduce((a, n) => a + n, 0) / values.length; + const stddev = Math.sqrt(values.reduce((a, n) => a + (n - mean) * (n - mean), 0) / values.length); + + if (Math.abs(mean) > 1e-4) { + return Math.abs(stddev / mean); + } + + return 0; +} + +type CovCovPb = number; +type CovCovPr = number; +type SumAlpha = number; + +async function detectChromaSubsampling(imageUrl: string): Promise<[CovCovPb, CovCovPr, SumAlpha]> { + const bitmap = await loadImageCroppedPowerOfTwo(imageUrl); + const gl = createOffscreenCanvasRenderingContext(); + + const mapProgram = createProgramFromSources(gl, vert, mapFragment); + const mapImage = getUniformLocation(gl, mapProgram, 'image'); + const mapWindowResolution = getUniformLocation(gl, mapProgram, 'windowResolution'); + const mapDynamicOffset = getUniformLocation(gl, mapProgram, 'dynamicOffset'); + + const reduceProgram = createProgramFromSources(gl, vert, reduceFragment); + const reduceImage = getUniformLocation(gl, reduceProgram, 'image'); + const reduceWindowResolution = getUniformLocation(gl, reduceProgram, 'windowResolution'); + const reduceReduceDimension = getUniformLocation(gl, reduceProgram, 'reduceDimension'); + + const sourceFormat = { + internalFormat: gl.RGBA, + format: gl.RGBA, + type: gl.UNSIGNED_BYTE, + }; + + const targetFormat = { + internalFormat: gl.RGBA32F, + format: gl.RGBA, + type: gl.FLOAT, + }; + + const sourceTexture = createNonMippedLinearTexture(gl, { + width: bitmap.width, + height: bitmap.height, + pixels: bitmap.data, + ...sourceFormat, + }); + + const mapFbo = createNonMippedLinearTextureFbo(gl, { + width: Math.max(1, bitmap.width >> 1), + height: Math.max(1, bitmap.height >> 1), + ...targetFormat, + }); + + const reduceHorizontalFbo = createNonMippedLinearTextureFbo(gl, { + width: 1, + height: Math.max(1, bitmap.height >> 1), + ...targetFormat, + }); + + const reduceVerticalFbo = createNonMippedLinearTextureFbo(gl, { + width: 1, + height: 1, + ...targetFormat, + }); + + function configure( + program: WebGLProgram, + srcLocation: WebGLUniformLocation, + srcTex: Texture, + dstResLocation: WebGLUniformLocation, + dstFramebuffer: Framebuffer, + ) { + // Set up program + gl.useProgram(program); + + // Bind FBO for offscreen rendering + gl.bindFramebuffer(gl.FRAMEBUFFER, dstFramebuffer.object); + + // Configure sampler + gl.uniform1i(srcLocation, 0); + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, srcTex.object); + + // Configure resolution + gl.uniform2ui(dstResLocation, dstFramebuffer.texture.width, dstFramebuffer.texture.height); + + // Set viewport and scissor + gl.viewport(0, 0, dstFramebuffer.texture.width, dstFramebuffer.texture.height); + gl.disable(gl.SCISSOR_TEST); + gl.disable(gl.DEPTH_TEST); + + // Discard existing contents + gl.clearColor(0, 0, 0, 0); + gl.clear(gl.COLOR_BUFFER_BIT); + } + + function generateSumCoeffs(round: number): [number, number, number] { + // Map pixels into the given format + configure(mapProgram, mapImage, sourceTexture, mapWindowResolution, mapFbo); + gl.uniform2i(mapDynamicOffset, ...getDynamicOffset(round)); + gl.drawArrays(gl.TRIANGLES, 0, 3); + + // Horizontal reduction + configure(reduceProgram, reduceImage, mapFbo.texture, reduceWindowResolution, reduceHorizontalFbo); + gl.uniform1ui(reduceReduceDimension, reduceHorizontal); + gl.drawArrays(gl.TRIANGLES, 0, 3); + + // Vertical reduction + configure(reduceProgram, reduceImage, reduceHorizontalFbo.texture, reduceWindowResolution, reduceVerticalFbo); + gl.uniform1ui(reduceReduceDimension, reduceVertical); + gl.drawArrays(gl.TRIANGLES, 0, 3); + + // Output + const sumCoeffs = new Float32Array(4); + gl.readPixels(0, 0, 1, 1, targetFormat.format, targetFormat.type, sumCoeffs); + + return [sumCoeffs[0], sumCoeffs[1], sumCoeffs[2]]; + } + + const allCovPb: number[] = []; + const allCovPr: number[] = []; + let sumAlpha: number = 0; + + for (let i = 0; i < 4; i++) { + const [covPb, covPr, alpha] = generateSumCoeffs(i); + allCovPb.push(covPb); + allCovPr.push(covPr); + sumAlpha += alpha; + } + + const covCovPb = coefficientOfVariation(allCovPb); + const covCovPr = coefficientOfVariation(allCovPr); + + return [covCovPb, covCovPr, sumAlpha]; +} + +export type SubsampleClassification = 'probablyNotSubsampled' | 'probablySubsampled' | 'hasTransparency'; + +export async function classifyChromaSubsampling(imageUrl: string): Promise { + const [covCovPb, covCovPr, sumAlpha] = await detectChromaSubsampling(imageUrl); + + if (sumAlpha > 0) { + // Regardless of whether it was subsampled, this image has transparency + // and so classifications about its quality are no longer relevant + return 'hasTransparency'; + } + + if (covCovPb * covCovPr > 1e-3) { + return 'probablySubsampled'; + } + + return 'probablyNotSubsampled'; +} + +(window as any).detectChromaSubsampling = detectChromaSubsampling; diff --git a/assets/js/vision/load.ts b/assets/js/vision/load.ts new file mode 100644 index 00000000..dd8631ad --- /dev/null +++ b/assets/js/vision/load.ts @@ -0,0 +1,36 @@ +export interface ResultImage { + width: number; + height: number; + data: ImageBitmap; +} + +function nextLowestDimension(dimension: number): number { + return Math.min(Math.pow(2, Math.floor(Math.log2(dimension))), 4096); +} + +export async function loadImageCroppedPowerOfTwo(url: string): Promise { + const image = document.createElement('img'); + const body = document.body; + + await new Promise((resolve, reject) => { + image.onload = () => resolve(); + image.onerror = () => reject(); + image.crossOrigin = ''; + image.style.width = '1px'; + image.style.height = '1px'; + image.src = url; + + body.insertAdjacentElement('beforeend', image); + }); + + const cropWidth = nextLowestDimension(image.naturalWidth); + const cropHeight = nextLowestDimension(image.naturalHeight); + const data = await createImageBitmap(image, 0, 0, cropWidth, cropHeight); + body.removeChild(image); + + return { + width: cropWidth, + height: cropHeight, + data, + }; +} diff --git a/assets/js/vision/webgl.ts b/assets/js/vision/webgl.ts new file mode 100644 index 00000000..cca2722f --- /dev/null +++ b/assets/js/vision/webgl.ts @@ -0,0 +1,125 @@ +export function compileShader(gl: WebGL2RenderingContext, shaderSource: string, shaderType: GLenum) { + const shader = gl.createShader(shaderType); + if (!shader) { + throw new Error(`failed to create shader of type ${shaderType}`); + } + + gl.shaderSource(shader, shaderSource); + gl.compileShader(shader); + + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + throw new Error(`Shader compilation failed: ${gl.getShaderInfoLog(shader)}`); + } + + return shader; +} + +export function createProgramFromSources( + gl: WebGL2RenderingContext, + vertexShaderSource: string, + fragmentShaderSource: string, +) { + const vertexShader = compileShader(gl, vertexShaderSource, gl.VERTEX_SHADER); + const fragmentShader = compileShader(gl, fragmentShaderSource, gl.FRAGMENT_SHADER); + + const program = gl.createProgram(); + if (!program) { + throw new Error('failed to create vertex + fragment program'); + } + + gl.attachShader(program, vertexShader); + gl.attachShader(program, fragmentShader); + gl.linkProgram(program); + + if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { + throw new Error(`Program link failed: ${gl.getProgramInfoLog(program)}`); + } + + return program; +} + +export function createOffscreenCanvasRenderingContext(): WebGL2RenderingContext { + const canvas = document.createElement('canvas'); + const gl = canvas.getContext('webgl2'); + + if (!gl) { + throw new Error('failed to create WebGL2 context'); + } + + if (!gl.getExtension('EXT_color_buffer_float')) { + throw new Error('failed to enable EXT_color_buffer_float extension'); + } + + return gl; +} + +export interface TextureParameters { + width: number; + height: number; + internalFormat: GLenum; + format: GLenum; + type: GLenum; + pixels?: ImageBitmap; +} + +export interface Texture extends TextureParameters { + object: WebGLTexture; +} + +export interface Framebuffer { + object: WebGLFramebuffer; + texture: Texture; +} + +export function createNonMippedLinearTexture(gl: WebGL2RenderingContext, params: TextureParameters): Texture { + const texture = gl.createTexture(); + if (!texture) { + throw new Error('failed to create texture'); + } + + const level = 0; + const internalFormat = params.internalFormat; + const border = 0; + const format = params.format; + const type = params.type; + const data = params.pixels; + + gl.bindTexture(gl.TEXTURE_2D, texture); + if (data) { + gl.texImage2D(gl.TEXTURE_2D, level, internalFormat, format, type, data); + } else { + gl.texImage2D(gl.TEXTURE_2D, level, internalFormat, params.width, params.height, border, format, type, null); + } + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + + return { object: texture, ...params }; +} + +export function createNonMippedLinearTextureFbo(gl: WebGL2RenderingContext, params: TextureParameters): Framebuffer { + const texture = createNonMippedLinearTexture(gl, params); + const fbo = gl.createFramebuffer(); + if (!fbo) { + throw new Error('failed to create framebuffer object'); + } + + gl.bindFramebuffer(gl.FRAMEBUFFER, fbo); + gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture.object, 0); + + return { object: fbo, texture }; +} + +export function getUniformLocation( + gl: WebGL2RenderingContext, + program: WebGLProgram, + name: string, +): WebGLUniformLocation { + const location = gl.getUniformLocation(program, name); + if (!location) { + throw new Error('failed to get uniform location'); + } + + return location; +}