Porting ShaderToy Chromakey example to P5.js

Question

I’m try to porting the shadertoy chromakey example to p5 with webcam as video source. After many time reading documentations of shaders, my code seems not working. I need some help.

I followed this guide to port the code for the p5

Fragment shader code:

#ifdef GL_ES
precision mediump float;
#endif

uniform sampler2D tex0;
uniform sampler2D tex1;

mat4 RGBtoYUV = mat4(0.257,  0.439, -0.148, 0.0,
                     0.504, -0.368, -0.291, 0.0,
                     0.098, -0.071,  0.439, 0.0,
                     0.0625, 0.500,  0.500, 1.0 );


vec4 chromaKey = vec4(0.05, 0.63, 0.14, 1);

vec2 maskRange = vec2(0.005, 0.26);


float colorclose(vec3 yuv, vec3 keyYuv, vec2 tol)
{
    float tmp = sqrt(pow(keyYuv.g - yuv.g, 2.0) + pow(keyYuv.b - yuv.b, 2.0));
    if (tmp < tol.x)
      return 0.0;
    else if (tmp < tol.y)
      return (tmp - tol.x)/(tol.y - tol.x);
    else
      return 1.0;
}


void main()
{
    vec2 fragPos =  gl_FragCoord.xy / iResolution.xy;
    vec4 texColor0 = texture(text0, fragPos);
    vec4 texColor1 = texture(text1, fragPos);

    vec4 keyYUV =  RGBtoYUV * chromaKey;
    vec4 yuv = RGBtoYUV * texColor0;

    float mask = 1.0 - colorclose(yuv.rgb, keyYUV.rgb, maskRange);
    gl_FragColor = max(texColor0 - mask * chromaKey, 0.0) + texColor1 * mask;
}

P5 sketch code:

let theShader;
let cam;

let img;

function preload(){

  theShader = loadShader('webcam.vert', 'webcam.frag');

  img = loadImage('http://www.quadrochave.com/wp-content/uploads/elementor/thumbs/nodulo_bannersite_ptodu%C3%A7%C3%A3o2-mpe2nvmu8s8o2uqcd7b2oh3mnuv9up05ubby33shz4.png');
}

function setup() {
  pixelDensity(1);

  createCanvas(windowWidth, windowHeight, WEBGL);
  noStroke();

  cam = createCapture(VIDEO);
  cam.size(windowWidth, windowHeight);

  cam.hide();
}

function draw() {
  // shader() sets the active shader with our shader
  shader(theShader);

  // passing cam as a texture
  theShader.setUniform('tex0', cam);
  theShader.setUniform('tex1', img);

  // rect gives us some geometry on the screen
  theShader.rect(0,0,width,height);

}

Test on Glitch

Shadertoy chromakey original fragment shader

Answer 1

The mayor issue is, that you didn't specify and set the uniform variable iResolution. But there are some more issues in the shader code (tex0 and tex1 rather than text0 and text1).

Fragment shader:

precision mediump float;

uniform sampler2D tex0;
uniform sampler2D tex1;
uniform vec2 iResolution;

mat4 RGBtoYUV = mat4(0.257,  0.439, -0.148, 0.0,
                        0.504, -0.368, -0.291, 0.0,
                        0.098, -0.071,  0.439, 0.0,
                        0.0625, 0.500,  0.500, 1.0 );


vec4 chromaKey = vec4(0.05, 0.63, 0.14, 1);

vec2 maskRange = vec2(0.005, 0.26);

float colorclose(vec3 yuv, vec3 keyYuv, vec2 tol)
{
    float tmp = sqrt(pow(keyYuv.g - yuv.g, 2.0) + pow(keyYuv.b - yuv.b, 2.0));
    if (tmp < tol.x)
        return 0.0;
    else if (tmp < tol.y)
        return (tmp - tol.x)/(tol.y - tol.x);
    else
        return 1.0;
}

void main()
{
    vec2 fragPos =  gl_FragCoord.xy / iResolution.xy;
    vec4 texColor0 = texture2D(tex0, fragPos);
    vec4 texColor1 = texture2D(tex1, fragPos);

    vec4 keyYUV =  RGBtoYUV * chromaKey;
    vec4 yuv = RGBtoYUV * texColor0;

    float mask = 1.0 - colorclose(yuv.rgb, keyYUV.rgb, maskRange);
    gl_FragColor = max(texColor0 - mask * chromaKey, 0.0) + texColor1 * mask;
}

Script:

let theShader;
let cam;
let img;

function setup() {
    createCanvas(windowWidth, windowHeight, WEBGL);

    theShader = loadShader('webcam.vert', 'webcam.frag');
    img = loadImage('http://www.quadrochave.com/wp-content/uploads/elementor/thumbs/nodulo_bannersite_ptodu%C3%A7%C3%A3o2-mpe2nvmu8s8o2uqcd7b2oh3mnuv9up05ubby33shz4.png');

    pixelDensity(1);
    noStroke();

    cam = createCapture(VIDEO);
    cam.size(windowWidth, windowHeight);

    cam.hide();
}

function draw() {
    // shader() sets the active shader with our shader
    shader(theShader);

    // passing cam as a texture
    theShader.setUniform('tex0', cam);
    theShader.setUniform('tex1', img);
    theShader.setUniform('iResolution', [width, height]);

    // rect gives us some geometry on the screen
    rect(0,0,width,height);
}

---

If the vertex shader provides the texture the coordinate:

// our vertex data
attribute vec3 aPosition;
attribute vec2 aTexCoord;

// lets get texcoords just for fun! 
varying vec2 vTexCoord;

void main() {
    // copy the texcoords
    vTexCoord = aTexCoord;

    // copy the position data into a vec4, using 1.0 as the w component
    vec4 positionVec4 = vec4(aPosition, 1.0);
    positionVec4.xy = positionVec4.xy * 2.0 - 1.0;

    // send the vertex information on to the fragment shader
    gl_Position = positionVec4;
}

then you can use this coordinate instead of gl_FragCoord.xy / iResolution.xy:

varying vec2 vTexCoord;

// [...]

void main() {
    vec2 fragPos = vTexCoord.xy;

    // [...]
}