Recuperating pixel data from fragment shader results in unexpected behavior

Question

I'm currently trying to use a fragment shader to transport some particle position data (and in the future, modify it). I have no problem sending the data to the shader using a sampler2D texture, but when I try to recuperate the data, My 20 particles suddenly have the wrong positions. I've printed as many outputs as possible and have minimized the code as much as I can yet still fail to understand where I'm wrong.

a reproducible minimized version is available on the p5js website here

Here is my sketch.js :

let theShader;
let shaderTexture;

let NUM_PARTICLES = 20;
let particleTexture;
let particles = [];

function preload(){
    theShader = loadShader('basic.vert', 'basic.frag');
}

function setup() {
  createCanvas(400, 400, WEBGL);
  
  // Initiate Particles
  for(let i = 0; i < NUM_PARTICLES;i++){
    particles.push(createVector(i*20,i*20,0));
  }
  
  // Initialize Shader
  shaderTexture = createGraphics(NUM_PARTICLES, 1, WEBGL);
  shaderTexture.noStroke();
  
  // Create Particle Texture
  particleTexture = createImage(NUM_PARTICLES, 1);
  
  // Fill Particle Texture
  particleTexture.loadPixels();
  for (let i = 0; i < NUM_PARTICLES; i++) {   
    particleTexture.pixels[i*4+0] = map(particles[i].x,0,width,0,255); // R
    particleTexture.pixels[i*4+1] = map(particles[i].y,0,height,0,255); // G
    particleTexture.pixels[i*4+2] = 0; // B
    particleTexture.pixels[i*4+3] = 255; // A
  }
  particleTexture.updatePixels();
}

function draw() {  
  translate(-width/2, -height/2);
  background(255);
  
  // Display Particles Before Modification
  for(let i = 0; i < NUM_PARTICLES;i++){
    circle(particles[i].x,particles[i].y,10); // draw circle at particle location
  }

  // Apply Texture
  shaderTexture.shader(theShader); // set shader
  theShader.setUniform('text', particleTexture); // send particleTexture to shader
  shaderTexture.rect(0,0,NUM_PARTICLES,1); // set rect to recieve shader out
  
  // Print Shader Output
  for(let i = 0; i < NUM_PARTICLES;i++){
    let newPos = shaderTexture.get(i, 0);
    print(newPos);
  }
  
  // Update and Display Particles
  for(let i = 0; i < NUM_PARTICLES;i++){
    let newPos = shaderTexture.get(i, 0);
    particles[i].x = map(newPos[0],0,255,0,width);
    particles[i].y = map(newPos[1],0,255,0,height);
    fill(255,0,0);
    circle(particles[i].x,particles[i].y,10);
  }

  noLoop();
}

and here is my fragment shader which should not be modifying anything :

#ifdef GL_ES
precision highp float;
#endif

uniform sampler2D text;

void main() {
  vec2 particle = texture2D(text, vec2(gl_FragCoord.x, gl_FragCoord.y)).xy;

  gl_FragColor = vec4(particle.x,particle.y,0.0,1.0); // R,G,B,A
}

also my vertex shader which is default :

#ifdef GL_ES
precision highp float;
#endif

attribute vec3 aPosition;

void main() {
  vec4 positionVec4 = vec4(aPosition, 1.0);

  positionVec4.xy = positionVec4.xy * 2.0 - 1.0; 

  gl_Position = positionVec4;
}

Rabbid76 · Accepted Answer · 2020-12-29T09:57:58.297

When looking up a texture with texture2D the texture coordinates must be specified in range [0.0, 1.0]. (0, 0) is the bottom left and (1, 1) is the top right. However gl_FragCoord.xy contains window coordinates, with top left (0.5, 0.5) and top right (width-0.5, height-0.5).
Hence you need to divide gl_FragCoord by the size of the viewport. Since you draw on a rectangle with the size NUM_PARTICLESx1 (width == NUM_PARTICLES, height = 1), you must divide gl_FragCoord.x by NUM_PARTICLES:

~~vec2 particle = texture2D(text, vec2(gl_FragCoord.x, gl_FragCoord.y)).xy;~~

vec2 particle = texture2D(text, vec2(gl_FragCoord.x/20.0, 0.0)).xy;

Alternatively you can add a size uniform to the fragment shader and divide gl_FragCoord.xy by size:

#ifdef GL_ES
precision highp float;
#endif

uniform sampler2D text;
uniform vec2 size;

void main() {
  
  vec2 particle = texture2D(text, gl_FragCoord.xy / size).xy;

  gl_FragColor = vec4(particle.x,particle.y,0.0,1.0); // R,G,B,A
}

Set the value of the size uniform by [NUM_PARTICLES, 1]:

// Apply Texture
shaderTexture.shader(theShader); // set shader
theShader.setUniform('text', particleTexture); // send particleTexture to shader
theShader.setUniform('size', [NUM_PARTICLES, 1]);
shaderTexture.rect(0,0,NUM_PARTICLES,1); // set rect to recieve shader out

If you are using OpenGL ES Shading Language 3.00 you have the option to use texelFetch to lookup the texture with integral pixel coordinates or get the size of the texture using textureSize.
Unfortunately p5.js (createCanvas()) doesn't seem to provide a WebGL 2.0 context, so this is not an option.

Recuperating pixel data from fragment shader results in unexpected behavior

1 Answers1