OpenGL nested stencils

Question

I want to get nested stencils to work in OpenGL. I call them masks from now on.

So the stencil buffer is cleared to all 0's. I make my first mask, the grey area.

Now that have to be all 1's in the stencil buffer, and normal drawing is not allowed to happen outside that mask.

Now I create a second mask which is a child of the first one: It would only update the stencil buffer if the area is inside the area of the previous mask:

Same if we would make another child mask, we would only increment if it was inside the parent mask.

Now if we would end the last mask (the green one), we can decrement the values where they where equal to the mask depth.

Ok that is the idea, but I have been stuck on getting this to work for weeks (not full time...).

Either masking does not work, or it does not work as expected. Here I provide code that makes the most sense to me.

I use processing, which can be downloaded here https://processing.org/ (download and run, simple as that).

PGL pgl;


Rect current_rect;


void setup() {
  size(600, 600, P3D);
  noStroke();

  // will have a depth of 0, equal to the stencil buffer after a clear
  current_rect = make_rect(null, 0, 0, width, height);
}

class Rect {
  float x;
  float y;
  float w;
  float h;
  ArrayList<Rect> children = new ArrayList<Rect>();
  Rect parent;
  int mask_depth;
}

Rect make_rect(Rect parent, float x, float y, float w, float h) {
  Rect r = new Rect();
  r.x = x;
  r.y = y;
  r.w = w;
  r.h = h;
  if (parent != null) {
    r.parent = parent;
    r.mask_depth = parent.mask_depth + 1;
    parent.children.add(r);
  }
  return r;
}



void draw() {

  if (frameCount == 2) noLoop();

  println();

  pgl = beginPGL();

  pgl.enable(PGL.STENCIL_TEST);
  pgl.clear(PGL.STENCIL_BUFFER_BIT);

  background(150, 80, 70);

  begin_mask(100, 100, 400, 400);

  fill(150);
  rect(100, 100, 400, 400);
  // SHOULD NOT FALL OUTSIDE OF THE MASK!
  debug_text(" A ");

  // disabled for now cause the above begin_mask
  // goes already wrong
  if (false) {
    begin_mask(50, 150, 500, 100); // B
    fill(100);
    rect(50, 150, 500, 100);
    debug_text(" B ");

    if (true) {
      begin_mask(200, 50, 100, 500); // C
      fill(50);
      rect(200, 50, 100, 500);
      debug_text(" C ");
      end_mask(); // C
    }


    end_mask(); // B

    // why is the one from above (C) on this one?
    // disable this one to see what I mean
    if (true) {
      begin_mask(50, 350, 500, 100); // D
      fill(75);
      rect(50, 350, 500, 100);
      fill(255);
      debug_text(" D ");
      end_mask(); // D
    }
  }

  end_mask(); // A


  flush();

  endPGL();
}




void begin_mask(float x, float y, float w, float h) {
  current_rect = make_rect(current_rect, x, y, w, h);

  flush();
  pgl.colorMask(false, false, false, false);
  pgl.depthMask(false);
  pgl.stencilOp(PGL.KEEP, PGL.KEEP, PGL.INCR);
  println("increment stencil when stencil is equal to: "+current_rect.parent.mask_depth);
  pgl.stencilFunc(PGL.EQUAL, current_rect.parent.mask_depth, 0xFF);

  // write to stencil buffer
  noStroke();
  fill(0);
  rect(x, y, w, h);
  flush();

  enable_normal_draw_mode();
}



void enable_normal_draw_mode() {
  pgl.stencilMask(0x00);
  println("normal write when stencil depth is: "+(current_rect.mask_depth));
  pgl.stencilFunc(PGL.GEQUAL, current_rect.mask_depth, 0xff);
  pgl.stencilOp(PGL.KEEP, PGL.KEEP, PGL.KEEP);
  pgl.colorMask(true, true, true, true);
  pgl.depthMask(true);
  flush();

  println("enable_normal_draw_mode "+current_rect.mask_depth);
}



void end_mask() {
  // decrement stencil mask as if we never existed
  pgl.stencilMask(0xff);
  pgl.stencilFunc(PGL.GEQUAL, current_rect.mask_depth, 0xff);
  pgl.stencilOp(PGL.KEEP, PGL.KEEP, PGL.DECR);
  pgl.colorMask(false, false, false, false);
  pgl.depthMask(false);
  noStroke();
  fill(0);
  rect(current_rect.x, current_rect.y, current_rect.w, current_rect.h);
  flush();

  current_rect = current_rect.parent;

  enable_normal_draw_mode();
}






void debug_text(String s) {
  fill(255);
  text(xxx(s), 0, 0);
}

String xxx(String to_repeat) {
  String result = "";
  String line = "";
  for (int x = 0; x < 50; x++) {
    line += to_repeat;
  }
  for (int y = 0; y < 50; y++) {
    result += line + "\n";
  }
  return result;
}