Why stencil test does not discard fragments?

Question

I want to run stencil test to outline cubes. It is from learnopengl https://learnopengl.com/Advanced-OpenGL/Stencil-testing.

A cubes that do outlining should have discarded fragments that are on a textured cubes. It does not happen, the outlining cubes cover the whole area.

static const char* cubeVtx = R"(
#version 460 core
layout (location = 0) in vec3 a_position;
layout (location = 1) in vec2 a_uv;

out vec2 v_uv;

uniform mat4 model;
uniform mat4 view;
uniform mat4 projection;

void main()
{
    v_uv = a_uv;    
    gl_Position = projection * view * model * vec4(a_position, 1.0f);
}

)";

static const char* cubeFrag = R"(
#version 460 core

layout (location = 0) out vec4 f_color;

in vec2 v_uv;

uniform sampler2D u_texture1;

void main()
{    
    f_color = texture(u_texture1, v_uv);
}

)";

static const char* singleColorVtx = R"(
#version 460 core
layout (location = 0) in vec3 a_position;

uniform mat4 model;
uniform mat4 view;
uniform mat4 projection;

void main()
{   
    gl_Position = projection * view * model * vec4(a_position, 1.0f);
}

)";

static const char* singleColorFrag = R"(
#version 460 core

layout (location = 0) out vec4 f_color;

void main()
{    
    f_color = vec4(1.0f, 0.0f, 0.0f, 1.0f);
}

)";

void Cube::InitAPI()
{
    shader = Shader::Create();
    ShaderProgram program[2] = {
        shader->ShaderFromSource(cubeVtx, ShaderType::Vertex),
        shader->ShaderFromSource(cubeFrag, ShaderType::Fragment)
    };
    shader->End(program, 2);

    singleColorShader = Shader::Create();
    ShaderProgram singleColorProgram[2] = {
        singleColorShader->ShaderFromSource(singleColorVtx, ShaderType::Vertex),
        singleColorShader->ShaderFromSource(singleColorFrag, ShaderType::Fragment)
    };
    singleColorShader->End(singleColorProgram, 2);

    glCreateVertexArrays(1, &vao);
    glBindVertexArray(vao);
    glCreateBuffers(1, &vbo);
    glBindBuffer(GL_ARRAY_BUFFER, vbo);
    glNamedBufferData(vbo, sizeof(cubeVertices), cubeVertices, GL_STATIC_DRAW);

    glEnableVertexAttribArray(0);
    glEnableVertexAttribArray(1);
    glVertexAttribPointer(0, 3, GL_FLOAT, false, 5 * sizeof(float), 0);
    glVertexAttribPointer(1, 2, GL_FLOAT, false, 5 * sizeof(float), (void*)(3 * sizeof(float)));

    glCreateVertexArrays(1, &planeVao);
    glBindVertexArray(planeVao);
    glCreateBuffers(1, &planeVbo);
    glBindBuffer(GL_ARRAY_BUFFER, planeVbo);
    glNamedBufferData(planeVbo, sizeof(planeVertices), planeVertices, GL_STATIC_DRAW);

    glEnableVertexAttribArray(0);
    glEnableVertexAttribArray(1);
    glVertexAttribPointer(0, 3, GL_FLOAT, false, 5 * sizeof(float), 0);
    glVertexAttribPointer(1, 2, GL_FLOAT, false, 5 * sizeof(float), (void*)(3 * sizeof(float)));

    texture = Texture::LoadPNG("Media/container2");
    planeTexture = Texture::LoadPNG("Media/matrix");

    glEnable(GL_DEPTH_TEST);
    glDepthFunc(GL_LESS);
    glEnable(GL_STENCIL_TEST);
    glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
    glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);

    shader->UseProgram();
    shader->Uniform1i("u_texture1", 0);
}

void Cube::Render()
{
    singleColorShader->UseProgram();
    glm::mat4 model = glm::mat4(1.0f);
    glm::mat4 view = camera.GetViewMatrix();
    glm::mat4 projection = glm::perspective(glm::radians(camera.Zoom), (float)1280.0f / (float)720.0f, 0.1f, 100.0f);
    singleColorShader->UniformMatrix4fv("view", view);
    singleColorShader->UniformMatrix4fv("projection", projection);

    shader->UseProgram();
    shader->UniformMatrix4fv("view", view);
    shader->UniformMatrix4fv("projection", projection);

    glStencilMask(0x00);
    glBindVertexArray(planeVao);
    planeTexture->Active();
    planeTexture->Bind();
    glDrawArrays(GL_TRIANGLES, 0, 6);

    glStencilFunc(GL_ALWAYS, 1, 0xFF);
    glStencilMask(0xFF);

    glBindVertexArray(vao);
    model = glm::translate(model, glm::vec3(-1.0f, 0.0f, -1.0f));
    shader->UniformMatrix4fv("model", model);
    texture->Active();
    texture->Bind();
    glDrawArrays(GL_TRIANGLES, 0, 36);

    model = glm::mat4(1.0f);
    model = glm::translate(model, glm::vec3(2.0f, 0.0f, 0.0f));
    shader->UniformMatrix4fv("model", model);
    glDrawArrays(GL_TRIANGLES, 0, 36);

    glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
    glStencilMask(0x00);
    glDisable(GL_DEPTH_TEST);
    singleColorShader->UseProgram();
    float scale = 1.1;

    glBindVertexArray(vao);
    model = glm::mat4(1.0f);
    model = glm::translate(model, glm::vec3(-1.0f, 0.0f, -1.0f));
    model = glm::scale(model, glm::vec3(scale, scale, scale));
    shader->UniformMatrix4fv("model", model);
    glDrawArrays(GL_TRIANGLES, 0, 36);

    model = glm::mat4(1.0f);
    model = glm::translate(model, glm::vec3(2.0f, 0.0f, 0.0f));
    model = glm::scale(model, glm::vec3(scale, scale, scale));
    shader->UniformMatrix4fv("model", model);
    glDrawArrays(GL_TRIANGLES, 0, 36);

    glBindVertexArray(0);

    glStencilMask(0xFF);
    glEnable(GL_DEPTH_TEST);
}

Test::Test()
{
    //tried many patterns and stencil values

    SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK, SDL_GL_CONTEXT_PROFILE_CORE);
    SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 4);
    SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 6);
    SDL_GL_SetAttribute(SDL_GL_DOUBLEBUFFER, 1);

    SDL_GL_SetAttribute(SDL_GL_STENCIL_SIZE, 8);
    window = SDL_CreateWindow("window", SDL_WINDOWPOS_CENTERED, SDL_WINDOWPOS_CENTERED, 1280, 720, SDL_WINDOW_OPENGL);

    context = SDL_GL_CreateContext(window);
    SDL_GL_MakeCurrent(window, context);

    glewExperimental = true;
    glewInit();

    cube.InitAPI();
}

void Test::MainLoop()
{
    while (!quit) {
        while (SDL_PollEvent(&event)) {
            if (event.type == SDL_QUIT) {
                quit = true;
            }
        }

        glClearColor(0.5f, 0.5f, 0.5f, 1.0f);
        glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);

        cube.Render();

        SDL_GL_SwapWindow(window);
    }
}

I still quite do not get the depth test thing and when I started thinking that I am getting stencil test, it all messed up, because it does not work.

Yes, I do clear color, depth and stencil buffer, somewhere else, but before Render() method.

What I got:

What I expect is a red outline, not the whole cube

Could you please explain like I were an idiot why it does not discard fragments that are 1

In general your set up should work. Are you sure that the default frambeuffer even have a stencil buffer? This would explain the issue, because if the there is no stencil buffer, then the value which is read form the stencil buffer is 0 and `glStencilFunc(GL_NOTEQUAL, 1, 0xFF)` will never fail. — Rabbid76, Jun 26 '19 at 19:33
Yes, the first at all what I did was to create? stencil buffer with ` SDL_GL_SetAttribute( SDL_GL_STENCIL_SIZE, 24);` — Shout, Jun 26 '19 at 20:08
Probably a stencil size of 24 won't work. Try `SDL_GL_SetAttribute(SDL_GL_STENCIL_SIZE, 8);` — Rabbid76, Jun 26 '19 at 20:09
Still the same result, can I check somehow if stencil buffer size is set correctly? — Shout, Jun 26 '19 at 20:34
`SDL_GL_SetAttribute` is called before `SDL_CreateWindow`, isn't it? — Rabbid76, Jun 26 '19 at 20:38
No, you've to call it before `SDL_CreateWindow` or `SDL_SetVideoMode.`, else it won't have any effect. — Rabbid76, Jun 26 '19 at 20:52

Rabbid76 · Answer 1 · 2019-06-26T21:47:13.117

1

Call SDL_Init, before calling any other SDL-function

SDL_Init(SDL_INIT_EVERYTHING);

See the documentation of SDL_Init():

Use this function to initialize the SDL library. This must be called before using most other SDL functions.

To make the stencil buffer work, the SDL_GL_STENCIL_SIZE attribute has to be set. By default this attribute is 0. In general a stencil buffer has 8 bits. e.g.:

SDL_GL_SetAttribute(SDL_GL_STENCIL_SIZE, 8);

SDL_GL_SetAttribute has to be called before SDL_CreateWindow else it won't have any effect.

See the documentation of SDL_GL_SetAttribute:

Use this function to set an OpenGL window attribute before window creation.

edited Jun 26 '19 at 21:47

answered Jun 26 '19 at 20:56

Rabbid76

202,892
27
131
174

That makes sense, but it also does not work, though :P. Check updated code. – Shout Jun 26 '19 at 21:12
@Shout Note that all attributes have to be set before `SDL_CreateWindow`. – Rabbid76 Jun 26 '19 at 21:19
Ok, ok, I see, but it does not make any changes. Still stencil buffer does not work. – Shout Jun 26 '19 at 21:21
@Shout I can't reproduce the issue. What is the return value of `SDL_GL_SetAttribute(SDL_GL_STENCIL_SIZE, 8)`? It should be 0. – Rabbid76 Jun 26 '19 at 21:29
attribute could not be set - `-1`. Imma check real quick what does it mean. It's wierd. – Shout Jun 26 '19 at 21:32

score 0 · Accepted Answer · answered Jun 26 '19 at 21:38

0

The original code calls SDL_Init after SDL_GL_SetAttribute() The code above, does not even call SDL_Init. Initializing SDL fixes everything.

answered Jun 26 '19 at 21:38

Shout

338
5
14

Why stencil test does not discard fragments?

2 Answers2