How to render OpenCV frames in OpenGL 3.2 GtkGLArea in Python?

Question

How do I render images captured by OpenCV in OpenGL 3.2 GtkGLArea in Python? Many of the examples online are 10 years old, outdated and are using OpenGL 2.1 or 1.1 examples.

The only way I can do this is through shaders. The only working example to get anything to draw in a GtkGLArea is this one:
Using Gtk GLArea in Python GTK3

I found this example in C++ and translated it into Python but I cannot get it to work or output anything.
How to Display Image using OpenGL

Here is my code, it does not display anything. What am I doing wrong?

Write Display Function called by OpenCV capture thread:

import os
import gi
gi.require_version("Gtk", "3.0")
from gi.repository import Gtk, GdkPixbuf, GLib

USE_OPENGL = True

def writeDisplay(uiBuilder, frame, imageDisplay):
    # Write Frame
    frame = cv.cvtColor(frame, cv.COLOR_BGR2RGB)

    if USE_OPENGL:
        # Render frame using OpenGL
        GLib.idle_add(imageDisplay.render, frame)

OpenGL Renderer GTK Widget

import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk
from OpenGL.GL import *
from OpenGL.GL import shaders
import numpy as np

VERTEX_SOURCE = '''
#version 330
layout (location=0) in vec3 position;
layout (location=1) in vec3 color;
layout (location=2) in vec2 texCoord;
out vec3 ourColor;
out vec2 TexCoord;
void main()
{
gl_Position = vec4(position,1.0);
ourColor = color;
TexCoord= vec2(texCoord.x,1.0-texCoord.y);
}'''

FRAGMENT_SOURCE ='''
#version 330
in vec3 ourColor;
in vec2 TexCoord;
out vec4 color;
uniform sampler2D ourTexture;
void main(){
color = texture(ourTexture , TexCoord);
};'''

recVertices = np.array([
    # Positions           Colors           Texture Coords
    0.5,  0.5, 0.0,   1.0, 0.0, 0.0,    1.0, 1.0,   # Top Right
    0.5, -0.5, 0.0,   0.0, 1.0, 0.0,    1.0, 0.0,   # Bottom Right
    -0.5, -0.5, 0.0,   0.0, 0.0, 1.0,   0.0, 0.0,   # Bottom Left
    -0.5,  0.5, 0.0,   1.0, 1.0, 0.0,   0.0, 1.0    # Top Left
], dtype=np.float32)

indices = np.array([
    0, 1, 3, # First Triangle
    1, 2, 3  # Second Triangle
])

def checkGlError(op: str):
    error = glGetError()
    if error is not None and error != 0:
        print("after %s() glError (0x%x)", op, error)

# Based on examples:
# https://stackoverflow.com/questions/42153819/how-to-load-and-display-an-image-in-opengl-es-3-0-using-c
# https://stackoverflow.com/questions/47565884/use-of-the-gtk-glarea-in-pygobject-gtk3
class OpenGLRenderer(Gtk.GLArea):
    def __init__(self):
        Gtk.GLArea.__init__(self)
        self.connect("realize", self.onRealize)
        self.connect("render", self.onRender)
        self.ctx = None
        self.frame = None
        self.area = None
        self.shaderProgram = None
        self.positionHandle = None
        self.textureId = None
        self.vao = None

    def onRealize(self, area):

        error = area.get_error()
        if error != None:
            print("your graphics card is probably too old : ", error)
        else:
            print(area, "realize... fine so far")

        self.ctx = self.get_context()
        self.ctx.make_current()

        print("OpenGL realized", self.ctx)

    def onRender(self, area, ctx):
        
        self.render(self.frame)
        return True

    def setupGraphics(self, width, height):

        if self.shaderProgram is None:
            # Load Shaders, Create program, Setup Graphics
            vertexShader = glCreateShader(GL_VERTEX_SHADER)
            glShaderSource(vertexShader, VERTEX_SOURCE)
            glCompileShader(vertexShader)

            pixelShader = glCreateShader(GL_FRAGMENT_SHADER)
            glShaderSource(pixelShader, FRAGMENT_SOURCE)
            glCompileShader(pixelShader)

            self.shaderProgram = glCreateProgram()
            glAttachShader(self.shaderProgram, vertexShader)
            glAttachShader(self.shaderProgram, pixelShader)
            glLinkProgram(self.shaderProgram)
            self.positionHandle = glGetAttribLocation(self.shaderProgram, "position")

        glViewport(0, 0, width, height)
    
    def initBuffers(self):
        # Initialize an buffer to store all the verticles and transfer them to the GPU
        self.vao = glGenVertexArrays(1) # Generate VAO
        vbos = glGenBuffers(1) # Generate VBO
        ebo = glGenBuffers(1) # Generate EPBO
        glBindVertexArray(self.vao) # Bind the Vertex Array

        glBindBuffer(GL_ARRAY_BUFFER, vbos) # Bind verticles array for OpenGL to use
        glBufferData(GL_ARRAY_BUFFER, len(recVertices), recVertices, GL_STATIC_DRAW)

        glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo) # Bind the indices for information about drawing sequence
        glBufferData(GL_ELEMENT_ARRAY_BUFFER, len(indices), indices, GL_STATIC_DRAW)
        
        # 1. set the vertex attributes pointers
        # Position Attribute
        glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(0))
        glEnableVertexAttribArray(0)
        # Color Attribute
        glVertexAttribPointer(1, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(3 * sizeof(GLfloat)))
        glEnableVertexAttribArray(1)
        # Texture Coordinate Attribute
        glVertexAttribPointer(2, 2, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(6 * sizeof(GLfloat)))
        glEnableVertexAttribArray(2)

        glBindVertexArray(0) # 3. Unbind VAO
    
    def generateTexture(self, frame):
        # Update Frame
        self.frame = frame

        # If we have a frame to display
        if frame is not None:
            # extract array from Image
            h, w, d = frame.shape

            # Generate Texture
            self.textureId = glGenTextures(1)
            glBindTexture(GL_TEXTURE_2D, self.textureId) # Bind our 2D texture so that following set up will be applied

            # Set texture wrapping parameter
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_MIRRORED_REPEAT)
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_MIRRORED_REPEAT)

            # Set texture Filtering parameter
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)

            glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, w, h, 0, GL_RGB, GL_UNSIGNED_BYTE, frame)
            glGenerateMipmap(GL_TEXTURE_2D)
            glBindTexture(GL_TEXTURE_2D, 0) # Unbind 2D textures

    def render(self, frame):

        # Set OpenGL Render Context
        if self.ctx is not None and frame is not None:
            self.ctx.make_current()

            # extract array from Image
            h, w, d = frame.shape

            # Initialize Graphics
            self.setupGraphics(w, h)

            # Generate Texture
            self.generateTexture(frame)

            # Clear Screen
            glClearColor(0, 0, 1, 1)
            glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)

            # Render Frame
            glUseProgram(self.shaderProgram)
            # checkGlError("glUseProgram")

            # glVertexAttribPointer(self.positionHandle, 2, GL_FLOAT, GL_FALSE, 0, recVertices)
            # checkGlError("glVertexAttribPointer")
            # glEnableVertexAttribArray(self.positionHandle)
            # checkGlError("glEnableVertexAttribArray")
            # glDrawArrays(GL_TRIANGLE_FAN, 0, 4)
            # checkGlError("glDrawArrays")
            glActiveTexture(GL_TEXTURE0)
            checkGlError("glActiveTexture")
            glBindTexture(GL_TEXTURE_2D, self.textureId)
            checkGlError("glBindTexture")
            mlocation = glGetUniformLocation(self.shaderProgram, "ourTexture")
            checkGlError("glGetUniformLocation")
            glUniform1i(mlocation, 0)
            checkGlError("glUniform1i")
            self.initBuffers()
            glBindVertexArray(self.vao)
            checkGlError("glBindVertexArray")
            glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0)
            
            # Queue Draw
            glFlush()
            self.queue_draw()

Answer 1

Here is the answer, this is tested with OpenCV frames which are a 3 dimentional array [width][height][RGB Color] which need to be flattened into a linear array. OpenGL will take the width and height of the image and traverse it properly. This guide was immensely helpful in getting this to work: https://open.gl/drawing

import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk
from OpenGL.GL import *
import numpy as np

VERTEX_SOURCE = '''
#version 330
layout (location=0) in vec3 position;
layout (location=1) in vec3 color;
layout (location=2) in vec2 texCoord;
out vec3 ourColor;
out vec2 TexCoord;
void main()
{
gl_Position = vec4(position,1.0);
ourColor = color;
TexCoord= vec2(texCoord.x,1.0-texCoord.y);
}'''

FRAGMENT_SOURCE ='''
#version 330
in vec3 ourColor;
in vec2 TexCoord;
out vec4 color;
uniform sampler2D ourTexture;
void main(){
color = texture(ourTexture, TexCoord);
};'''

recVertices = np.array([
    # Positions           Colors           Texture Coords
    1.0,  1.0, 0.0,   1.0, 0.0, 0.0,    1.0, 1.0,   # Top Right    0
    1.0, -1.0, 0.0,   0.0, 1.0, 0.0,    1.0, 0.0,   # Bottom Right 1
    -1.0, -1.0, 0.0,   0.0, 0.0, 1.0,   0.0, 0.0,   # Bottom Left  2
    -1.0,  1.0, 0.0,   1.0, 1.0, 0.0,   0.0, 1.0,   # Top Left     3
    1.0,  1.0, 0.0,   1.0, 0.0, 0.0,    1.0, 1.0,   # Top Right    4
], dtype=np.float32)

def checkGlError(op: str):
    error = glGetError()
    if error is not None and error != 0:
        print("after %s() glError (0x%x)", op, error)

# Based on examples:
# https://stackoverflow.com/questions/42153819/how-to-load-and-display-an-image-in-opengl-es-3-0-using-c
# https://stackoverflow.com/questions/47565884/use-of-the-gtk-glarea-in-pygobject-gtk3
class OpenGLRenderer(Gtk.GLArea):
    def __init__(self):
        Gtk.GLArea.__init__(self)
        self.connect("realize", self.onRealize)
        self.connect("render", self.onRender)
        self.ctx = None
        self.frame = None
        self.area = None
        self.shaderProgram = None
        self.positionHandle = None
        self.textureId = None
        self.vao = None
        self.vbos = None

    def onRealize(self, area):

        error = area.get_error()
        if error != None:
            print("your graphics card is probably too old : ", error)
        else:
            print(area, "realize... fine so far")

        self.ctx = self.get_context()
        self.ctx.make_current()

        print("OpenGL realized", self.ctx)

    def onRender(self, area, ctx):
        
        self.render(self.frame)
        return True

    def setupGraphics(self):

        if self.shaderProgram is None:
            # Load Shaders, Create program, Setup Graphics
            vertexShader = glCreateShader(GL_VERTEX_SHADER)
            glShaderSource(vertexShader, VERTEX_SOURCE)
            glCompileShader(vertexShader)
            status = glGetShaderiv(vertexShader, GL_COMPILE_STATUS)
            print("Compile vertexShader status: " + str(status == GL_TRUE))

            pixelShader = glCreateShader(GL_FRAGMENT_SHADER)
            glShaderSource(pixelShader, FRAGMENT_SOURCE)
            glCompileShader(pixelShader)
            status = glGetShaderiv(pixelShader, GL_COMPILE_STATUS)
            print("Compile vertexShader status: " + str(status == GL_TRUE))

            self.shaderProgram = glCreateProgram()
            glAttachShader(self.shaderProgram, vertexShader)
            glAttachShader(self.shaderProgram, pixelShader)
            glLinkProgram(self.shaderProgram)
            glBindFragDataLocation(self.shaderProgram, 0, "color")
            self.positionHandle = glGetAttribLocation(self.shaderProgram, "position")

            # Initalize Vertex Buffers
            self.initBuffers()
    
    def initBuffers(self):
        # Initialize an buffer to store all the verticles and transfer them to the GPU
        self.vao = glGenVertexArrays(1) # Generate VAO
        self.vbos = glGenBuffers(1) # Generate VBO
        glBindVertexArray(self.vao) # Bind the Vertex Array

        glBindBuffer(GL_ARRAY_BUFFER, self.vbos) # Bind verticles array for OpenGL to use
        glBufferData(GL_ARRAY_BUFFER, sizeof(GLfloat) * len(recVertices), recVertices, GL_STATIC_DRAW)
        
        # 1. set the vertex attributes pointers
        # Position Attribute
        glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(0))
        glEnableVertexAttribArray(0)
        # Color Attribute
        glVertexAttribPointer(1, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(3 * sizeof(GLfloat)))
        glEnableVertexAttribArray(1)
        # Texture Coordinate Attribute
        glVertexAttribPointer(2, 2, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(6 * sizeof(GLfloat)))
        glEnableVertexAttribArray(2)

        glBindVertexArray(0) # 3. Unbind VAO
    
    def generateTexture(self, frame):
        # Update Frame
        self.frame = frame

        # Delete previous textures to avoid memory leak
        if self.textureId is not None:
            glDeleteTextures(1, [self.textureId])

        # If we have a frame to display
        if frame is not None:
            # extract array from Image
            h, w, d = frame.shape

            # Frame is a 3 dimentional array where shape eg. (1920, 1080, 3)
            # Where it is w, h, and 3 values for color
            # https://www.educba.com/numpy-flatten/
            pixels = frame.flatten(order = 'C')

            # Generate Texture
            self.textureId = glGenTextures(1)
            glBindTexture(GL_TEXTURE_2D, self.textureId) # Bind our 2D texture so that following set up will be applied

            # Set texture wrapping parameter
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE)
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)

            # Set texture Filtering parameter
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR)
            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)

            glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, w, h, 0, GL_RGB, GL_UNSIGNED_BYTE, pixels)
            glGenerateMipmap(GL_TEXTURE_2D)
            glBindTexture(GL_TEXTURE_2D, 0) # Unbind 2D textures

    def render(self, frame):

        # Set OpenGL Render Context
        if self.ctx is not None and frame is not None:
            self.ctx.make_current()

            # Initialize Graphics
            self.setupGraphics()

            # Generate Texture
            self.generateTexture(frame)

            # Clear Screen
            glClearColor(0, 0, 1, 1)
            glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)

            # Use Shader Program, Bind Vertex Array and Texture
            glUseProgram(self.shaderProgram)
            checkGlError("glUseProgram")
            glActiveTexture(GL_TEXTURE0)
            checkGlError("glActiveTexture")
            glBindTexture(GL_TEXTURE_2D, self.textureId)
            checkGlError("glBindTexture")
            mlocation = glGetUniformLocation(self.shaderProgram, "ourTexture")
            checkGlError("glGetUniformLocation")
            glUniform1i(mlocation, 0)
            checkGlError("glUniform1i")
            glBindVertexArray(self.vao)
            checkGlError("glBindVertexArray")

            # Render Frame
            glDrawArrays(GL_TRIANGLES, 0, 3)
            glDrawArrays(GL_TRIANGLES, 2, 3)
            
            # Queue Draw
            glFlush()
            self.queue_draw()

Rendering Textures in GtkGLArea in Python Full Example

import gi

gi.require_version("Gtk", "3.0")
from gi.repository import Gtk
import numpy as np
from OpenGL.GL import *
from OpenGL.GL import shaders

FRAGMENT_SOURCE ='''
#version 330
in vec3 Color;
in vec2 Texcoord;
out vec4 outColor;
uniform sampler2D imageTexture;
void main()
{
    outColor = texture(imageTexture, Texcoord);
}'''

VERTEX_SOURCE = '''
#version 330
layout (location=0) in vec3 position;
layout (location=1) in vec3 color;
layout (location=2) in vec2 texcoord;
out vec3 Color;
out vec2 Texcoord;
void main()
{
    Color = color;
    Texcoord = texcoord;
    gl_Position = vec4(position, 1.0);
}'''

def on_realize(self, area):        
    # We need to make the context current if we want to
    # call GL API
    area.make_current()

def on_render(area, context):
    print("%s\n", glGetString(GL_VERSION))
    area.make_current()

    ############################################
    # Init Shaders
    ############################################
    glClearColor(0, 0, 0, 1)
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
    VERTEX_SHADER_PROG = shaders.compileShader(VERTEX_SOURCE, GL_VERTEX_SHADER)
    FRAGMENT_SHADER_PROG = shaders.compileShader(FRAGMENT_SOURCE, GL_FRAGMENT_SHADER)
    shaderProgram = shaders.compileProgram(VERTEX_SHADER_PROG, FRAGMENT_SHADER_PROG)
    
    ############################################
    # Init Buffers
    ############################################
    # Create a new VAO (Vertex Array Object) and bind it
    vao = glGenVertexArrays(1)
    glBindVertexArray(vao)
    # Generate buffers to hold our vertices
    vertex_buffer = glGenBuffers(1)
    glBindBuffer(GL_ARRAY_BUFFER, vertex_buffer)
    # Send the data over to the buffer
    vertices = np.array([
        # Positions      Color           Texchords 
        1.0,  1.0, 0.0,  0.0, 1.0, 0.0,  1.0, 0.0, # Top Right       0
        1.0, -1.0, 0.0,  0.0, 0.0, 1.0,  1.0, 1.0, # Bottom Right    1
        -1.0, -1.0, 0.0, 1.0, 1.0, 1.0,  0.0, 1.0, # Bottom Left     2
        -1.0,  1.0, 0.0, 1.0, 0.0, 0.0,  0.0, 0.0, # Top Left        3
        1.0,  1.0, 0.0,  0.0, 1.0, 0.0,  1.0, 0.0, # Top Right       4
    ], dtype=np.float32)
    
    size = sizeof(GLfloat) * len(vertices)
    glBufferData(GL_ARRAY_BUFFER, size, vertices, GL_STATIC_DRAW)

    # Specify the layout of the vertex data
    posAttrib = glGetAttribLocation(shaderProgram, "position")
    glEnableVertexAttribArray(posAttrib)
    glVertexAttribPointer(posAttrib, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(0))

    colAttrib = glGetAttribLocation(shaderProgram, "color")
    glEnableVertexAttribArray(colAttrib)
    glVertexAttribPointer(colAttrib, 3, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(3 * sizeof(GLfloat)))

    texAttrib = glGetAttribLocation(shaderProgram, "texcoord")
    glEnableVertexAttribArray(texAttrib)
    glVertexAttribPointer(texAttrib, 2, GL_FLOAT, GL_FALSE, 8 * sizeof(GLfloat), ctypes.c_void_p(6 * sizeof(GLfloat)))

    # Unbind the VAO first (Important)
    glBindVertexArray(0)

    ############################################
    # Render
    ############################################
    glBindBuffer(GL_ARRAY_BUFFER, 0)
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
    glUseProgram(shaderProgram)
    glBindVertexArray(vao)

    # Load Textures
    width = 2
    height = 2
    textureId = glGenTextures(1)
    glActiveTexture(GL_TEXTURE0)
    glBindTexture(GL_TEXTURE_2D, textureId)

    # Black/white checkerboard
    pixels = [
        0.0, 0.0, 0.0,   1.0, 1.0, 1.0,
        1.0, 1.0, 1.0,   0.0, 0.0, 0.0
    ]

    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, width, height, 0, GL_RGB, GL_FLOAT, pixels)
    glUniform1i(glGetUniformLocation(shaderProgram, "imageTexture"), 0)
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE)
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)

    glDrawArrays(GL_TRIANGLES, 0, 3)
    glDrawArrays(GL_TRIANGLES, 2, 3)
    glBindVertexArray(0)
    glUseProgram(0)

    # we completed our drawing; the draw commands will be
    # flushed at the end of the signal emission chain, and
    # the buffers will be drawn on the window
    return True

win = Gtk.Window()
area = Gtk.GLArea()
#area.set_required_version(2, 1)
#major, minor = area.get_required_version()
#print("Version " + str(major) + "." + str(minor))
area.connect('render', on_render)
area.connect('realize', on_realize)
win.connect("destroy", Gtk.main_quit)
win.add(area)
win.show_all()
Gtk.main()