1

I'm new to Vispy and opengl in general. I've taken to adapt the realtime_signals demo to my datasets. The data I work with is non-stationary and usually exhibits a trend of sorts. As a consequence, zooming into the data usually doesn't work well since drifting values fall out of the window.

I'm trying to add min-max normalization along the y-axis to the example code. That way, whatever my zoom level is, data should remain centered on the window. However, my solution produces glitches I can't explain.

from vispy import gloo
from vispy import app
import numpy as np
import math
import numpy.ma as ma



#Data
num_samples = 1000
num_features = 3
df_raw = np.reshape(1+(np.random.normal(size=num_samples*num_features, loc=[0], scale=[0.01])), [num_samples, num_features]).cumprod(0).astype('float32')
df_raw_std = 1+((df_raw-np.min(df_raw,0))*2)/(np.min(df_raw,0)-np.max(df_raw,0))

# Generate the signals as a (num_features, num_samples) array. 320x1000
y = df_raw_std.transpose()

# Signal 2D index of each vertex (row and col) and x-index (sample index
# within each signal).
index_col = np.c_[np.tile(np.arange(num_samples), num_features),np.repeat(np.arange(num_features), num_samples)].astype(np.float32)
y_flat = np.reshape(y,y.shape[0]*y.shape[1])
index_y_scaled_orig = np.c_[-1 + 2*(index_col[:,0] / num_samples),y_flat].astype(np.float32)
index_y_scaled = index_y_scaled_orig.copy()

index_min = np.c_[np.tile(np.arange(num_samples), num_features),np.repeat(1, num_samples*num_features)].astype(np.float32)
index_max = np.c_[np.tile(np.arange(num_samples), num_features),np.repeat(1, num_samples*num_features)].astype(np.float32)


#This is called once for each vertex
VERT_SHADER = """
#version 120
//scaling. Running minimum and maximum of visible time series
attribute float index_min;
attribute float index_max;

// y coordinate of the position.
attribute float y;

// row, and time index.
attribute vec2 index_col;

// 2D scaling factor (zooming).
uniform vec2 scale;
uniform vec2 num_features;

// Number of samples per signal.
uniform float num_samples;

// for fragment shader
varying vec2 v_index;

// Varying variables used for clipping in the fragment shader.
varying vec2 v_position;
varying vec4 v_ab;

void main() {
    float nrows = num_features.x;

    // Compute the x coordinate from the time index
    float x = -1 + 2 * (index_col.x / (num_samples-1));

    //0 is zoom from center. 1 is zoom on the right. -1 is zoom on the left. WE should map mouse x pos to this.
    float zoom_x_pos = 0.0;

    // RELATIVE LINE POSITION
    // =============================
    // Manipulate x/y position here?
    // =============================
    // vec2 position = vec2(x - (1 - 1 / scale.x)*zoom_x_pos,y); // DEACTIVATED SCALING, NICE PLOTS EMERGE
    vec2 position = vec2(x - (1 - 1 / scale.x)*zoom_x_pos,(y-index_min)/(index_max-index_min)); //SCALING, GLITCHY

    // SPREAD
    //does not scale the x pos, just the y pos by an equal amount per row
    float spread = 1;
    vec2 a = vec2(spread, spread/nrows);

    // LOCATION
    vec2 b = vec2(0, -1 + 2*(index_col.y+.5) / nrows);

    // COMBINE RELATIVE LINE POSITION + SPREAD + LOCATION
    gl_Position = vec4(a*scale*position+b, 0.0, 1.0);

    // WRAP UP
    v_index = index_col;
    // For clipping test in the fragment shader.
    v_position = gl_Position.xy;
    v_ab = vec4(a, b);
}
"""

FRAG_SHADER = """
#version 120
varying vec2 v_index;
varying vec2 v_position;
varying vec4 v_ab;
void main() {
    gl_FragColor = vec4(1., 1., 1., 1.);
    // Discard the fragments between the signals (emulate glMultiDrawArrays).

    if (fract(v_index.y) > 0.)
        discard;

    // Clipping test.
    vec2 test = abs((v_position.xy-v_ab.zw)/v_ab.xy);
    if ((test.x > 1) || (test.y > 1))
        discard;
}
"""


class Canvas(app.Canvas):
    def __init__(self):
        app.Canvas.__init__(self, title='Use your wheel to zoom!',
                            keys='interactive')
        self.program = gloo.Program(VERT_SHADER, FRAG_SHADER)
        self.program['y'] = y.reshape(-1, 1)
        self.program['index_col'] = index_col
        self.program['scale'] = (1., 1.)
        self.program['num_features'] = (num_features, 1)
        self.program['num_samples'] = num_samples
        self.program['index_min'] = index_min[:,0].reshape(-1, 1)
        self.program['index_max'] = index_max[:,0].reshape(-1, 1)
        gloo.set_viewport(0, 0, *self.physical_size)

        self._timer = app.Timer('auto', connect=self.on_timer, start=True)

        gloo.set_state(clear_color='black', blend=True,
                       blend_func=('src_alpha', 'one_minus_src_alpha'))

        self.show()

    def on_resize(self, event):
        gloo.set_viewport(0, 0, *event.physical_size)

    def on_mouse_wheel(self, event):
        dx = np.sign(event.delta[1]) * .05
        scale_x, scale_y = self.program['scale']

        index_y_scaled[:,0] = index_y_scaled_orig[:,0] * scale_x
        index_y_scaled[:, 1] = index_y_scaled_orig[:, 1] * scale_x
        valid = ((index_y_scaled[:,0]>-1)*(index_y_scaled[:,0]<1))

        index_y_scaled_reshaped = (np.reshape(index_y_scaled[:, 1],[num_features,num_samples]))
        shown = ma.masked_array(index_y_scaled_reshaped, mask=np.logical_not(valid))
        runmin = np.array(np.min(shown, 1))
        runmax = np.array(np.max(shown, 1))
        index_min[:, 1] = np.repeat(runmin, num_samples)
        index_max[:, 1] = np.repeat(runmax, num_samples)

        print(scale_x)
        print(runmin)
        print(runmax)

        self.program['index_min'] = index_min[:,1].reshape(-1, 1)
        self.program['index_max'] = index_max[:,1].reshape(-1, 1)
        #print(self.program['print_position'])

        scale_x_new, scale_y_new = (scale_x * math.exp(1.0*dx),
                                    scale_y * math.exp(1.0*dx))
        #print(scale_x_new)
        self.program['scale'] = (max(1, scale_x_new), max(1, scale_y_new))
        self.update()

    def on_timer(self, event):
        """Add some data at the end of each signal (real-time signals)."""
        self.program['y'].set_data(y.ravel().astype(np.float32)) #(10920,)
        self.update()

    def on_draw(self, event):
        gloo.clear()
        self.program.draw('line_strip')

if __name__ == '__main__':
    c = Canvas()
    app.run()
  1. What am I doing wrong? i am "estimating" the visible line selection outside opengl and passing scale correction parameters to the opengl pipeline. Yet I get obvious visual glitches as well as distorted lines
  2. Is there a smarter way of approaching this problem in in vispy? Perhaps a way to solve the normalization in the fragment shader or via camera tricks?
user3641187
  • 405
  • 5
  • 10

1 Answers1

1

Turns out the glitches were from a bug in computing the normalization parameters outside opengl. I've posted the correct code below - note that the graph is normalizing, adapting to zoom values,

Still, I am not happy with my current solution. It requires the normalization parameters to still be computed outside opengl, potentially massively slowing down execution once I move to higher data amounts and many lineplots to visualize. I understand that it is not possible to do this in the vertex shader since it only operates over individual vertices - normalization requires knowledge of the relative position of all other vertices in a given lineplot.

I wonder however if it is possible to achieve normalization - which is just a linear affine scale and transform of a line object - for example within the geometry shader. I am still new to opengl, but if I understand the pipeline correctly it will require each lineplot (3 in my example code) to be defined in its own vertex buffer primitive. I can then consume these primitives in a geometry shader, iterate over the vertices of a given lineplot using a loop over gl_in, compute the overall min and max and then translate and scale the position of each vertex in gl_in.

Is this possible with vispy? I know I can define a geometry shader, but I have trouble actually creating individual primitives for each line plot. In my example code, i think I am treating all 3 line plots as a single primitive.

from vispy import gloo
from vispy import app
import numpy as np
import math
import numpy.ma as ma


import matplotlib.pyplot as plt
import pandas as pd


#Data
num_samples = 10000
num_features = 3
df_raw = np.reshape(1+(np.random.normal(size=num_samples*num_features, loc=[0], scale=[0.01])), [num_samples, num_features]).cumprod(0).astype('float32')
df_raw_std = 1+((df_raw-np.min(df_raw,0))*2)/(np.min(df_raw,0)-np.max(df_raw,0))

# Generate the signals as a (num_features, num_samples) array. 320x1000
y = df_raw_std.transpose()

# Signal 2D index of each vertex (row and col) and x-index (sample index
# within each signal).
index_col = np.c_[np.tile(np.arange(num_samples), num_features),np.repeat(np.arange(num_features), num_samples)].astype(np.float32)
y_flat = y.flatten()

index_y_scaled_orig = np.c_[-1 + 2*(index_col[:,0] / num_samples),y_flat].astype(np.float32)
index_y_scaled = index_y_scaled_orig.copy()

index_min = np.c_[np.tile(np.arange(num_samples), num_features),np.repeat(1, num_samples*num_features)].astype(np.float32)
index_max = np.c_[np.tile(np.arange(num_samples), num_features),np.repeat(1, num_samples*num_features)].astype(np.float32)


#This is called once for each vertex
VERT_SHADER = """
#version 120
//scaling. Running minimum and maximum of visible time series
attribute float index_min;
attribute float index_max;

// y coordinate of the position.
attribute float y;

// row, and time index.
attribute vec2 index_col;

// 2D scaling factor (zooming).
uniform vec2 scale;
uniform vec2 num_features;

// Number of samples per signal.
uniform float num_samples;

// for fragment shader
varying vec2 v_index;

// Varying variables used for clipping in the fragment shader.
varying vec2 v_position;
varying vec4 v_ab;

void main() {
    float nrows = num_features.x;

    // Compute the x coordinate from the time index
    float x = -1 + 2 * (index_col.x / (num_samples-1));

    //0 is zoom from center. 1 is zoom on the right. -1 is zoom on the left. WE should map mouse x pos to this.
    float zoom_x_pos = 0.0;

    // RELATIVE LINE POSITION
    // =============================
    // Manipulate x/y position here?
    // =============================
    // vec2 position = vec2(x - (1 - 1 / scale.x)*zoom_x_pos,y); // DEACTIVATED SCALING, NICE PLOTS EMERGE
    vec2 position = vec2(x - (1 - 1 / scale.x)*zoom_x_pos,y); //SCALING, GLITCHY

    vec2 yscale_a = vec2(0., index_min);
    vec2 yscale_b = vec2(1., 2/(index_max-index_min));
    vec2 yscale_c = vec2(0., -1/nrows);


    // SPREAD
    //does not scale the x pos, just the y pos by an equal amount per row
    float spread = 1;
    vec2 a = vec2(spread, spread/nrows);

    // LOCATION
    vec2 b = vec2(0, -1 + 2*(index_col.y+.5) / nrows);

    // COMBINE RELATIVE LINE POSITION + SPREAD + LOCATION
    // gl_Position = vec4(a*(scale*position-yscale_a)*yscale_b+b, 0.0, 1.0);
    gl_Position = vec4(a*(scale*position-yscale_a)*yscale_b+b+yscale_c, 0.0, 1.0);



    // WRAP UP
    v_index = index_col;
    // For clipping test in the fragment shader.
    v_position = gl_Position.xy;
    v_ab = vec4(a, b);
}
"""

FRAG_SHADER = """
#version 120
varying vec2 v_index;
varying vec2 v_position;
varying vec4 v_ab;
void main() {
    gl_FragColor = vec4(1., 1., 1., 1.);
    // Discard the fragments between the signals (emulate glMultiDrawArrays).

    if (fract(v_index.y) > 0.)
        discard;

    // Clipping test.
    vec2 test = abs((v_position.xy-v_ab.zw)/v_ab.xy);
    if ((test.x > 1) || (test.y > 1))
        discard;
}
"""


class Canvas(app.Canvas):
    def __init__(self):
        app.Canvas.__init__(self, title='Use your wheel to zoom!',
                            keys='interactive')
        self.program = gloo.Program(VERT_SHADER, FRAG_SHADER)
        self.program['y'] = y_flat
        self.program['index_col'] = index_col
        self.program['scale'] = (1., 1.)
        self.program['num_features'] = (num_features, 1)
        self.program['num_samples'] = num_samples
        self.program['index_min'] = index_min[:,0].flatten()
        self.program['index_max'] = index_max[:,0].flatten()
        gloo.set_viewport(0, 0, *self.physical_size)

        self._timer = app.Timer('auto', connect=self.on_timer, start=True)

        gloo.set_state(clear_color='black', blend=True,
                       blend_func=('src_alpha', 'one_minus_src_alpha'))

        self.show()

    def on_resize(self, event):
        gloo.set_viewport(0, 0, *event.physical_size)

    def on_mouse_wheel(self, event):
        dx = np.sign(event.delta[1]) * .05
        scale_x, scale_y = self.program['scale']

        index_y_scaled[:,0] = index_y_scaled_orig[:,0] * scale_x
        valid = ((index_y_scaled[:,0]>-1)*(index_y_scaled[:,0]<1))
        y_flat_scaled = y_flat * scale_x
        shown = ma.masked_array(y_flat_scaled, mask=np.logical_not(valid))
        shown_reshaped = (shown.reshape(num_features,num_samples))

        runmin = np.array(np.min(shown_reshaped, 1))
        runmax = np.array(np.max(shown_reshaped, 1))
        index_min[:, 1] = np.repeat(runmin, num_samples)
        index_max[:, 1] = np.repeat(runmax, num_samples)

        # scale_x = 10
        # scale_y = 10
        # print(scale_x)
        # print(scale_y)
        # print(runmin)
        # print(runmax)
        # forplot=(y_flat_scaled*valid).reshape(num_features,num_samples).transpose()
        # pd.DataFrame(forplot).plot(subplots=True)
        # forplot2 = (((y_flat_scaled * valid - index_min[:, 1])).flatten()).reshape([num_features, num_samples]).transpose()
        # pd.DataFrame(forplot2).plot(subplots=True)
        # forplot3 = (((y_flat_scaled * valid - index_min[:, 1]) / (index_max[:, 1] - index_min[:, 1])).flatten()).reshape([num_features, num_samples]).transpose()
        # pd.DataFrame(forplot3).plot(subplots=True)

        self.program['index_min'] = index_min[:,1].flatten()
        self.program['index_max'] = index_max[:,1].flatten()
        #print(self.program['print_position'])

        scale_x_new, scale_y_new = (scale_x * math.exp(1.0*dx),
                                    scale_y * math.exp(1.0*dx))
        #print(scale_x_new)
        self.program['scale'] = (max(1, scale_x_new), max(1, scale_y_new))
        self.update()

    def on_timer(self, event):
        """Add some data at the end of each signal (real-time signals)."""
        # y[:, :-1] = y[:, 1:]
        # y[:, -1:] = np.random.normal(size=3, loc=[0], scale=[0.01]).reshape(3, 1)

        self.program['y'].set_data(y.flatten().astype(np.float32)) #(10920,)
        self.update()

    def on_draw(self, event):
        gloo.clear()
        self.program.draw('line_strip')

if __name__ == '__main__':
    c = Canvas()
    app.run()
user3641187
  • 405
  • 5
  • 10