Bokeh GUI breaking down with adding/removing lines

Question

I am trying to make a simple GUI where I can select which data to load and display in a Bokeh plot. I can then remove that data. The basic idea is shown below:

However, I run into a host of issues:

1. Hover tool does not work with added lines

2. Interactive legend does not work with added lines

3. When I try to remove data via plot.renderers.remove(line_to_remove), the whole bokeh plot goes crazy (see 2nd figure below)

4. line_to_remove.visible = False does not work

I'm sorry to drop all these problems at once, but i've been spending days trying to get all these kinks out and have a smooth bokeh workflow with no luck. Thinking about switching to another library, which would suck since bokeh is no aesthetically pleasing.

Here is all my code which is compilable:

# general imports
from bokeh.client import push_session
import numpy as np
from scipy import signal
from timeit import default_timer as timer
import itertools
import ctypes
import copy
import pickle
import os
import os.path

#bokeh, plotting
from bokeh.io import show, output_notebook, output_file, save, reset_output, curdoc
from bokeh.layouts import row, column, widgetbox
from bokeh.models import ColumnDataSource, Select, MultiSelect
from bokeh.models.widgets import Slider, TextInput, Dropdown, Button
from bokeh.plotting import figure
from tornado.ioloop import IOLoop
from bokeh.application.handlers import FunctionHandler
from bokeh.application import Application
from bokeh.server.server import Server
from bokeh.palettes import Dark2_5 as palette
from bokeh.palettes import Set1 as palettec #Set1, Category10

io_loop = IOLoop.current()

masterDir = 'C:\\Data\\BrownData\\'

_DEBUG_MODE = True



def modify_doc(doc):

    TOOLS = "pan,wheel_zoom,box_zoom,reset,hover,save,resize"
    SCREEN_WIDTH = ctypes.windll.user32.GetSystemMetrics(0)
    SCREEN_HEIGHT = ctypes.windll.user32.GetSystemMetrics(1)
    active_scroll = 'wheel_zoom'
    colors2 = itertools.cycle(palettec)
    colors = list(zip(range(10), colors2))
    col = copy.deepcopy(palettec[7])

    if not _DEBUG_MODE:
        pass
    else:
        print('In debug mode')
        Nsubdir, Nfiles, Nsigs = 2, 3, 4
        dataInfo ={}
        dataInfo['subDirNames'] = ['Subdir' + str(x) for x in range(Nsubdir)]
        dataInfo['subDirFiles'] = [['File_' + str(y) + '_' + str(x) for x in range(Nfiles)] for y in range(Nsubdir)]
        dataInfo['analogSignalNames'] = [
            [['Sig_' + str(y) + str(x) + '_' + str(w) for w in range(Nsigs)] for x in range(Nfiles)] for y in range(Nsubdir)]
        debugData = [np.array([0, 1, 0, 2, -1]).T * (i + 1) for i in range(20)]
        debugData = [np.arange(1, 6, 1)] + debugData

    subdir_dropdown = Select(title="Subdirectory", value=" ", options=[" "]+dataInfo['subDirNames'])
    files_dropdown = Select(title="File", value=" ", options=[' '])
    signal_dropdown = MultiSelect(title="Analog Signals:", value=[' '], options=[' '], size=7)

    dataq={}
    dataq['x'] = debugData[0]
    dataq['y1'] = debugData[1]
    dataq['y2'] = debugData[2]
    source = ColumnDataSource(dataq)

    # Set up layouts and add to document
    inputs = widgetbox(subdir_dropdown, files_dropdown, signal_dropdown, width=int(SCREEN_WIDTH * .15))
    #final = row(inputs, plt, plt_fft, width=int(SCREEN_WIDTH * .9))
    mainLayout = row(row(inputs, name='Widgets'), name='mainLayout')
    doc.add_root(mainLayout)
    #session = push_session(doc)

    plt_reset = 0

    def getSignal(name, indx):
        return debugData[indx]

    def subdir_callback(attrname, old, new):
        # first clear plot
        rootLayout = doc.get_model_by_name('mainLayout')
        listOfSubLayouts = rootLayout.children
        plotToRemove = doc.get_model_by_name('sigplot')
        if plotToRemove is not None:
            listOfSubLayouts.remove(plotToRemove)
        plt_reset, col = 1, palettec[7]
        # change files menu options
        indx = dataInfo['subDirNames'].index(subdir_dropdown.value)
        files_dropdown.options = dataInfo['subDirFiles'][indx]

    def files_callback(attrname, old, new):
        # first clear plot
        rootLayout = doc.get_model_by_name('mainLayout')
        listOfSubLayouts = rootLayout.children
        plotToRemove = doc.get_model_by_name('sigplot')
        if plotToRemove is not None:
            listOfSubLayouts.remove(plotToRemove)
        plt_reset, col = 1, palettec[7]
        # then start new plot
        if not doc.get_model_by_name('sigplot'):
            sigplot = figure(name='sigplot', tools=TOOLS)
            plotToAdd = sigplot
            r1 = sigplot.line(x=[1,2], y=[2,1], legend='test')
            sigplot.legend.location = "top_left"
            sigplot.legend.click_policy = "hide"
        else:
            plotToAdd = doc.get_model_by_name('sigplot')
        listOfSubLayouts.append(plotToAdd)

        # change menu options
        sd_indx = dataInfo['subDirNames'].index(subdir_dropdown.value)
        file_indx = dataInfo['subDirFiles'][sd_indx].index(files_dropdown.value)
        signal_dropdown.options = dataInfo['analogSignalNames'][sd_indx][file_indx]



    def analogSigs_callback(attrname, old, new):
        nonlocal plt_reset, col
        sigplot = doc.get_model_by_name('sigplot')

        if old == [' ']:
            old = []
        if plt_reset == 1:
            old = []

        if len(new) > len(old):  # add line
            new_element = list(set(new) - set(old))
            if len(new_element) > 1:
                print('WARNING: somehow more than 1 new element chosen')
            new_element = new_element[0]
            sig_indx = signal_dropdown.options.index(new_element)

            if new_element not in source.data:
                source.add(getSignal(new_element, sig_indx), name=new_element)
            else:
                tmp_line = sigplot.select_one({'name': new_element})
                #tmp_line.visible = True
            sigplot.line(x='x', y=new_element, source=source, legend='ch' + new_element, line_width=1, color=col[0], name=new_element)
            del col[0]

        if len(new) < len(old):  # remove line
            removed_elements = list(set(old) - set(new))
            for elem in removed_elements:
                tmp_line = sigplot.select_one({'name': elem})
                #tmp_line.visible = False
                sigplot.renderers.remove(tmp_line)
        if len(new) == len(old):  # switch single line
            tmp_line = sigplot.select_one({'name': old[0]})
            #tmp_line.visible = False
            sigplot.renderers.remove(tmp_line)
            analogSigs_callback(attrname, [], new)

    subdir_dropdown.on_change('value', subdir_callback)
    files_dropdown.on_change('value', files_callback)
    signal_dropdown.on_change('value', analogSigs_callback)

    print('done')


bokeh_app = Application(FunctionHandler(modify_doc))
server = Server({'/': bokeh_app}, io_loop=io_loop, port=5006)
server.start()



if __name__ == '__main__':

    #modify_doc(None)
    print('Opening Bokeh application on http://localhost:5006/')
    io_loop.add_callback(server.show, "/")
    io_loop.start()

Answer 1

Following from comments with a working example

The source can change length. It just can't have columns of different lengths at the same time. So when you load new data, you replace source.data with a new dict, whatever the new length.

Example:

from bokeh.layouts import column
from bokeh.models import ColumnDataSource, Select
from bokeh.plotting import figure, show, output_notebook
from bokeh.application import Application
from bokeh.application.handlers import FunctionHandler

import numpy as np

output_notebook()

def get_data(dir, file, signal):
    # Here do whatever your data reading should be doing.
    print(f"looking for data in dir '{dir}', file '{file}', signal '{signal}'")

    # sample data: random series with length equal the file number
    index = int(signal.split('_')[1])
    return {'x': np.arange(index), 'y': np.random.random(index)}


def modify_doc(doc):
    p = figure()
    p.xaxis.axis_label = 'x'  # that enables showing axis even with no initial data
    p.yaxis.axis_label = 'y'
    s1 = Select(title="Directory", value=None, options=['dir_' + str(i) for i in range(3)])
    s2 = Select(title="File", value=None, options=['file_' + str(i) for i in range(4)])
    s3 = Select(title="Signal", value=None, options=['signal_' + str(i) for i in range(6)])

    source = ColumnDataSource(data={'x': [], 'y': []})

    p.line(x='x', y='y', source=source)

    def update(attr, old, new):
        new_data = get_data(s1.value, s2.value, s3.value)
        source.data = new_data

    for s in [s1, s2, s3]:
        s.on_change('value', update)

    doc.add_root(column(s1, s2, s3, p))

app = Application(FunctionHandler(modify_doc))
show(app, notebook_url='localhost:8888')

If your axis type change (linear, log, datetime, factor/categorical), then it will be a bit more complex and I'm not sure every transition is possible.