Simple tool/library to visualize huge python dict

Question

I have a huge dict structure like this one:

my_data = {
    'key1': {
        '_': 'value1': 'aaa'
    },
    'key2': {
        '_': 'value2': 'bbb',
        'key2.1': {
            '_': 'ccc',
            'key2.1.1': {
                '_': 'ddd'
            }
        }
        'key2.2': {
            '_': 'eee',
            'key2.2.1': {
                '_': 'fff'
            }
            'key2.2.2': {
                '_': 'ggg'
            }               
        }
    }
}

and so on.

I want to display it to user in a kind of tree representation, using GTK, TK or anything to be able to browse it collapse and expand branches and probably search keys and values.

May be I do not need to develop such a tool by hands and there is already something that can visualize this kind of data out of the box?

Answer 1

I do not know of a ready-to-use tool, but you could use Traits UI to swiftly develop your own

from enthought.traits.api \
    import HasTraits, Instance

from enthought.traits.ui.api \
    import View, VGroup, Item, ValueEditor

class DictEditor(HasTraits):
    Object = Instance( object )

    def __init__(self, obj, **traits):
        super(DictEditor, self).__init__(**traits)
        self.Object = obj

    def trait_view(self, name=None, view_elements=None):
        return View(
          VGroup(
            Item( 'Object',
                  label      = 'Debug',
                  id         = 'debug',
                  editor     = ValueEditor(),
                  style      = 'custom',
                  dock       = 'horizontal',
                  show_label = False
            ),
          ),
          title     = 'Dictionary Editor',
          width     = 800,
          height    = 600,
          resizable = True,
        )


def build_sample_data():
    my_data = dict(zip(range(10),range(10,20)))
    my_data[11] = dict(zip(range(10),range(10,20)))
    my_data[11][11] = dict(zip(range(10),range(10,20)))
    return my_data

# Test
if __name__ == '__main__':
    my_data = build_sample_data()
    b = DictEditor(my_data)
    b.configure_traits()

That's it. You will have a GUI like:

Traits UI uses the Model-View-Controller approach to create GUI without having the need to programatically create every widget. Here, I use the predefined ValueEditor to display arbitrary types. You can now extend it to support searching, filtering etc...

EDIT

Simple extension to support filtering:

# -*- coding: utf-8 -*-
"""
Created on Fri Feb 22 12:52:28 2013

@author: kranzth
"""
from enthought.traits.api \
    import HasTraits, Instance, Str, on_trait_change

from enthought.traits.ui.api \
    import View, VGroup, Item, ValueEditor, TextEditor

from copy import deepcopy

class DictEditor(HasTraits):
    SearchTerm = Str()
    Object = Instance( object )

    def __init__(self, obj, **traits):
        super(DictEditor, self).__init__(**traits)
        self._original_object = obj
        self.Object = self._filter(obj)

    def trait_view(self, name=None, view_elements=None):
        return View(
          VGroup(
            Item( 'SearchTerm',
                  label      = 'Search:',
                  id         = 'search',
                  editor     = TextEditor(),
                  #style      = 'custom',
                  dock       = 'horizontal',
                  show_label = True
            ),
            Item( 'Object',
                  label      = 'Debug',
                  id         = 'debug',
                  editor     = ValueEditor(),
                  style      = 'custom',
                  dock       = 'horizontal',
                  show_label = False
            ),
          ),
          title     = 'Dictionary Editor',
          width     = 800,
          height    = 600,
          resizable = True,
        )

    @on_trait_change("SearchTerm")
    def search(self):
        self.Object = self._filter(self._original_object, self.SearchTerm)

    def _filter(self, object_, search_term=None):
        def has_matching_leaf(obj):
            if isinstance(obj, list):
                return any(
                        map(has_matching_leaf, obj))
            if isinstance(obj, dict):
                return any(
                        map(has_matching_leaf, obj.values()))
            else:
                try:
                    if not str(obj) == search_term:
                        return False
                    return True
                except ValueError:
                    False

        obj = deepcopy(object_)
        if search_term is None:
            return obj

        if isinstance(obj, dict):
            for k in obj.keys():
                if not has_matching_leaf(obj[k]):
                    del obj[k]

            for k in obj.keys():
                if isinstance(obj, dict):
                    obj[k] = self._filter(obj[k], search_term)
                elif isinstance(obj, list):
                    filter(has_matching_leaf,obj[k])

        return obj



def build_sample_data():
    def make_one_level_dict():
        return dict(zip(range(100),
                        range(100,150) + map(str,range(150,200))))

    my_data = make_one_level_dict()
    my_data[11] = make_one_level_dict()
    my_data[11][11] = make_one_level_dict()
    return my_data

# Test
if __name__ == '__main__':
    my_data = build_sample_data()
    b = DictEditor(my_data)
    b.configure_traits()

will give you a textbox with "filter-as-you-type". The search isn't completely correct for all cases, but you can figure out the idea.

Please note that in this sample the data in the dict are partly integers and partly strings, and both types will be found.

Answer 2

I'm finally ended up with converting my data into json as @PavelAnossov suggested and using d3 Tree Layout.

Answer 3

There are some great answers already in here, but I believe this one qualifies as "simple" (it uses only python bult-in libraries tkinter and uuid).

It is based on John Gaines Jr.'s answer in another question, modified by Will Ware to support lists, modified by me to also support tuples (runs on python 3).

I've also reorganized it so that you can call the viewer with something as simple as tk_tree_view(data), passing in a dictionary (as in the example at the end).

import uuid
import tkinter as tk
from tkinter import ttk


def j_tree(tree, parent, dic):
    for key in sorted(dic.keys()):
        uid = uuid.uuid4()
        if isinstance(dic[key], dict):
            tree.insert(parent, 'end', uid, text=key)
            j_tree(tree, uid, dic[key])
        elif isinstance(dic[key], tuple):
            tree.insert(parent, 'end', uid, text=str(key) + '()')
            j_tree(tree, uid,
                   dict([(i, x) for i, x in enumerate(dic[key])]))
        elif isinstance(dic[key], list):
            tree.insert(parent, 'end', uid, text=str(key) + '[]')
            j_tree(tree, uid,
                   dict([(i, x) for i, x in enumerate(dic[key])]))
        else:
            value = dic[key]
            if isinstance(value, str):
                value = value.replace(' ', '_')
            tree.insert(parent, 'end', uid, text=key, value=value)


def tk_tree_view(data):
    # Setup the root UI
    root = tk.Tk()
    root.title("tk_tree_view")
    root.columnconfigure(0, weight=1)
    root.rowconfigure(0, weight=1)

    # Setup the Frames
    tree_frame = ttk.Frame(root, padding="3")
    tree_frame.grid(row=0, column=0, sticky=tk.NSEW)

    # Setup the Tree
    tree = ttk.Treeview(tree_frame, columns=('Values'))
    tree.column('Values', width=100, anchor='center')
    tree.heading('Values', text='Values')
    j_tree(tree, '', data)
    tree.pack(fill=tk.BOTH, expand=1)

    # Limit windows minimum dimensions
    root.update_idletasks()
    root.minsize(root.winfo_reqwidth(), root.winfo_reqheight())
    root.mainloop()


if __name__ == "__main__":
    # Setup some test data
    data = {
        "firstName": "John",
        "lastName": "Smith",
        "gender": "male",
        "age": 32,
        "address": {
            "streetAddress": "21 2nd Street",
            "city": "New York",
            "state": "NY",
            "postalCode": "10021"},
        "phoneNumbers": [
            {"type": "home", "number": "212 555-1234"},
            {"type": "fax",
             "number": "646 555-4567",
             "alphabet": [
                 "abc",
                 "def",
                 "ghi"]
             }
        ]}

    # call it with
    tk_tree_view(data)

It looks like this:

Answer 4

Just complement @Thorsten's answer. The package traits has been refactored since a long time ago. The correct way to run Thorsten's code is:

1. install traits module: sudo apt-get install python-traitsui

2. change the module import lines in his code to:

   from traits.api \
   import HasTraits, Instance, Str, on_trait_change
   
   from traitsui.api \
   import View, VGroup, Item, ValueEditor, TextEditor
   

Answer 5

If you're using an IDE set a breakpoint after the dictionary is initialized and has the data you want to explore then run in debug mode. There should be a "Variables" view in debug mode where you can expand and collapse the dictionary as you mentioned.

Answer 6

This simple function prints a dictionary in table form. It can also deal with nested dictionaries.

def visualise_dict(d,lvl=0):

    # go through the dictionary alphabetically 
    for k in sorted(d):

        # print the table header if we're at the beginning
        if lvl == 0 and k == sorted(d)[0]:
            print('{:<25} {:<15} {:<10}'.format('KEY','LEVEL','TYPE'))
            print('-'*79)

        indent = '  '*lvl # indent the table to visualise hierarchy
        t = str(type(d[k]))

        # print details of each entry
        print("{:<25} {:<15} {:<10}".format(indent+str(k),lvl,t))

        # if the entry is a dictionary
        if type(d[k])==dict:
            # visualise THAT dictionary with +1 indent
            visualise_dict(d[k],lvl+1)

With a sample dictionary:

d = {}
d.update({1:{},2:{}})
d[1]['foo'] = {}
d[1]['foo']['bar'] = 1
d[2]['bar'] = 5.2

visualise_dict(d)

returns

In [1]: visualise_dict(d)
KEY                       LEVEL           TYPE      
-------------------------------------------------------------------------------
1                         0               <class 'dict'>
  foo                     1               <class 'dict'>
    bar                   2               <class 'int'>
2                         0               <class 'dict'>
  bar                     1               <class 'float'>

Answer 7

If you use a browser with the JSONViewer extension this might work for you:

import json
import tempfile
import os
import subprocess
def view_obj(obj):
  (f, filepath)= tempfile.mkstemp()
  os.close(f)
  with open(filepath, 'w') as f:
    json.dump(obj, f)
    subprocess.call(["google-chrome", filepath])

view_obj({'key':'value'})  # Opens Chrome and renders JSON nicely

Answer 8

I modified Lucas's answer above so that it will accept dictionaries with mixed key types (like integers AND strings). The way I did it was with json encoding and decoding so all the keys are strings.

import uuid
import json
import tkinter as tk
from tkinter import ttk


def j_tree(tree, parent, dic):
    for key in sorted(dic.keys()):
        uid = uuid.uuid4()
        if isinstance(dic[key], dict):
            tree.insert(parent, 'end', uid, text=key)
            j_tree(tree, uid, dic[key])
        elif isinstance(dic[key], tuple):
            tree.insert(parent, 'end', uid, text=str(key) + '()')
            j_tree(tree, uid,
                   dict([(i, x) for i, x in enumerate(dic[key])]))
        elif isinstance(dic[key], list):
            tree.insert(parent, 'end', uid, text=str(key) + '[]')
            j_tree(tree, uid,
                   dict([(i, x) for i, x in enumerate(dic[key])]))
        else:
            value = dic[key]
            if isinstance(value, str):
                value = value.replace(' ', '_')
            tree.insert(parent, 'end', uid, text=key, value=value)


def tk_tree_view(data):
    # Setup the root UI
    data = json.dumps(data)
    data = json.loads(data) 
    root = tk.Tk()
    root.title("tk_tree_view")
    root.columnconfigure(0, weight=1)
    root.rowconfigure(0, weight=1)

    # Setup the Frames
    tree_frame = ttk.Frame(root, padding="3")
    tree_frame.grid(row=0, column=0, sticky=tk.NSEW)

    # Setup the Tree
    tree = ttk.Treeview(tree_frame, columns=('Values'))
    tree.column('Values', width=100, anchor='center')
    tree.heading('Values', text='Values')
    j_tree(tree, '', data)
    tree.pack(fill=tk.BOTH, expand=1)

    # Limit windows minimum dimensions
    root.update_idletasks()
    root.minsize(root.winfo_reqwidth(), root.winfo_reqheight())
    root.mainloop()


if __name__ == "__main__":
    # Setup some test data
    data = {
        "firstName": "John",
        "lastName": "Smith",
        "gender": "male",
        "age": 32,
        "address": {
            "streetAddress": "21 2nd Street",
            "city": "New York",
            "state": "NY",
            "postalCode": "10021"},
        "phoneNumbers": [
            {"type": "home", "number": "212 555-1234"},
            {"type": "fax",
             "number": "646 555-4567",
             "alphabet": [
                 "abc",
                 "def",
                 "ghi"]
             }
        ]}

I find I can paste this code into ipython.
Then define data as my dictionary

data = [paste the big dict here]

and then call it with

tk_tree_view(data)

This kind of example really shows off the simplicity of tk.