Returning large data from RPC (Crossbar + Autobahn|Python)

Question

I am trying to transmit large data through websockets using crossbar/autobahn's RPC. My setup is as follow:

• Python 2.7
• A crossbar router (version 17.8.1.post1)
• A back-end that will try to send a large pandas DataFrame as a json string
• A front-end that will want to receive this string

In essence my front-end is trying to call a function that will return a large string.

class MyComponent(ApplicationSession):

@inlineCallbacks
def onJoin(self, details):
    print("session ready")
    try:
        res = yield self.call(u'data.get')

And I get this error:

2017-08-09T16:38:10+0200 session closed with reason wamp.close.transport_lost [WAMP transport was lost without closing the session before]
2017-08-09T16:38:10+0200 Cancelling 1 outstanding requests
2017-08-09T16:38:10+0200 call error: ApplicationError(error=<wamp.close.transport_lost>, args=[u'WAMP transport was lost without closing the session before'], kwargs={}, enc_algo=None)

It seems crossbar is kicking me out because my client session looks dead to him, but I thought that autobahn would chunk my data and that the call would not block the client reactor.

I enabled a few things in my crossbar configuration to improve websocket treatment; thanks to that I was able to transmit larger amount of data but eventually I would hit a limit (config file largely copied and pasted from sam & max).

                        "options": {
                            "enable_webstatus": false,
                            "max_frame_size": 16777216,
                            "auto_fragment_size": 65536,
                            "fail_by_drop": true,
                            "open_handshake_timeout": 2500,
                            "close_handshake_timeout": 1000,
                            "auto_ping_interval": 10000,
                            "auto_ping_timeout": 5000,
                            "auto_ping_size": 4,
                            "compression": {
                                "deflate": {
                                    "request_no_context_takeover": false,
                                    "request_max_window_bits": 11,
                                    "no_context_takeover": false,
                                    "max_window_bits": 11,
                                    "memory_level": 4
                               }
                            }
                        }

Any ideas, takes, things that I am doing wrong?

Thank you,

---

Client code:

from __future__ import print_function
import pandas as pd

from autobahn.twisted.wamp import ApplicationSession
from twisted.internet.defer import inlineCallbacks


class MyComponent(ApplicationSession):

    @inlineCallbacks
    def onJoin(self, details):
        print("session ready")
        try:
            res = yield self.call(u'data.get')
            print('Got the data')
            data = pd.read_json(res)
            print("call result: {}".format(data.head()))
            print("call result shape: {0}, {1}".format(*data.shape))
        except Exception as e:
            print("call error: {0}".format(e))


if __name__ == "__main__":
    from autobahn.twisted.wamp import ApplicationRunner

    runner = ApplicationRunner(url=u"ws://127.0.0.1:8080/ws", realm=u"realm1")
    runner.run(MyComponent)

Backend code

from __future__ import absolute_import, division, print_function

from twisted.internet.defer import inlineCallbacks
from autobahn.twisted.wamp import ApplicationSession
from twisted.internet import reactor, defer, threads

# Imports
import pandas as pd


def get_data():
    """Returns a DataFrame of stuff as a JSON

    :return: str, data as a JSON string
    """
    data = pd.DataFrame({
        'col1': pd.np.arange(1000000),
        'col2': "I'm big",
        'col3': 'Like really big',
    })
    print("call result shape: {0}, {1}".format(*data.shape))
    print(data.memory_usage().sum())
    print(data.head())
    return data.to_json()


class MyBackend(ApplicationSession):

    def __init__(self, config):
        ApplicationSession.__init__(self, config)

    @inlineCallbacks
    def onJoin(self, details):

        # Register a procedure for remote calling
        @inlineCallbacks
        def async_daily_price(eqt_list):
            res = yield threads.deferToThread(get_data)
            defer.returnValue(res)

        yield self.register(async_daily_price, u'data.get')


if __name__ == "__main__":
    from autobahn.twisted.wamp import ApplicationRunner

    runner = ApplicationRunner(url=u"ws://127.0.0.1:8080/ws", realm=u"realm1")
    runner.run(MyBackend)

Configuration

{
"version": 2,
"controller": {},
"workers": [
    {
        "type": "router",
        "realms": [
            {
                "name": "realm1",
                "roles": [
                    {
                        "name": "anonymous",
                        "permissions": [
                            {
                                "uri": "",
                                "match": "prefix",
                                "allow": {
                                    "call": true,
                                    "register": true,
                                    "publish": true,
                                    "subscribe": true
                                },
                                "disclose": {
                                    "caller": false,
                                    "publisher": false
                                },
                                "cache": true
                            }
                        ]
                    }
                ]
            }
        ],
        "transports": [
            {
                "type": "universal",
                "endpoint": {
                    "type": "tcp",
                    "port": 8080
                },
                "rawsocket": {
                },
                "websocket": {
                    "ws": {
                        "type": "websocket",
                        "options": {
                            "enable_webstatus": false,
                            "max_frame_size": 16777216,
                            "auto_fragment_size": 65536,
                            "fail_by_drop": true,
                            "open_handshake_timeout": 2500,
                            "close_handshake_timeout": 1000,
                            "auto_ping_interval": 10000,
                            "auto_ping_timeout": 5000,
                            "auto_ping_size": 4,
                            "compression": {
                                "deflate": {
                                    "request_no_context_takeover": false,
                                    "request_max_window_bits": 11,
                                    "no_context_takeover": false,
                                    "max_window_bits": 11,
                                    "memory_level": 4
                               }
                            }
                        }
                    }
                },
                "web": {
                    "paths": {
                        "/": {
                            "type": "static",
                            }
                        }
                    }
                }
            ]
        }
    ]
}

Answer 1

The solution suggested by the crossbar.io group was to use the progressive result option of the RPC.

A full working example is located at https://github.com/crossbario/autobahn-python/tree/master/examples/twisted/wamp/rpc/progress

In my code I had to add a the chunking of the result in the backend

        step = 10000
        if details.progress and len(res) > step:
            for i in xrange(0, len(res), step):
                details.progress(res[i:i+step])
        else:
            defer.returnValue(res)

And to the caller

        res = yield self.call(
            u'data.get'
            options=CallOptions(
                on_progress=partial(on_progress, res=res_list)
            )
        )

Where my function on_progress adds the chunks to a result list

def on_progress(x, res):
    res.append(x)

Picking the right chunk size will do the trick.