Send dictionary in python socket

Question

I want to send dictionary via python socket.

I tried to encode sending data and then encode at the receiving point

but the problem in unconvertable received data and can not convert to dictionary.

Here is my code:

Class client.py:

HEADER_LENGTH = 512
IP = "127.0.0.1"
PORT = 963

my_username = input("Username: ")
client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client_socket.connect((IP, PORT))
client_socket.setblocking(False)
username = my_username.encode('utf-8')
username_header = f"{len(username):<{HEADER_LENGTH}}".encode('utf-8')
client_socket.send(username_header + username)
while True:
    message = input(f"{my_username} > ")
    if message:
        message = message.encode('utf-8')
        message_header = f"{len(message)}:<{HEADER_LENGTH}".encode('utf-8')
        # {'header': {message_header}, 'data': {message}}
        send_dict = {} # 'header': message_header, 'data': message
        send_dict["header"] = str(message_header)
        send_dict["data"] = str(message)
        client_socket.send(json.dumps(send_dict).encode('utf-8'))  # message_header + message
    try:
        while True:
            # Receive ...
            username_header = client_socket.recv(HEADER_LENGTH)
            if not len(username_header):
                print("Connection closed by the server")
                sys.exit()
            username_length = int(username_header.decode('utf-8').strip())
            username = client_socket.recv(username_length).decode(('utf-8'))

            message_header = client_socket.recv(HEADER_LENGTH)
            message_length = int(message_header.decode('utf-8').strip())
            message = client_socket.recv(message_length).decode(('utf-8'))
            print(f"{username} > {message}")

    except IOError as e:
        if e.errno != errno.EAGAIN and e.errno != errno.EWOULDBLOCK:
            print('Reading error', str(e))
            sys.exit()
        continue
    except Exception as e:
        print('General error', str(e))
        sys.exit()

Class server.py:

HEADER_LENGTH = 512
IP = "127.0.0.1"
PORT = 963

server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((IP, PORT))
server_socket.listen()

socket_list = [server_socket]
clients = {}


def receive_message(client_socket):
    # print(f"client socket: {client_socket}")
    # try:
    message_header = client_socket.recv(HEADER_LENGTH)
    if not len(message_header):
        print("return from receive message /condition/")
        return False
    print(f"message header: {type(message_header)}")
    print(f"message header: {message_header}")
    print(f"message header decode : {type(message_header.decode('utf-8').strip())}")
    message_header_dict = message_header.decode('utf-8').strip()
    # print(f"message header replaced : {message_header}")
    print(f"message header replaced : {type(message_header_dict)}")
    print(f"message header replaced : {message_header_dict}")
    # print("message header header : " + message_header_dict['header'].decode('utf-8').strip())
    json_message_header = json.loads(message_header_dict)
    print(f"message header replaced : {json_message_header}")
    print(f"message header replaced : {json_message_header['data']}") # Error line
    message_length = int(message_header.decode("utf-8").strip())
    return {"header": message_header, "data": client_socket.recv(message_length)}
    # except:
        # print("return from receive message /except/")
        # return False


while True:
    read_sockets, _, exception_sockets = select.select(socket_list, [], socket_list)
    for notified_socket in read_sockets:
        if notified_socket == server_socket:
            client_socket, client_address = server_socket.accept()
            user = receive_message(client_socket)
            if user == False:
                continue
            socket_list.append(client_socket)
            clients[client_socket] = user
            print(
                f"Accept new connection from {client_address[0]}:{client_address[1]} username:{user['data'].decode('utf-8')}")
        else:
            message = receive_message(notified_socket)
            if message == False:
                print(f"close connection from: {clients[notified_socket]['data'].decode('utf-8')}")
                socket_list.remove(notified_socket)
                del clients[notified_socket]
                continue
            user = clients[notified_socket]
            print(f"Received message from: {user['data'].decode('utf-8')}: {message['data'].decode('utf-8')}")
            for client_socket in clients:
                if client_socket != notified_socket:
                    client_socket.send(user['header'] + user['data'] + message['header'] + message['data'])
    for notified_socket in exception_sockets:
        socket_list.remove(notified_socket)
        del clients[notified_socket]

What is the actual problem? Show us the errors. – John Gordon Jul 22 '20 at 15:27 — John Gordon, Jul 22 '20 at 15:27

SamR · Answer 1 · 2020-07-29T14:45:33.603

Your client sends JSON. Your server then reads the entire JSON message and tries to interpret it as an integer. Boom.

You basically have 3 options for transferring the full message: Send bytes and assume you receive the entire message and no more from a single read (least reliable); use a delimiter (e.g., CRLF) and make sure it doesn't appear in the message (restricted data stream); or send a byte-count followed by the message bytes (length+data, the most reliable). It looks like you've mixed the first and third approaches, which basically never works.

To serialize a dict for transmission as JSON using length+data, first JSON-encode the dict, then UTF8-encode the resultant string to get the message bytes. From this point until the server deserializes the message, the code is working with bytes.

To send length+data, the struct module is your friend. Define the pack/unpack specifier (basically, selected-size integer plus bytestring). For example, "!Hs" would support message bytestrings up to 64K.

In the client...

Get the encoded length of the UTF8-encoded message string.
Use struct.pack to create the binary message (<length-header><message-bytes>).
Use socket.sendall to send the binary message (sending the message may take multiple send calls, and sendall takes care of that for you).

In the server...

Use socket.recv to receive just the header bytes (e.g., 4 or 8 bytes).
Unpack the byte count using struct.unpack.
Use socket.recv to receive just the message bytes (i.e., byte_count bytes)
To deserialize the dict from the received (serialized) message bytes, UTF8-decode then JSON-decode.

score 0 · Answer 2 · answered Aug 23 '20 at 06:27

you could use pickle or its faster version cPickle to serialize data and deserialize data.

import pickle

# for serializing data 
pickle.dumps(data_to_serialized)
# for deserializing 
pickle.loads(data_to_be_deserialized)

Here is sample function for sending and receiving data using pickle for serialization:

def send_data(conn, data):
        serialized_data = pickle.dumps(data)
        conn.sendall(struct.pack('>I', len(serialized_data)))
        conn.sendall(serialized_data)


def receive_data(conn):
    data_size = struct.unpack('>I', conn.recv(4))[0]
    received_payload = b""
    reamining_payload_size = data_size
    while reamining_payload_size != 0:
        received_payload += conn.recv(reamining_payload_size)
        reamining_payload_size = data_size - len(received_payload)
    data = pickle.loads(received_payload)

    return data

you could find sample program at https://github.com/vijendra1125/Python-Socket-Programming.git

Send dictionary in python socket

2 Answers2