opencv - python video streaming in tkinter

Question

I am trying to send a video frame in parts to the server and then build it up again in the client (kind of video streaming) with a UDP socket using threads, tkinter and opencv, but I don't understand my mistake/error, and don't really know if the way I am dividing the video parts is correct.

Server:

import socket


IP = "0.0.0.0"
PORT = 8080
LEN = 12000


def main():
    server_socket = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM)
    server_socket.bind((IP, PORT))
    while True:
        msg, add = server_socket.recvfrom(LEN)
        server_socket.sendto(msg, add)


if __name__ == '__main__':
    main()

Client:

import socket
from tkinter import *
from PIL import ImageTk, Image
import cv2
import numpy
import threading


IP = "127.0.0.1"
PORT = 8080
my_socket = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM)
my_socket.connect((IP, PORT))
root = Tk()
main_label = Label(root)
main_label.grid()
cap = cv2.VideoCapture("video.mp4")
LEN = 12000


def send_msg(frame):
    for i in range(0, len(frame), LEN):
        data = frame[i:i+LEN]
        if len(data) == LEN:
            my_socket.sendto(data, (IP, PORT))
        else:
            my_socket.sendto(data + b"done", (IP, PORT))


def recv_msg():
    while True:
        data = b""
        msg, add = my_socket.recvfrom(LEN)
        while b"done" not in msg:
            data += msg
            msg, add = my_socket.recvfrom(LEN)
        data += msg.split(b"done")[0]
        numpy_img = numpy.frombuffer(data, dtype=numpy.uint8)
        numpy_img = numpy_img.reshape((720, 1280, 4))
        img = Image.fromarray(numpy_img)
        tk_img = ImageTk.PhotoImage(image=img)
        main_label.configure(image=tk_img)
        main_label.tk_img = tk_img


def video_stream():
    ret, frame = cap.read()
    cv2image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGBA)
    bytes_img = cv2image.tobytes()
    send_msg(bytes_img)
    main_label.after(50, video_stream)


def main():
    video_stream()
    t1 = threading.Thread(target=recv_msg)
    t1.start()
    root.mainloop()


if __name__ == '__main__':
    main()

Error:

Exception in thread Thread-1:
Traceback (most recent call last):
  File "C:UserseitanAnaconda3libhreading.py", line 917, in _bootstrap_inner
    self.run()
  File "C:UserseitanAnaconda3libhreading.py", line 865, in run
    self._target(*self._args, **self._kwargs)
  File "E:/Chat/ex_client.py", line 38, in recv_msg
    numpy_img = numpy_img.reshape((720, 1280, 4))
ValueError: cannot reshape array of size 3182400 into shape (720,1280,4)

Would really appreciate help! thank you very much!

Answer 1

In the error message, the size matches 1105x720x4 (not 1280x720x4).

Does it work if you set it to that resolution?

numpy_img = numpy_img.reshape((720, 1105, 4))
img = Image.fromarray(numpy_img)

Also, can you make a test run, which does not forms an image etc., but just prints that the image is received, the size of data etc. and see is that consistent (e.g. the camera actually capturing in that resolution? or it is so because of some default size of the camera window? - I don't know).

data = b""
data += msg.split(b"done")[0]

That would be inefficient, I think you would better use a numpy array with the expected size and use slicing like you split the frames: [i:i+LEN] etc.

Also, it'd be a good idea to start with hand-shake about the resolution and format (and I guess you realize that, sending it uncompressed as it is an overkill even for LAN).

You may try with different packet sizes (LEN=12000) in order to see whether the error is the same:

 LEN=...
 msg, add = my_socket.recvfrom(LEN)