I am Using WiFi UDP to send numbers from my computer to an adafruit led matrix display with arduino mega 2560 wifi R3 but it only sends once

Question

I am using an Songhe Mega2560 + WiFi R3 ATmega2560+ESP8266 32Mb Memory USB-TTL CH340G to power and send numbers to a 64x32 adafruit led matrix display from my computer. I am kind of a beginner at this but I got it to be able to send numbers to the display from my computer. The only problem is it only works once but I am able to source the problem. To start the led matrix display you need a function called 'matrix.begin()' but when I comment out this function and have it print to the serial it works just fine and I can send numbers to the board over and over again. Another thing is that this function can not go into the setup function because it messes up the initialization of the wifi for some reason. These are the manual settings for the board: Switches 1, 2, 3, and 4 are on and it is switched to TXD3.

#include <WiFiEsp.h>
#include <WiFiEspUdp.h>

// Emulate Serial1 on pins 6/7 if not present
#ifndef HAVE_HWSERIAL1
#include "SoftwareSerial.h"
SoftwareSerial Serial1(6, 7); // RX, TX
#endif

char ssid[] = "mynetwork";            //  network SSID
char pass[] = "mypassword";        //  network password
int status = WL_IDLE_STATUS;     // the Wifi radio's status

unsigned int localPort = 10002;  // local port to listen on

char packetBuffer[255];          // buffer to hold incoming packet
int count = 0;
int pos = 0;
WiFiEspUDP Udp;

//for the display
#include <RGBmatrixPanel.h>
#include <String.h>

#define CLK 11
#define OE   9
#define LAT 10
#define A   A0
#define B   A1
#define C   A2
#define D   A3

bool isMatrixSetup = false;

RGBmatrixPanel matrix(A, B, C, D, CLK, LAT, OE, false, 64);

void setup() {
  // initialize serial for debugging
  Serial.begin(115200);
  // initialize serial for ESP module
  Serial3.begin(115200);
  // initialize ESP module
  WiFi.init(&Serial3);

  // check for the presence of the shield:
  if (WiFi.status() == WL_NO_SHIELD) {
    Serial.println("WiFi shield not present");
    // don't continue:
    while (true);
  }

  // attempt to connect to WiFi network
  while ( status != WL_CONNECTED) {
    Serial.print("Connecting to: ");
    Serial.println(ssid);
    // Connect to WPA/WPA2 network
    status = WiFi.begin(ssid, pass);
  }
  
  printWifiStatus();

  // if you get a connection, report back via serial:
  Udp.begin(localPort);
}

void loop() {

  // if there's data available, read a packet
  int packetSize = Udp.parsePacket();
  if (packetSize) {

    // read the packet into packetBufffer
    int len = Udp.read(packetBuffer, 255);
    if (len > 0) {
      packetBuffer[len] = 0;
    }
  
    //run setupmatrix only once
    if (!isMatrixSetup) {
      setupmatrix();
      isMatrixSetup = true;
    }

    count = atoi(packetBuffer);
    Serial.println(count);
    
    //To keep the text centered  
    String countStr = String(count);
    int digitCount = countStr.length();
    pos = (33 - (6 * digitCount));
    Serial.println(pos);
    matrix.setCursor(pos, 15);
    
    //print count on display
    matrix.setTextSize(2);
    matrix.setTextColor(matrix.Color333(7,7,7));
    matrix.fillRect(0, 11, matrix.width(), matrix.height() - 11, matrix.Color333(0, 0, 0)); 
    matrix.println(count);
  }
}


void printWifiStatus() {

  // print your WiFi shield's IP address:
  IPAddress ip = WiFi.localIP();
  Serial.print("IP Address: ");
  Serial.println(ip);
}

void setupmatrix() {
  delay(5000);
  
  matrix.begin();

  //matrix.fillScreen(matrix.Color333(0, 0, 0));

  matrix.setTextSize(1);     // size 1 == 8 pixels high
  matrix.setTextWrap(false); // wrap at end of line

  matrix.setCursor(7, 2);
  matrix.setTextColor(matrix.Color333(255,130,0));
  matrix.println("total: ");
  Serial.println("setup");
}

I tried changing the sequencing of the program and moving matrix.begin() to different places and I expected it to receive numbers from my computer more than once.