How do I stop audio lag in java?

Question

I am making a skype like program. I have a "VoiceChat" class to handle the voice chat connections. I have a "ClientAudio" class to handle the client audio output. I also have a "ClientAudioRec" to handle receiving audio from the server and playing it. In the "ClientAudio" class I store the audio data in a buffer until it is full to try and reduce lag. Then I send it to the server. If I have two people in the call it lags unless the buffer is at 1000. When I add another person to the call it lags so much that you can't understand anyone. But I don't want to extend the buffer every time someone else joins because 1000 is already like a two-second delay. If anyone knows how to reduce the lag or make it almost instant that would really help. thank you in advance!!!

VoiceChat (Server Side)

import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.net.Socket;

public class VoiceChat extends Thread {
    private Socket s;
    private ObjectOutputStream out;
    private ObjectInputStream in;
    private Thread acceptThread;
    private int VoicePort;
    private int num;

    public VoiceChat(Socket sVoice, int VoicePort, Thread acceptThread) {
        s = sVoice;
        this.VoicePort = VoicePort;
        this.acceptThread = acceptThread;
        num = chat.threads.indexOf(this.acceptThread)+1;
        try {
            out = new ObjectOutputStream(s.getOutputStream());
            in = new ObjectInputStream(s.getInputStream());
            if(VoicePort <= 65511) {
                chat.users1_Voice.add(out);
            }else {
                chat.users2.add(out);
            }
        } catch (IOException e) {
            System.out.println("ERROR2");
            e.printStackTrace();
        }

    }
    public void run() {
            int bytesRead = 0;
            byte[] inBytes = new byte[100];
            while(bytesRead != -1) {
                    try {
                        bytesRead = in.read(inBytes, 0, inBytes.length);
                        }catch (IOException e) {
                        System.out.println("ERROR2");
                        if(VoicePort <= 65511) {
                            chat.users1_Voice.remove(out);
                        }else {
                            chat.users2.remove(out);
                        }
                    }
                    if(bytesRead > 0) {
                        System.out.println(chat.users1_Voice);
                            send(inBytes, bytesRead, out);
                            System.out.println("SENDING DATA");
                    }
              }
        }

    public void send(byte[] soundData, int bytesRead, ObjectOutputStream out) {
        if(VoicePort <= 65511) {
            for(ObjectOutputStream o : chat.users1_Voice) {
                try {
                    if(o != out) {
                        o.write(soundData, 0, bytesRead);
                        o.flush();
                    }
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }else {
            for(ObjectOutputStream o : chat.users2) {
                try {
                    if(o != out) {
                        o.write(soundData, 0, bytesRead);
                        o.flush();
                    }
                } catch (IOException e) {
                }
            }
        }
    }
}

ClientAudio

import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.net.Socket;
import javax.sound.sampled.AudioFormat;
import javax.sound.sampled.AudioSystem;
import javax.sound.sampled.DataLine;
import javax.sound.sampled.LineUnavailableException;
import javax.sound.sampled.TargetDataLine;

public class ClientAudio implements Runnable {
    private ObjectOutputStream out = null;
    private AudioFormat af;
    private TargetDataLine microphone;

ClientAudio(AudioFormat af) {
    this.af = af;
}
public void run() {
    try {
        client.voiceSocket = new Socket("***IP***", client.VoicePort);
        Thread car = new Thread(new ClientAudioRec(af));
        car.start();
        out = new ObjectOutputStream(client.voiceSocket.getOutputStream());
        DataLine.Info info = new DataLine.Info(TargetDataLine.class, af);
        microphone = (TargetDataLine)AudioSystem.getLine(info);
        microphone.open(af);
        microphone.start();
        int bytesRead = 0;
        int offset = 0;
        byte[] soundData = new byte[1000];
        while(bytesRead != -1 && client.callRunning == true) {
            bytesRead = microphone.read(soundData, 0, soundData.length);
            if(bytesRead >= 0) {
                offset += bytesRead;
                if(offset == soundData.length) {
                    out.write(soundData, 0, bytesRead);
                    offset = 0;
                }
            }
        }
        microphone.close();
        try {
            client.voiceSocket.close();
        } catch (IOException e1) {
            e1.printStackTrace();
        }
        System.out.println("Client Audio Stopped Because Client Disconnected From Call");
    } catch (IOException | LineUnavailableException e) {
        microphone.close();
        try {
            client.voiceSocket.close();
        } catch (IOException e1) {
            System.out.println("Plug Microphone In...");
        }
        System.out.println("Client Audio Stopped...");
    }
}
}

ClientAudioRec

import java.io.IOException;
import java.io.ObjectInputStream;
import java.net.Socket;
import javax.sound.sampled.AudioFormat;
import javax.sound.sampled.AudioSystem;
import javax.sound.sampled.DataLine;
import javax.sound.sampled.LineUnavailableException;
import javax.sound.sampled.SourceDataLine;

public class ClientAudioRec implements Runnable {
    private SourceDataLine inSpeaker = null;
    private AudioFormat audioformat;
    private ObjectInputStream in;

ClientAudioRec(AudioFormat af) {
    audioformat = af;
    try {
        in = new ObjectInputStream(client.voiceSocket.getInputStream());
    } catch (IOException e) {
        e.printStackTrace();
    }
}

public void run() {
    DataLine.Info info = new DataLine.Info(SourceDataLine.class, audioformat);
    try {
        inSpeaker = (SourceDataLine)AudioSystem.getLine(info);
        inSpeaker.open(audioformat);
    } catch (LineUnavailableException e1) {
        System.out.println("ERROR");
        e1.printStackTrace();
    }

    int bytesRead = 0;
    byte[] inSound = new byte[100000];
    inSpeaker.start();
    while(bytesRead != -1 && client.callRunning == true) {
        try{bytesRead = in.read(inSound, 0, inSound.length);} catch (Exception e){
            inSpeaker.close();
            System.out.println("Speaker Closed");
        }
        if(bytesRead >= 0) {
            inSpeaker.write(inSound, 0, bytesRead);
        }
    }
    inSpeaker.close();
    System.out.println("Speaker Closed Because Client Disconneted From Call");
}
}