What is the condition to stop splitting data in Non-Uniform Quantizer?

Question

I'm trying to implement a non-uniform quantizer with N-level of quantization. I have already done some work and it works, the problem is it goes into infinite loop when N(the number of levels) exceeds "4".

If anyone can point out any hints to know where the wrong is, I'd appreciate it.

public static Vector<Integer> split(Vector <Integer> image,float average,int n)
{

float lowerAverage = average - 1;
float upperAverage = average + 1;
Vector<Float> averages = new Vector<Float>();
Vector <Integer> leftData = new Vector <Integer>();
Vector <Integer> rightData = new Vector <Integer>();
averages.add(lowerAverage);
averages.add(upperAverage);

//FIND ALL AVERAGES
    while (averages.size()<n)
    {
       //I THINK THE PROBLEM HAPPENS HERE AS IT KEEP PRINTING "loop"
        System.out.println("loop");
        for (int i = 0; i < image.size(); i++) 
        {
            if (Math.abs(image.get(i)-lowerAverage) <= Math.abs(image.get(i)-upperAverage))
            {
                leftData.add(image.get(i));

            }
            else
            {
                rightData.add(image.get(i));
            }

        }
        lowerAverage = average(leftData) - 1;
        upperAverage = average (leftData) + 1;

        averages.removeAllElements();
        averages.add(lowerAverage);
        averages.add(upperAverage);

        lowerAverage = average(rightData) - 1;
        upperAverage = average (rightData) + 1;
        averages.add(lowerAverage);
        averages.add(upperAverage);
    }
//*****************************************************************************************
//CREATE DATASETS WITH NUMBER == AVERAGES.SIZE()
    Vector <DataSet> v = new Vector <DataSet>();
    for (int i = 0; i < averages.size(); i++) {
        DataSet temp = new DataSet();
        temp.setName(averages.get(i));
        v.add(temp);
    }
//*****************************************************************************************
//SPLIT ORIGINAL DATA ACCORDING TO AVERAGES
    float name;
    for (int i = 0; i < image.size(); i++) 
    {
        float min =Math.abs(image.get(i)-averages.get(0));
        name = averages.get(0);
        for (int j = 1; j < averages.size(); j++) 
        {
            if (Math.abs(image.get(i)-averages.get(j)) < min)
                    {
                        min = Math.abs(image.get(i)-averages.get(j));
                        name = averages.get(j);
                    }
        }
        getDataset(v, name).addData(image.get(i));

    }
//*****************************************************************************************
//CALCULATE EACH DATASET AVERAGE
    for (int i = 0; i < v.size(); i++) 
    {
        v.get(i).UpdateAverage();

    }
//*****************************************************************************************
//THIS IS JUST FOR TESTING
Vector<Integer> Qinv = new Vector<Integer>();
for (int i = 0; i < v.size(); i++) 
{
    Qinv.add(v.get(i).getAverage());
}


return Qinv;


}