Customizing TwoDArrayWritable in Hadoop and Not able to iterate the same in reducer

Question

Trying to emit 2 double dimensional array as value from mapper. In hadoop we have TwoDArrayWritable which takes 1 - 2D array as input. In order to achieve my usecase, I tried to edit TwoDArrayWritable to take input of 2 - 2D array

/**
 * A Writable for 2D arrays containing a matrix of instances of a class.
 */
public class MyTwoDArrayWritable implements Writable {


    private Class valueClass;
    private Writable[][] values;


    private Class valueClass1;
    private Writable[][] values1;


    public MyTwoDArrayWritable(Class valueClass,Class valueClass1) {
        this.valueClass = valueClass;
        this.valueClass1 = valueClass1;
    }

    public MyTwoDArrayWritable(Class valueClass, DoubleWritable[][] values,Class valueClass1, DoubleWritable[][] values1) {
        this(valueClass, valueClass1);
        this.values = values;
        this.values1 = values1;
    }

    public Object toArray() {
        int dimensions[] = {values.length, 0};
        Object result = Array.newInstance(valueClass, dimensions);
        for (int i = 0; i < values.length; i++) {
            Object resultRow = Array.newInstance(valueClass, values[i].length);
            Array.set(result, i, resultRow);
            for (int j = 0; j < values[i].length; j++) {
                Array.set(resultRow, j, values[i][j]);
            }
        }
        return result;
    }



    /**
     * @return the valueClass
     */
    public Class getValueClass() {
        return valueClass;
    }

    /**
     * @param valueClass the valueClass to set
     */
    public void setValueClass(Class valueClass) {
        this.valueClass = valueClass;
    }

    /**
     * @return the values
     */
    public Writable[][] getValues() {
        return values;
    }

    /**
     * @param values the values to set
     */
    public void setValues(DoubleWritable[][] values,DoubleWritable[][] values1) {
        this.values = values;
        this.values = values1;
    }

    /**
     * @return the valueClass1
     */
    public Class getValueClass1() {
        return valueClass1;
    }

    /**
     * @param valueClass1 the valueClass1 to set
     */
    public void setValueClass1(Class valueClass1) {
        this.valueClass1 = valueClass1;
    }

    /**
     * @return the values1
     */
    public Writable[][] getValues1() {
        return values1;
    }


    public void readFields(DataInput in) throws IOException {
        // construct matrix
        values = new Writable[in.readInt()][];
        for (int i = 0; i < values.length; i++) {
            values[i] = new Writable[in.readInt()];
        }

        // construct values
        for (int i = 0; i < values.length; i++) {
            for (int j = 0; j < values[i].length; j++) {
                Writable value;                             // construct value
                try {
                    value = (Writable) valueClass.newInstance();
                } catch (InstantiationException e) {
                    throw new RuntimeException(e.toString());
                } catch (IllegalAccessException e) {
                    throw new RuntimeException(e.toString());
                }
                value.readFields(in);                       // read a value
                values[i][j] = value;                       // store it in values
            }
        }
    }

    public void write(DataOutput out) throws IOException {
        out.writeInt(values.length);                 // write values
        for (int i = 0; i < values.length; i++) {
            out.writeInt(values[i].length);
        }
        for (int i = 0; i < values.length; i++) {
            for (int j = 0; j < values[i].length; j++) {
                values[i][j].write(out);
            }
        }
    }


}

And emited 2 2D double array from mapper.

MyTwoDArrayWritable array = new MyTwoDArrayWritable (DoubleWritable.class,DoubleWritable.class);
DoubleWritable[][] myInnerArray = new DoubleWritable[EtransEkey.length][EtransEkey[0].length];
DoubleWritable[][] myInnerArray1 = new DoubleWritable[EtransDevalue.length][EtransDevalue[0].length];
// set values in myInnerArray
for (int k1 = 0; k1 < EtransEkey.length; k1++) {
 for(int j1=0;j1< EtransEkey[0].length;j1++){
     myInnerArray[k1][j1] = new DoubleWritable(EtransEkey[k1][j1]);

 }
}

for (int k1 = 0; k1 < EtransDevalue.length; k1++) {
 for(int j1=0;j1< EtransDevalue[0].length;j1++){
     myInnerArray1[k1][j1] = new DoubleWritable(EtransDevalue[k1][j1]);
 }
}

array.set(myInnerArray,myInnerArray1); 

Showing error in array.set(myInnerArray,myInnerArray1);

/*
 * The method set(DoubleWritable[][], DoubleWritable[][]) is undefined for the type MyTwoDArrayWritableritable
 */

EDIT: How to iterate through these values in Reducer to get myInnerArray matrix and myInnerArray1 matrix?

So far what I did is

for (MyTwoDArrayWritable c : values) {
            System.out.println(c.getValues());
            DoubleWritable[][] myInnerArray = new DoubleWritable[KdimRow][KdimCol];
            for (int k1 = 0; k1 < KdimRow; k1++) {
                 for(int j1=0;j1< KdimCol;j1++){
                     myInnerArray[k1][j1] = new DoubleWritable();

                 }
        }

But how to store them back to a double array?

Answer 1

You have not defined the set method in MyTwoDArrayWritable, that is why that error is shown. Instead of calling array.set, you should use the method you have already defined which does exactly what you need: setValues, so replace

array.set(myInnerArray,myInnerArray1); 

with

array.setValues(myInnerArray,myInnerArray1);