Overfitting Deep Neural Network with Regression

Question

I build a neural network with regression for predict a prime of insurance data. My Loss function take a very high value ( like 123000) and decrease unitl 30000; my accuracy remain constant at 1. I'm in a situation of overfitting?

In Traing set I use a regularization, drpout, to avoid an overfitting situation

This is the code:

##########
#Parameters
##########
learning_rate = 0.001
training_epochs = 150
#Session 2
batch_size = 1000
display_step = 1
total_len = train_x.shape[0]
# Network Parameters
n_hidden_1 = 150 # 1st layer number of features
n_hidden_2 = 150 # 2nd layer number of features
#n_hidden_3 = 150
#n_hidden_4 = 150
n_input = train_x.shape[1]
n_classes = 1

###############
#tf Graph input
###############

x = tf.placeholder("float", [None, train_x.shape[1]])
y = tf.placeholder("float", [None,n_classes])
keep_prob = tf.placeholder(tf.float32)

#######
#Model
#######

####
#Dropout
####
def multilayer_perceptron_drop(x, weights, biases):
    # Hidden layer with RELU activation
    layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])
    layer_1 = tf.nn.relu(layer_1)
    # DROP-OUT 
    drop_out_1 = tf.nn.dropout(layer_1, keep_prob)  
    # Hidden layer with RELU activation
    #layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2'])
    layer_2 = tf.add(tf.matmul(drop_out_1, weights['h2']), biases['b2'])
    layer_2 = tf.nn.relu(layer_2)
    # DROP-OUT 
    drop_out_2 = tf.nn.dropout(layer_2, keep_prob)  
    # Hidden layer with RELU activation
    #layer_3 = tf.add(tf.matmul(layer_2, weights['h3']), biases['b3'])
    #layer_3 = tf.nn.relu(layer_3)
    # Hidden layer with RELU activation
    #layer_4 = tf.add(tf.matmul(layer_3, weights['h4']), biases['b4'])
    #layer_4 = tf.nn.relu(layer_4)
    # Output layer with linear activation
    out_layer = tf.matmul(drop_out_2, weights['out']) + biases['out']
    return out_layer

#################
# Without dropout
#################

def multilayer_perceptron_test(x, weights, biases):
    # Hidden layer with RELU activation
    layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])
    layer_1 = tf.nn.relu(layer_1)  
    # Hidden layer with RELU activation
    layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2'])
    layer_2 = tf.nn.relu(layer_2)  
    # Output layer with linear activation
    out_layer = tf.matmul(layer_2, weights['out']) + biases['out']
    return out_layer


##################################
#Definizione layers weight & bias
##################################
weights = {
    'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1], 0,0.01, dtype=tf.float32)),
    'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2], 0,0.1,dtype=tf.float32)),
    #'h3': tf.Variable(tf.random_normal([n_hidden_2, n_hidden_3])),
    #'h4': tf.Variable(tf.random_normal([n_hidden_3, n_hidden_4])),
    'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes], 0, 0.1,dtype=tf.float32))
}
biases = {
    'b1': tf.Variable(tf.random_normal([n_hidden_1], 0,0.1,dtype=tf.float32)),
    'b2': tf.Variable(tf.random_normal([n_hidden_2], 0,0.1,dtype=tf.float32)),
    #'b3': tf.Variable(tf.random_normal([n_hidden_3])),
    #'b4': tf.Variable(tf.random_normal([n_hidden_4])),
    'out': tf.Variable(tf.random_normal([n_classes], 0, 0.1,dtype=tf.float32))
}

pred_drop = multilayer_perceptron_drop(x, weights, biases)
pred_test = multilayer_perceptron_test(x, weights, biases)

#####
#MSE 
#####
cost = tf.reduce_mean(tf.square(pred_drop-y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)

cost_test = tf.reduce_mean(tf.square(pred_test-y))

optimizer_test = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost_test)

#Accuracy
correct_prediction = tf.equal(tf.argmax(pred_drop, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))

correct_prediction_test = tf.equal(tf.argmax(pred_test, 1), tf.argmax(y, 1))
accuracy_test = tf.reduce_mean(tf.cast(correct_prediction_test, "float"))