As mentioned in the title, I implemented a stacked autoencoder using tensorflow to compress image data into lower dimensions. However, the reconstructed image, after learning finished, contains static noise across the entire dataset. So I tried to only train the autoencoder with one black image, the learning curve stays the same after reproducing this noisy image.
(upper left is the learning curve, bottom left is the all black image and bottom right is the reconstructed noisy image)
Here is how I defined my stacked autoencoder. The mask is used when I want to make it a denoising autoencoder, which in this case not used at all.
# Hyper parameters
learning_rate = 0.0005
training_epochs = 5000
batch_size = 1
total_batch = 1
corruption = 0.3
# Network parameters
n_hidden_6 = 30
n_hidden_5 = 100
n_hidden_4 = 250
n_hidden_3 = 500
n_hidden_2 = 800
n_hidden_1 = 1000
n_input = 1200
# tf Graph input
X = tf.placeholder("float", [None, n_input])
mask = tf.placeholder("float", [None, n_input])
weights = {
'encoder_h1': tf.Variable(tf.random_normal([n_input, n_hidden_1]), name = 'encoder_h1'),
'encoder_h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2]), name = 'encoder_h2'),
'encoder_h3': tf.Variable(tf.random_normal([n_hidden_2, n_hidden_3]), name = 'encoder_h3'),
'encoder_h4': tf.Variable(tf.random_normal([n_hidden_3, n_hidden_4]), name = 'encoder_h4'),
'encoder_h5': tf.Variable(tf.random_normal([n_hidden_4, n_hidden_5]), name = 'encoder_h5'),
'encoder_h6': tf.Variable(tf.random_normal([n_hidden_5, n_hidden_6]), name = 'encoder_h6'),
'decoder_h1': tf.Variable(tf.random_normal([n_hidden_6, n_hidden_5]), name = 'decoder_h1'),
'decoder_h2': tf.Variable(tf.random_normal([n_hidden_5, n_hidden_4]), name = 'decoder_h2'),
'decoder_h3': tf.Variable(tf.random_normal([n_hidden_4, n_hidden_3]), name = 'decoder_h3'),
'decoder_h4': tf.Variable(tf.random_normal([n_hidden_3, n_hidden_2]), name = 'decoder_h4'),
'decoder_h5': tf.Variable(tf.random_normal([n_hidden_2, n_hidden_1]), name = 'decoder_h5'),
'decoder_h6': tf.Variable(tf.random_normal([n_hidden_1, n_input]), name = 'decoder_h6'),
}
biases = {
'encoder_b1': tf.Variable(tf.random_normal([n_hidden_1]), name = 'encoder_b1'),
'encoder_b2': tf.Variable(tf.random_normal([n_hidden_2]), name = 'encoder_b2'),
'encoder_b3': tf.Variable(tf.random_normal([n_hidden_3]), name = 'encoder_b3'),
'encoder_b4': tf.Variable(tf.random_normal([n_hidden_4]), name = 'encoder_b4'),
'encoder_b5': tf.Variable(tf.random_normal([n_hidden_5]), name = 'encoder_b5'),
'encoder_b6': tf.Variable(tf.random_normal([n_hidden_6]), name = 'encoder_b6'),
'decoder_b1': tf.Variable(tf.random_normal([n_hidden_5]), name = 'decoder_b1'),
'decoder_b2': tf.Variable(tf.random_normal([n_hidden_4]), name = 'decoder_b2'),
'decoder_b3': tf.Variable(tf.random_normal([n_hidden_3]), name = 'decoder_b3'),
'decoder_b4': tf.Variable(tf.random_normal([n_hidden_2]), name = 'decoder_b4'),
'decoder_b5': tf.Variable(tf.random_normal([n_hidden_1]), name = 'decoder_b5'),
'decoder_b6': tf.Variable(tf.random_normal([n_input]), name = 'decoder_b6'),
}
# Building the encoder
def encoder(x, mask):
# corruption
x_c = x * mask
# Encoder Hidden layer with sigmoid activation #1
layer_1 = tf.nn.sigmoid(tf.add(tf.matmul(x_c, weights['encoder_h1']),
biases['encoder_b1']))
# Encoder Hidden layer with sigmoid activation #2
layer_2 = tf.nn.sigmoid(tf.add(tf.matmul(layer_1, weights['encoder_h2']),
biases['encoder_b2']))
# Encoder Hidden layer with sigmoid activation #3
layer_3 = tf.nn.sigmoid(tf.add(tf.matmul(layer_2, weights['encoder_h3']),
biases['encoder_b3']))
# Encoder Hidden layer with sigmoid activation #4
layer_4 = tf.nn.sigmoid(tf.add(tf.matmul(layer_3, weights['encoder_h4']),
biases['encoder_b4']))
# Encoder Hidden layer with sigmoid activation #5
layer_5 = tf.nn.sigmoid(tf.add(tf.matmul(layer_4, weights['encoder_h5']),
biases['encoder_b5']))
# Encoder Hidden layer with sigmoid activation #6
layer_6 = tf.nn.sigmoid(tf.add(tf.matmul(layer_5, weights['encoder_h6']),
biases['encoder_b6']))
return layer_6
# Building the decoder
def decoder(x):
# Decoder Hidden layer with sigmoid activation #1
layer_1 = tf.nn.sigmoid(tf.add(tf.matmul(x, weights['decoder_h1']),
biases['decoder_b1']))
# Decoder Hidden layer with sigmoid activation #2
layer_2 = tf.nn.sigmoid(tf.add(tf.matmul(layer_1, weights['decoder_h2']),
biases['decoder_b2']))
# Decoder Hidden layer with sigmoid activation #3
layer_3 = tf.nn.sigmoid(tf.add(tf.matmul(layer_2, weights['decoder_h3']),
biases['decoder_b3']))
# Decoder Hidden layer with sigmoid activation #4
layer_4 = tf.nn.sigmoid(tf.add(tf.matmul(layer_3, weights['decoder_h4']),
biases['decoder_b4']))
# Decoder Hidden layer with sigmoid activation #5
layer_5 = tf.nn.sigmoid(tf.add(tf.matmul(layer_4, weights['decoder_h5']),
biases['decoder_b5']))
# Decoder Hidden layer with sigmoid activation #6
logit = tf.add(tf.matmul(layer_5, weights['decoder_h6']), biases['decoder_b6'])
layer_6 = tf.nn.sigmoid(logit)
return layer_6, logit
# Construct model
encoder_op = encoder(X, mask)
decoder_op, logit = decoder(encoder_op)
# Prediction
y_pred = decoder_op
y_true = X
# Define loss and optimizer, minimize the squared error
cost = tf.reduce_mean(tf.pow(y_true - y_pred, 2))
#optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)
#cost = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels = y_true, logits = logit))
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)
# Initializing the variables
init = tf.global_variables_initializer()
Can anyone please help me explain what can be the potential problems of this implementation? Is it better to use other loss functions or optimizers? Thanks in advance!
