I build a encoder-decoder model with attention in Keras and it worked well. Then I trained this model for a long time and saved the model. Now I want to restore encoder-decoder from model but there is a problem in dot layer.
Build model:
encoder_inputs = Input(shape=(None,))
embed_encoder = Embedding(characters_length, 10)(encoder_inputs)
encoder_output, forward_h, forward_c, backward_h, backward_c = Bidirectional(LSTM(lstm_dim, return_sequences=True, return_state=True))(embed_encoder)
state_h = Concatenate()([forward_h, backward_h])
state_c = Concatenate()([forward_c, backward_c])
encoder_states = [state_h, state_c]
decoder_inputs = Input(shape=(None,))
embed_decoder = Embedding(characters_length, 10)(decoder_inputs)
decoder_output, _, _ = LSTM(2*lstm_dim, return_sequences=True, return_state=True)(embed_decoder, initial_state=encoder_states)
# Attention layer
attention = dot([decoder_output, encoder_output], axes=[2, 2])
attention = Activation('softmax')(attention)
context = dot([attention, encoder_output], axes=[2,1])
decoder_concat_output = concatenate([context, decoder_output])
decoder_predicte = Dense(characters_length, activation='softmax')(decoder_concat_output)
model = Model([encoder_inputs, decoder_inputs], decoder_predicte)
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
model.fit([encoder_input_data, decoder_input_data], decoder_target_data, batch_size=512, epochs=5, validation_split=0.2)
model.save("model5.h5")
Restore encoder-decoder:
model = models.load_model("model5.h5")
encoder_inputs = model.input[0] # input_1
encoder_outputs, forward_h, forward_c, backward_h, backward_c = model.layers[3].output # bidirectioanal_lstm_1
state_h = Concatenate()([forward_h, backward_h])
state_c = Concatenate()([forward_c, backward_c])
encoder_states = [state_h, state_c]
encoder_model = Model(encoder_inputs, encoder_states)
decoder_inputs = model.input[1] # input_2
embed_decoder = model.layers[4] # embedding layer
embed_decoder = embed_decoder(decoder_inputs)
decoder_state_h = Input(shape=(2*latent_dim,))
decoder_state_c = Input(shape=(2*latent_dim,))
decoder_states_inputs = [decoder_state_h, decoder_state_c]
decoder_lstm = model.layers[7]
decoder_outputs, state_h_dec, state_c_dec = decoder_lstm(embed_decoder, initial_state=decoder_states_inputs)
decoder_states = [state_h_dec, state_c_dec]
encoder_as_input = Input(shape=(None, 2*latent_dim,))
dot_layer = model.layers[8] # dot layer
attention = dot_layer([decoder_outputs, encoder_as_input])
attention = model.layers[9](attention) # Activation layer
context = model.layers[10]([attention, encoder_as_input]) # dot layer
decoder_concat_output = model.layers[11]([context, decoder_outputs]) # concatenate layer
decoder_dense = model.layers[12] # dense layer
decoder_outputs = decoder_dense(decoder_concat_output)
decoder_model = Model([decoder_inputs] + decoder_states_inputs, [decoder_outputs] + decoder_states)
Here is the error:
ValueError Traceback (most recent call last)
<ipython-input-4-a0086c415583> in <module>()
65 print(decoder_state_c)
66 print(encoder_as_input)
---> 67 decoder_model = Model([decoder_inputs] + decoder_states_inputs, [decoder_outputs] + decoder_states)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/functional.py in _map_graph_network(inputs, outputs)
929 'The following previous layers '
930 'were accessed without issue: ' +
--> 931 str(layers_with_complete_input))
932 for x in nest.flatten(node.outputs):
933 computable_tensors.add(id(x))
ValueError: Graph disconnected: cannot obtain value for tensor Tensor("input_6:0", shape=(None, None, 512), dtype=float32) at layer "dot_36". The following previous layers were accessed without issue: ['embedding_51', 'lstm_51']
How can I restore encoder and decoder when I am using dot layer as attention layer?
