I am trying to add a FastText embedding layer to the famous text classification architecture with CNN: https://github.com/dennybritz/cnn-text-classification-tf
I load my FastText embedding like this:
embedding = np.load('embedding/my_embedding_file')
# type(embedding): gensim.models.fasttext.FastText
vocab = embedding.wv.vocab
vocab_processor.fit(vocab)
x_train = np.array(list(vocab_processor.transform(x_train)))
x_dev = np.array(list(vocab_processor.transform(x_dev)))
vocabulary_size = len(vocab) #300k
And below is how I add my embedding layer in Tensorflow:
# Embedding layer
with tf.device('/cpu:0'), tf.name_scope("embedding"):
embedding_weights_ = tf.Variable(tf.constant(0.0, shape=[vocab_size, embedding_size]), trainable=False, name="embedding_weights")
embedding_weights = tf.assign(embedding_weights_, self.embedding_placeholder)
self.embedded_chars = tf.nn.embedding_lookup(embedding_weights, self.input_x)
self.embedded_chars_expanded = tf.expand_dims(self.embedded_chars, -1)
# Create a convolution + maxpool layer for each filter size
pooled_outputs = []
for i, filter_size in enumerate(filter_sizes):
with tf.name_scope("conv-maxpool-%s" % filter_size):
# Convolution Layer
filter_shape = [filter_size, embedding_size, 1, num_filters]
W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name="W")
b = tf.Variable(tf.constant(0.1, shape=[num_filters]), name="b")
conv = tf.nn.conv2d(
self.embedded_chars_expanded, #my embedding to the next layer
W,
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
Initialization of the CNN model:
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
sess = tf.Session(config=session_conf)
with sess.as_default():
cnn = TextCNN(
sentence_length = max_document_length,
num_classes=17,
vocab_size=len(vocab_processor.vocabulary_),
embedding_size=FLAGS.embedding_dim,
filter_sizes=list(map(int, FLAGS.filter_sizes.split(","))),
num_filters=FLAGS.num_filters,
l2_reg_lambda=FLAGS.l2_reg_lambda)
Single training step:
def train_step(x_batch, y_batch):
feed_dict = {
cnn.input_x: x_batch,
cnn.input_y: y_batch,
cnn.dropout_keep_prob: FLAGS.dropout_keep_prob,
cnn.embedding_placeholder:embedding
}
_, step, summaries, loss, accuracy = sess.run(
[train_op, global_step, train_summary_op, cnn.loss, cnn.accuracy],
feed_dict)
time_str = datetime.datetime.now().isoformat()
print("{}: step {}, loss {:g}, acc {:g}".format(time_str, step, loss, accuracy))
train_summary_writer.add_summary(summaries, step)
For the reference, below is how my x_train and y_train looks like:
When I start training, I get this error:
File "<ipython-input-24-2ec193e9123f>", line 122, in <module>
train_step(x_batch, y_batch)
File "<ipython-input-24-2ec193e9123f>", line 92, in train_step
feed_dict)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 889, in run
run_metadata_ptr)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1089, in _run
np_val = np.asarray(subfeed_val, dtype=subfeed_dtype)
File "C:\ProgramData\Anaconda3\lib\site-packages\numpy\core\numeric.py", line 492, in asarray
return array(a, dtype, copy=False, order=order)
ValueError: setting an array element with a sequence.
This error seems like a numpy array creation error, where it is tried to create an array element as a sequence. BUT: I don't receive any error when I remove the embedding while my train data is completely the same thus batches are as well.
What could cause this error?