I want to use tf.data library for training speed.
But my code, raise error message, like as bellows.
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training.py", line 727, in fit
use_multiprocessing=use_multiprocessing)
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training_arrays.py", line 675, in fit
steps_name='steps_per_epoch')
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training_arrays.py", line 169, in model_iteration
ins = _prepare_feed_values(model, inputs, targets, sample_weights, mode)
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training_arrays.py", line 535, in _prepare_feed_values
extract_tensors_from_dataset=True)
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training.py", line 2471, in _standardize_user_data
exception_prefix='input')
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training_utils.py", line 517, in standardize_input_data
standardize_single_array(x, shape) for (x, shape) in zip(data, shapes)
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training_utils.py", line 517, in <listcomp>
standardize_single_array(x, shape) for (x, shape) in zip(data, shapes)
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training_utils.py", line 442, in standardize_single_array
if (x.shape is not None and len(x.shape) == 1 and
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/framework/tensor_shape.py", line 827, in __len__
raise ValueError("Cannot take the length of shape with unknown rank.")
ValueError: Cannot take the length of shape with unknown rank.
I don't know why and how can I fix it. My dataset code is as bellows.
def _py_parse_line(line):
line = line.decode('utf-8')
parsed_line = line.split("\t")
label = int(parsed_line[0])
rawSentence = str(parsed_line[1])
morphemePOS = str(parsed_line[2])
NE_LIST = str(parsed_line[3])
preprocessedDatum = FTV.featureToVectorLexMorpDictNElist(
[rawSentence], [morphemePOS], [NE_LIST]
)
syllable_feature = preprocessedDatum[0][0]
sdiDict_feature = preprocessedDatum[1][0]
morphemePos_feature = preprocessedDatum[2][0]
neDict_feautre = preprocessedDatum[3][0]
return syllable_feature, sdiDict_feature, morphemePos_feature, neDict_feautre, label
def _decode_tsv(line):
type_list = [tf.int32, tf.int8, tf.int32, tf.int8, tf.int64]
data_list = tf.py_func(_py_parse_line, [line], type_list)
label = data_list[4]
features = {"lexical_input": data_list[0],
"dictInfo_input": data_list[1],
"morphemePos_input": data_list[2],
"ne_input": data_list[3]
}
d = features, label
return d
dataset = tf.data.TextLineDataset("/root/Workspace/ias_sdi_trainer/generatedModel/sdi_nugu.dtg.wholeData.txt")
dataset = dataset.shuffle(4096)
dataset = dataset.map(_decode_tsv)
dataset = dataset.batch(batchSize )
dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
I saw inside of tensor with "make_oneshot_iterator()" and tf.Session. There was data what I think.
I was this question, too. But It cannot help me. Cannot take the length of Shape with unknown rank
Is there any good way to use tf.data for keras? I want to use data from textline in tf.data
Added
My tensorflow version is 1.14. I used dataset in fit, like as bellows.
sdi_model.fit(dataset, epochs=MAX_EPOC)
I used dataset with generator too. Code is as bellows.
dataFetcher = dataset.make_one_shot_iterator()
def gen(dataFetcher):
with tf.Session() as sess:
while True:
next_elem = dataFetcher.get_next()
x_batch,y_batch = sess.run(next_elem)
yield x_batch, y_batch
sdi_model.fit_generator(gen(dataFetcher), steps_per_epoch=100, epochs=MAX_EPOC)
With generator, I received a error message, "sess is empty graph"
