Why do I have different results for the same model?

Question

I use Skopt to find the best hyper-parameters for my model. I fix all the seed with the following code:

import random
from tensorflow.compat.v1.keras.backend import set_session
from tensorflow.compat.v1.keras.backend import clear_session
from tensorflow.compat.v1.keras.backend import get_session
from numpy.random import seed

# Reset Keras Session
def reset_keras():
    sess = tf.compat.v1.keras.backend.get_session()
    tf.compat.v1.keras.backend.clear_session()
    sess.close()
    sess = tf.compat.v1.keras.backend.get_session()

    try:
        del classifier # this is from global space - change this as you need
    except:
        pass

    # use the same config as you used to create the session
    config = tf.compat.v1.ConfigProto()
    config.gpu_options.per_process_gpu_memory_fraction = 1
    config.gpu_options.visible_device_list = "0"
    tf.compat.v1.keras.backend.set_session(tf.compat.v1.Session(config=config))
    os.environ['PYTHONHASHSEED'] = '2'
    os.environ['TF_DETERMINISTIC_OPS'] = '1'

    seed(2)
    tf.random.set_seed(2)
    random.seed(2)

I use this function before using my model, to clean and restart the seed.

I also fix the seed for each initializer in my model.

def create_model(num_input_nodes,drop_LSTM1,num_LSTM_layers,num_LSTM_nodes,drop_LSTM2,
                     num_input_dense,drop_Dense1,num_dense_layers,num_HLD_nodes,
                     drop_Dense2,optim,activation,loss):

        #start the model making process and create our first layer
            

    
        
        
        model = tf.keras.Sequential()
       #create a loop making a new LSTTM layer for the amount passed to this model.
        #naming the layers helps avoid tensorflow error deep in the stack trace.
        model.add(tf.keras.layers.Embedding(input_dim=vocabulary_size, output_dim=embedding_dim, 
                                            input_length=sequence_length,
                                            embeddings_initializer=tf.keras.initializers.Constant(embedding_matrix),
                                            trainable=True))
        if num_LSTM_layers>0:
            
            model.add(tf.keras.layers.LSTM(num_input_nodes,
                                           dropout=drop_LSTM1,
                                           kernel_initializer=tf.keras.initializers.glorot_uniform(seed=2),
    recurrent_initializer=tf.keras.initializers.Orthogonal(seed=2),
                                           return_sequences=True))
            i=0
    
            while i<num_LSTM_layers-1:
    
                model.add(tf.keras.layers.LSTM(num_LSTM_nodes, return_sequences=True,
                                               kernel_initializer=tf.keras.initializers.glorot_uniform(seed=2),
    recurrent_initializer=tf.keras.initializers.Orthogonal(seed=2), dropout=drop_LSTM2))
                i+=1

            model.add(tf.keras.layers.LSTM(num_LSTM_nodes, dropout=drop_LSTM2,kernel_initializer=tf.keras.initializers.glorot_uniform(seed=2),
    recurrent_initializer=tf.keras.initializers.Orthogonal(seed=2), return_sequences=False))
        else : 
            model.add(tf.keras.layers.LSTM(num_input_nodes, dropout=drop_LSTM1,kernel_initializer=tf.keras.initializers.glorot_uniform(seed=2),
    recurrent_initializer=tf.keras.initializers.Orthogonal(seed=2), return_sequences=False))

        model.add(tf.keras.layers.Dense(num_input_dense,
                         activation=activation,
                                       kernel_initializer=tf.keras.initializers.glorot_uniform(seed=2)))
        model.add(tf.keras.layers.Dropout(drop_Dense1,seed=2))
        if num_dense_layers>0:
            for i in range(num_dense_layers):
                
                model.add(tf.keras.layers.Dense(num_HLD_nodes,
                         activation=activation,
                                               kernel_initializer=tf.keras.initializers.glorot_uniform(seed=2)))
                model.add(tf.keras.layers.Dropout(drop_Dense2,seed=2))
        #add our classification layer.
        model.add(tf.keras.layers.Dense(2,activation='softmax',
                                        kernel_initializer=tf.keras.initializers.glorot_uniform(seed=2)))

        #setup our optimizer and compile

        model.compile(optimizer=optim, loss=loss,
                     metrics=[      tf.keras.metrics.Precision(name='precision'),
          tf.keras.metrics.Recall(name='recall'),F1Score(num_classes=2,name='F1',average='macro')])
        print(model.summary())
        return model

Moreover, my Embedding Layer is initialized with a custom Embedding Matrix. But I already see a difference. Do I need to fix another seed ?

EDIT

Seems I forgot to fix shuffle=Falsein the fit method.

score 0 · Answer 1 · answered Dec 24 '20 at 12:52

0

it's hard to find out exactly what you need to fix in this code, but check if you already fixed data split, dropout, or init values in your model.

answered Dec 24 '20 at 12:52

Devi Khositashvili

556
2
13

it's exactly the same train/test data. I also fix all the dropout and init values, except LSTM dropout but it's impossible to fix it... – isaaccs Dec 24 '20 at 13:41
this should be useful - https://stackoverflow.com/questions/49175704/tensorflow-reproducing-results-when-using-dropout – Devi Khositashvili Dec 24 '20 at 17:08
I already fix the TF seed's in my function `reset_keras` – isaaccs Dec 24 '20 at 18:57

Why do I have different results for the same model?

1 Answers1