Using keras tuner for hyperparameter tuning and extracting trial statistics with Tensorboard workaround

Question

I'm using keras tuner for hyperparameter tuning my sequential neural network in keras. My aim is to let keras tuner do a lot of trials and then save all statistics - loss, hyperparameters, trial numbers and epochs - to a file, so I can plot them myself and get a better overview over how keras tuner conducted the tests. Note that this is a regression, so I'm using mean squared error (mse) as a loss function, not accuracy. Here's an example of my network and the setup of keras tuner (based on the "Getting started" tutorial of keras tuner):

import numpy as np
import keras
from tensorflow.keras.optimizers import Adam
from tensorflow.keras import layers
import keras_tuner
from sklearn.model_selection import train_test_split
from tensorboard.backend.event_processing import event_accumulator

# generate random data
random_data = np.random.rand(100,4)
x_data = random_data[:, :-1]
y_data = random_data[:, -1:]
input_dimensions = len(x_data[0, :])
# split data into train and test
x_train, x_eval, y_train, y_eval = train_test_split(x_data, y_data, test_size=0.3, random_state=101)

# create keras tuner and model
def build_model(hp):
    model = keras.Sequential()
    # test number of layers, number of neurons in each layer and activation function
    for i in range(hp.Int("num_layers", 2, 4)):
        model.add(layers.Dense(
                units=hp.Int(f"units_{i}", min_value=32, max_value=1024, step=32),
                activation=hp.Choice("activation", ["relu", "sigmoid"])))
    model.add(layers.Dense(1, activation="linear"))
    model.compile(optimizer=Adam(learning_rate=0.0005), 
                  loss='mse')

    return model

build_model(keras_tuner.HyperParameters())

tuner = keras_tuner.RandomSearch(
    hypermodel=build_model, 
    objective=keras_tuner.Objective('loss', direction="min"), 
    max_trials=5,
    executions_per_trial=3, 
    overwrite=True, 
    project_name="keras_tuner_test")

For extracting the statistics I'm using the Tensorboard callback method (just to be clear: I don't want to actually use Tensorboard. I want only the data and then decide for myself how to display it) with the following code based on this link or this link:

sample_log_directory = <path to directory>

tensorboard_callback = keras.callbacks.TensorBoard(log_dir=sample_log_directory)
tuner.search(x_train, y_train, epochs=3, validation_data=(x_eval, y_eval), callbacks=[tensorboard_callback])

def extract_history(best_trial):
    acc = []
    val_acc = []
    loss = []
    val_loss = []

    for set_data in ['train', 'validation']:
        if set_data == 'train':
            print(sample_log_directory + best_trial + '/execution0/' + set_data)
            ea = event_accumulator.EventAccumulator(sample_log_directory + best_trial + '/execution0/' + set_data)
            ea.Reload()
                            
            for i in range(len(ea.Scalars('epoch_loss'))):
                acc.append(ea.Scalars('epoch_acc')[i][2])
                loss.append(ea.Scalars('epoch_loss')[i][2])
                #lr.append(ea.Scalars('epoch_lr')[i][2])
    
        if set_data == 'validation':
            ea = event_accumulator.EventAccumulator(sample_log_directory  + best_trial + '/execution0/' + set_data)
            ea.Reload()
            for i in range(len(ea.Scalars('epoch_loss'))):
                val_acc.append(ea.Scalars('epoch_acc')[i][2])
                val_loss.append(ea.Scalars('epoch_loss')[i][2])

    return acc, val_acc, loss, val_loss

best_trial = tuner.oracle.get_best_trials()[0].trial_id
acc, val_acc, loss, val_loss = extract_history(best_trial)

Unfortunately, when doing that I get the error message KeyError: 'Key epoch_loss was not found in Reservoir'. It seems like with e.g. ea.Scalars('epoch_acc') I merely have the wrong key (as I said at the beginning, this is a regression and therefore not using accuracy). How can I find out which keys are correct? I tried inspecting ea.scalars.Keys() which results in an empty list, although it seems that ea.scalars is not the same as ea.Scalars.

Answer 1

I recently bumped into this issue with saving and plot trial history metrics, so I made this gist to extend kerastuner randomsearch, save metrics and plot them at the end of each trial. You can also plot the best models in a single plot, here's the code hope it helps

import os
import json
import matplotlib.pyplot as plt
from kerastuner.tuners import RandomSearch as _RandomSearch
from kerastuner.engine.trial import Trial


class RandomSearch(_RandomSearch):
    histories = {}

    def on_epoch_end(self, trial, model, epoch, logs=None):
        trial_id = trial.trial_id
        trial_history = self.histories.setdefault(trial_id, {})
        for metric, value in logs.items():
            trial_history.setdefault(metric, []).append(value)

    def on_trial_end(self, trial):
        super().on_trial_end(trial)

        self.plot_and_save_history(trial)

    def plot_and_save_history(self, trial: Trial):
        # this will save a plot of the trial training metrics by step in each train folder, it'll also save the metrics as a .json
        trial_id = trial.trial_id
        trial_history = self.histories[trial_id]

        # Create directory for trial if it doesn't exist
        trial_dir = self.get_trial_dir(trial_id)
        os.makedirs(trial_dir, exist_ok=True)

        plt.figure(figsize=(12, 8))  # Create a larger figure

        for metric_name, metric_values in trial_history.items():
            plt.plot(metric_values, label=metric_name)  # Add label for legend

        plt.title("Metrics History")
        plt.xlabel("Step")
        plt.ylabel("Value")
        plt.legend()  # Add legend
        plt.savefig(os.path.join(trial_dir, "metrics.png"))
        plt.close()

        # Save history to JSON file
        history_file = os.path.join(trial_dir, "history.json")
        with open(history_file, "w") as f:
            json.dump(trial_history, f, indent=4)

    def plot_best_trials(self, num_models, wrap_columns=4, smooth_factor=0.5, suffix=""):
        # this will retrieve the best trials, load the saved metrics and plot them in a single plot
        best_trials = self.oracle.get_best_trials(num_models)
        num_trials = len(best_trials)

        rows = (num_trials - 1) // wrap_columns + 1
        cols = min(num_trials, wrap_columns)

        fig, axes = plt.subplots(rows, cols, figsize=(20, 5))
        # fig.subplots_adjust(hspace=0.4)
        # fig.tight_layout()

        for i, trial in enumerate(best_trials):
            trial_id = trial.trial_id
            trial_dir = self.get_trial_dir(trial_id)
            history_file = os.path.join(trial_dir, "history.json")

            with open(history_file, "r") as f:
                trial_history = json.load(f)

            ax = axes[i // cols, i % cols] if rows > 1 else axes[i]
            for metric_name, metric_values in trial_history.items():
                ax.plot(metric_values, label=f"Trial {trial_id}: {metric_name}")

            ax.set_title(f"Trial {trial_id}")
            ax.set_xlabel("Step")
            ax.set_ylabel("Value")
            ax.legend()

        plt.savefig(os.path.join(self.oracle._project_dir, f"best_trials_metrics_{suffix}.png"))
        plt.close()

I see you are using the tensorboard callback but I found it's kinda overcomplicated for a simple use case like this