Bayesian Optimisation of an LSTM Model

1 | Import Libraries

### Maths and Data-Handling
import numpy as np
#from copy import deepcopy
import pandas as pd
from pysmilesutils.tokenize import *

### Timing
import timeit

### Visualisation
import matplotlib.pyplot as plt
from rdkit import Chem
from rdkit.Chem import Draw

### Scikit
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from skopt import gp_minimize
from skopt.plots import plot_convergence

### PyTorch
import torch
from torch import nn   

WEB_COLOR = "#085E57"     ### core colour from my personal webpage, defined for convenience

2 | Data Wrangling

2a | Import data and Tokenise

# load dataset as pandas.DataFrame object
logPdata = pd.read_csv('logP_dataset.csv', names=['SMILES', 'ClogP'])

print('log(P) DataFrame columns: ', logPdata.columns)
print('log(P) DataFrame shape: ', logPdata.shape)

log(P) DataFrame columns:  Index(['SMILES', 'ClogP'], dtype='object')
log(P) DataFrame shape:  (14610, 2)

# tokenize SMILES strings
smiles_list = logPdata['SMILES'].to_list()
tokenizer = SMILESTokenizer(smiles=smiles_list)


# add to dataframe
logPdata['Tokenize'] = tokenizer(smiles_list)

print("Tokenizer vocabulary dictionary:", tokenizer.vocabulary)
print('log(P) DataFrame columns: ', logPdata.columns)
print('log(P) DataFrame shape: ', logPdata.shape)

Tokenizer vocabulary dictionary: {' ': 0, '^': 1, '&': 2, '?': 3, 'C': 4, '[': 5, '@': 6, 'H': 7, ']': 8, '(': 9, ')': 10, 'l': 11, '=': 12, 'B': 13, 'r': 14, 'N': 15, 'O': 16, '1': 17, '3': 18, '2': 19, 'P': 20, 'F': 21, 'I': 22, '+': 23, 'S': 24, '-': 25, '/': 26, '\\': 27, '.': 28, '8': 29, '#': 30, '4': 31, '9': 32, '5': 33, '7': 34, '0': 35, '6': 36}
log(P) DataFrame columns:  Index(['SMILES', 'ClogP', 'Tokenize'], dtype='object')
log(P) DataFrame shape:  (14610, 3)

# check length of each tokenized SMILES string
# store in 'tokens_len' list
tokens_len = list()

for tokens in logPdata['Tokenize']:
    entry_length = tokens.size()[0]
    tokens_len.append(entry_length)


# add to dataframe
logPdata['Tokenized Length'] = tokens_len

print('log(P) DataFrame columns: ', logPdata.columns)
print('log(P) DataFrame shape: ', logPdata.shape)
print('Maximum Token Length: ', max(tokens_len))

log(P) DataFrame columns:  Index(['SMILES', 'ClogP', 'Tokenize', 'Tokenized Length'], dtype='object')
log(P) DataFrame shape:  (14610, 4)
Maximum Token Length:  140

2b | Filter SMILES length

# display histogram of the frequency of tokenized length
fig, ax = plt.subplots(1, 2, figsize=(6,3))

### chart scaling
ax[1].set_yscale('log')

### gridlines
ax[0].spines[['right','top']].set_visible(False)
ax[0].grid(which='major', axis='y', zorder=1)
ax[1].spines[['right','top']].set_visible(False)
ax[1].grid(which='major', axis='y', zorder=1)

### data
ax[0].hist(tokens_len, label='Tokenized SMILEs Length', color=WEB_COLOR, zorder=2)
ax[1].hist(tokens_len, label='Tokenized SMILEs Length', color=WEB_COLOR, zorder=2)

### chart labels
plt.title(
    f'Variability in Token Length', 
    fontweight='bold', fontsize=11
)
fig.supxlabel('Tokenized Length', fontsize=11)
ax[0].set_ylabel('Frequency (lin)', fontsize=11)
ax[1].set_ylabel('Frequency (log)', fontsize=11)


fig.tight_layout()
plt.show()

# filter dataframe by token length
# data is too sparse above this limit to provide sufficient quality for the model 
FILTER_TO_LENGTH = 20
logPdata_filtered = logPdata[logPdata['Tokenized Length'] < FILTER_TO_LENGTH]

print('log(P) DataFrame shape after filtering: ', logPdata_filtered.shape)

log(P) DataFrame shape after filtering:  (13288, 4)

2c | Train-test split

# split into training and testing datasets
split_data = train_test_split(
    logPdata_filtered, test_size=0.1, shuffle=True
    )

train, test = split_data[0], split_data[1]

print('Training data shape: ', train.shape)
print('Testing data shape: ', test.shape)

Training data shape:  (11959, 4)
Testing data shape:  (1329, 4)

2d | Convert to standardised PyTorch tensor objects

def format_descriptors(data:pd.DataFrame, max_token_len:int):
    '''
    Formats SMILES strings to convert them into a padded PyTorch tensor object of set length.
    '''
    # create blank numpy.array object of correct size
    X = np.zeros((len(data), max_token_len))

    # overwrite blank numpy.array with tokenised data
    # ensures length of all entries is standardised by padding with '0' values
    for i, token in zip(np.arange(0,len(data),1), data['Tokenize']):
        token_arr = np.asarray(token)
        X[i][:len(token_arr)] = token_arr

    # convert to torch.tensor object
    X = torch.tensor(X[:, :, None], dtype = torch.float32)

    print('Shape of Descriptor Data: ', X.shape)

    return X


def format_labels(data:pd.DataFrame):
    '''
    Converts Clog(P) data to a PyTorch tensor object.
    '''
    # convert Clog(P) data to a torch.tensor object
    Y = np.asarray(data['ClogP'])
    Y = torch.tensor(Y[:, None], dtype=torch.float32)

    print('Shape of True Value Data: ', Y.shape)

    return Y

# format training and testing data
trainX = format_descriptors(train, FILTER_TO_LENGTH)   ### training data descriptors
testX = format_descriptors(test, FILTER_TO_LENGTH)     ### testing data descriptors

trainY_true = format_labels(train)                     ### training data true Clog(P) values
testY_true = format_labels(test)                       ### testing data true Clog(P) values

Shape of Descriptor Data:  torch.Size([11959, 20, 1])
Shape of Descriptor Data:  torch.Size([1329, 20, 1])
Shape of True Value Data:  torch.Size([11959, 1])
Shape of True Value Data:  torch.Size([1329, 1])

3 | LSTM Model Architecture and Experiment Procedure

class LSTMModel(nn.Module):
    def __init__(self, input_dim, hidden_dim, layer_dim, output_dim, dropout_rate=0.0):
        '''
        Define LSTM model architecture
        '''
        # ???
        super(LSTMModel, self).__init__()

        # define internal variables
        self.hidden_dim = hidden_dim
        self.layer_dim = layer_dim

        # LSTM layer
        self.lstm = nn.LSTM(input_dim, hidden_dim, layer_dim, batch_first=True)

        # Dropout layer, mitigates overfitting by dropping some neurons during training 
                # to encourage the model to learn redundant representations
        self.dropout = nn.Dropout(dropout_rate)

        # Linear layer, reformats LSTM output to the desired dimensionality
        self.fc = nn.Linear(hidden_dim, output_dim)


    def forward(self, x, h0=None, c0=None):
        '''
        Run LSTM model
        '''
        # ???
        if h0 is None or c0 is None:
            h0 = torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).to(x.device)
            c0 = torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).to(x.device)

        # ???
        out, (hn, cn) = self.lstm(x, (h0, c0))
        out = self.fc(out[:, -1, :])

        return out, hn, cn

def run_experiment(
        #train, trainX, trainY_true, 
        #test, testX, testY_true, 
        hidden_dim, learning_rate, num_epochs, layer_dim=1, dropout_rate=0.0
    ):
    '''
    Runs one complete training and evaluation of an LSTM model.
    
    (1) Model, optimiser, and loss criterion are defined
    (2) Model is trained
    (3) Model is evaluated with training data and predicted values stored
    (4) Model is evaluated with test data and predicted values stored
    (5) Post-processing to output results in a usable format

    '''
    ##############################################
    #----------------DEFINE MODEL----------------#
    ##############################################

    # instantiate model, optimizer, and loss criterion
    model = LSTMModel(
        input_dim=1, 
        hidden_dim=hidden_dim, 
        layer_dim=layer_dim, 
        output_dim=1,
        dropout_rate=dropout_rate
    )
    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
    criterion = nn.MSELoss()
    

    ##############################################
    #----------------TRAIN MODEL-----------------#
    ##############################################

    # set initial hidden state, h0 and cell state, c0
    h0, c0 = None, None

    # blank list to store loss values 
    loss_by_epoch = list()

    # iterate training for set number of epochs
    for epoch in range(num_epochs):

        model.train()                   # switch to model training mode
        optimizer.zero_grad()           # reset gradients to zero for next training iteration

        outputs, h0, c0 = model(trainX, h0, c0)     # train model and update h0, c0

        loss = criterion(outputs, trainY_true)      # calculate loss
        
        loss.backward()                             # backpropagation
                                                        # compute gradients of the loss

        optimizer.step()                            # optimisation
                                                        # adjust parameters based on gradients 
                                                        # with aim of minimising the loss
        
        h0, c0 = h0.detach(), c0.detach()           # detach h0 and c0 from gradients
        
        # add loss value to storage list
        loss_by_epoch.append(loss.item())

        # print every 10th loss value during model training
        if (epoch + 1) % 10 == 0:
            print(f'Epoch [{epoch+1}/{num_epochs}], Loss: {loss.item():.4f}')


    ##############################################
    #---------EVALUTE WITH TRAINING DATA---------#
    ##############################################

    # switch to model evaluation mode
    model.eval()

    # evaluate model and detach tensor
    trainY_pred, _, _ = model(trainX, h0, c0)
    trainY_pred_arr = trainY_pred.detach().numpy()

    # calculate errors as predicted values minus true values and detach tensor
    train_error = trainY_pred-trainY_true
    train_error = train_error.detach().numpy()


    ##############################################
    #-----------EVALUTE WITH TEST DATA-----------#
    ##############################################

    # switch to model evaluation mode
    model.eval()

    # evaluate model and detach tensor
    testY_pred, _, _ = model(testX, None, None)
    testY_pred_arr = testY_pred.detach().numpy()

    # calculate individual errors as predicted values minus true values and detach tensor
    test_error = testY_pred-testY_true
    test_error = test_error.detach().numpy()


    ##############################################
    #--------------POST-PROCESSING---------------#
    ##############################################

    # add predicted values to log(P) DataFrame for training and test data
    # add individal error values to same DataFrame
    train['ClogP (Predicted)'] = trainY_pred_arr
    train['Error'] = train_error

    test['ClogP (Predicted)'] = testY_pred_arr
    test['Error'] = test_error

    # calculate root mean square error for training and test datasets
    rmse_train = np.sqrt(
        mean_squared_error(train['ClogP'], train['ClogP (Predicted)'])
    )

    rmse_test = np.sqrt(
        mean_squared_error(test['ClogP'], test['ClogP (Predicted)'])
    )

    # print RMSE values
    print(f'Training RMSE: {rmse_train:.5f}')
    print(f'Testing RMSE: {rmse_test:.5f}')


    return model, rmse_train, rmse_test, loss_by_epoch

4 | Run Single User-Defined Experiment

# define variables
HIDDEN_DIM = 100
LEARNING_RATE = 0.01
NUM_EPOCHS = 30
NUM_LAYERS = 1
DROPOUT_RATE = 0.0

# run experiment
_, rmse_train, rmse_test, _ = run_experiment(
    #train, trainX, trainY_true, test, testX, testY_true,
    HIDDEN_DIM, LEARNING_RATE, NUM_EPOCHS, NUM_LAYERS, DROPOUT_RATE
)

Epoch [10/30], Loss: 1.8420
Epoch [20/30], Loss: 1.5884
Epoch [30/30], Loss: 1.5820
Training RMSE: 1.25667
Testing RMSE: 1.26029

##############################################
#---------------VISUALISATION----------------#
##############################################

# plot true vs. predicted log(P) values for training and testing data 
fig, ax = plt.subplots(1, 2, figsize=(8,4))
ax[0].grid(True, color='lightgrey', zorder=1)
ax[1].grid(True, color='lightgrey', zorder=1)

### data
ax[0].scatter(train['ClogP'], train['ClogP (Predicted)'], c='#1E4DAA', s=5, alpha=0.15, label='Training Data', zorder=2)
ax[1].scatter(test['ClogP'], test['ClogP (Predicted)'], c=WEB_COLOR, s=5, alpha=0.15, label='Test Data', zorder=2)

### ideal fit
ax[0].plot(np.linspace(-3.75,6.25,50),np.linspace(-3.75,6.25,50), c='grey', linestyle=":", zorder=3)
ax[1].plot(np.linspace(-3.75,6.25,50),np.linspace(-3.75,6.25,50), c='grey', linestyle=":", zorder=3)

### chart size
ax[0].set_xlim(-3.75, 6.25)
ax[0].set_ylim(-3.75, 6.25)
ax[1].set_xlim(-3.75, 6.25)
ax[1].set_ylim(-3.75, 6.25)

### legends
ax[0].legend(fontsize=10)
ax[1].legend(fontsize=10)

### chart labels
fig.supxlabel('True Clog(P)')
fig.supylabel('Predicted Clog(P)')
fig.suptitle(
    f'HIDDEN_DIM: {HIDDEN_DIM} | LEARNING_RATE: {LEARNING_RATE} | NUM_LAYERS: {NUM_LAYERS} | NUM_EPOCHS: {NUM_EPOCHS} | DROPOUT_RATE: {DROPOUT_RATE}', 
    fontsize=9, fontweight='bold', 
)

### annotate with RMSEs
ax[0].annotate(f'RMSE: {rmse_train:.3f}', xy=(3,-3.5), fontsize=10)
ax[1].annotate(f'RMSE: {rmse_test:.3f}', xy=(3,-3.5), fontsize=10)

fig.tight_layout()
plt.show()

5 | Testing Comparison Score for Predicted and True Values

# mock up values equivalent to the RMSE of the training and test datasets
xs = np.linspace(0,1.5,20)
ys = np.linspace(0,1.5,20)

# blank lists to store data for plotting
x_list, y_list = list(), list()
size_list = list()
variance_list = list()
score_list = list()

# user-defined weighting factor
BETA = 1.1

# iterate through each combination of x and y
for x in xs:
    for y in ys:
        size = (x + y)/2            # calculate average value of x and y
        variance = abs(x-y)         # calculate absolute difference between x and y

        # add to storage lists
        x_list.append(x)
        y_list.append(y)
        size_list.append(size)
        variance_list.append(variance)

        #calculate score and add to storage list
        score_list.append(size+(BETA*variance))

# plot toy data for combination score
fig, ax = plt.subplots(1, 3, figsize=(8,2.5))

# plot one variable's RMSE against combined score to show all possible scores
ax[0].grid(visible=True, zorder=1)
ax[0].scatter(x_list, score_list, s=10, c=WEB_COLOR, zorder=2)
ax[0].set_xlabel('RMSE of X or Y', fontsize=10)

# plot average size of RMSE against combined score to show all possible scores
ax[1].grid(visible=True, zorder=1)
ax[1].scatter(size_list, score_list, s=10, c=WEB_COLOR, zorder=2)
ax[1].set_xlabel('Size', fontsize=10)

# plot variation in RMSE against combined score to show all possible scores
ax[2].grid(visible=True, zorder=1)
ax[2].scatter(variance_list, score_list, s=10, c=WEB_COLOR, zorder=2)
ax[2].set_xlabel('Variance', fontsize=10)

fig.supylabel(f'Score (β={BETA})', fontsize=10)

# NOTE: that all combined score axes are set to a maximum of 1.5
# this is because 1.5 has been chosen as a maximum RMSE value for the variables, X Y
# plotting values above this reveals artefacts in the spread of data shown here
ax[0].set_ylim(-0.1,1.6)
ax[1].set_ylim(-0.1,1.6)
ax[2].set_ylim(-0.1,1.6)

fig.tight_layout()
plt.show()

6 | Bayesian Optimisation of Hyperparameters

6a | Define Scoring Metric and Bayesian Optimiser Function

def score_experiment(rmse_train, rmse_test, beta):
    '''
    Calculate the comparison score between the RMSE of the training and test data
    
    Score has two components: 
       - Average size of the two RMSE values
       - Variance, defined as the absolute difference between the two scores

    The score is calculated as the sum of the two components, weighted by the beta factor
    The typical value for beta in this work is 1.1
    '''
    av_size = (rmse_train + rmse_test)/2
    variance = abs(rmse_train - rmse_test)

    score = av_size+(beta*variance)

    return score



def f(x):
    '''
    "Function" for the Bayesian optimisation
    '''
    print('-------------EXPERIMENT START-------------')

    # process datatypes for search space coordinates
    hidden_dim = round(x[0])
    learning_rate = (x[1])
    num_epochs = round(x[2])
    num_layers_int = int(x[3])
    dropout_rate = (x[4])

    print('CONDITIONS:')
    print(f'HIDDEN_DIM = {hidden_dim}, LEARNING_RATE = {learning_rate},\nNUM_EPOCHS = {num_epochs}, NUM_LAYERS = {num_layers_int}, DROPOUT_RATE = {dropout_rate}')

    # run experiment
    start = timeit.timeit()                                     # mark start time
    _, rmse_train, rmse_test, _ = run_experiment(
        #train, trainX, trainY_true, test, testX, testY_true,
        hidden_dim, learning_rate, num_epochs, num_layers_int, dropout_rate
    )
    end = timeit.timeit()                                       # mark end time

    print(f'DURATION: {end-start}')

    # score experiment
    score = score_experiment(rmse_train, rmse_test, beta=1.1)
    
    print(f'SCORE: {score}')

    print('--------------EXPERIMENT END--------------\n\n\n')

    return score

6b | Run Optimisation

# define limits on hyperparameter values
search_space = [
    (100, 300),         # hidden dimension size
    (1e-05, 1e-02),     # learning rate for optimizer
    (100, 1000),        # number of epochs
    ('1', '2'),         # number of hidden layers in LSTM
    (0.0, 0.5),         # dropout rate
]

# run Bayesian optimisation
bayesian_result = gp_minimize(
    f,                      # the function to minimize
    search_space,           # the bounds on each dimension of x
    acq_func="EI",          # the acquisition function
    n_initial_points=5,     # number of evaluations of f before model estimation
    n_calls=25,             # the number of evaluations of f
    #noise=0.1**2,          # the noise level (optional)
    random_state=1234       # the random seed
)

# print optimisation results
print(bayesian_result)

-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 200, LEARNING_RATE = 0.008180206026754532,
NUM_EPOCHS = 651, NUM_LAYERS = 2, DROPOUT_RATE = 0.4303348856888437
Epoch [10/651], Loss: 1.8232
Epoch [20/651], Loss: 1.5814
Epoch [30/651], Loss: 1.5631
Epoch [40/651], Loss: 1.4412
Epoch [50/651], Loss: 1.6473
Epoch [60/651], Loss: 1.5784
Epoch [70/651], Loss: 1.5640
Epoch [80/651], Loss: 1.5479
Epoch [90/651], Loss: 1.5226
Epoch [100/651], Loss: 1.8019
Epoch [110/651], Loss: 1.5885
Epoch [120/651], Loss: 1.5543
Epoch [130/651], Loss: 1.5325
Epoch [140/651], Loss: 1.6128
Epoch [150/651], Loss: 1.5708
Epoch [160/651], Loss: 1.4883
Epoch [170/651], Loss: 1.3747
Epoch [180/651], Loss: 1.3282
Epoch [190/651], Loss: 1.3193
Epoch [200/651], Loss: 1.2947
Epoch [210/651], Loss: 1.2256
Epoch [220/651], Loss: 1.2752
Epoch [230/651], Loss: 1.2160
Epoch [240/651], Loss: 1.1645
Epoch [250/651], Loss: 1.0997
Epoch [260/651], Loss: 1.0365
Epoch [270/651], Loss: 1.1238
Epoch [280/651], Loss: 1.0037
Epoch [290/651], Loss: 1.0036
Epoch [300/651], Loss: 1.1384
Epoch [310/651], Loss: 0.9288
Epoch [320/651], Loss: 0.7829
Epoch [330/651], Loss: 0.6827
Epoch [340/651], Loss: 0.5561
Epoch [350/651], Loss: 0.4865
Epoch [360/651], Loss: 0.4253
Epoch [370/651], Loss: 0.4126
Epoch [380/651], Loss: 0.4816
Epoch [390/651], Loss: 0.4648
Epoch [400/651], Loss: 0.4080
Epoch [410/651], Loss: 0.3683
Epoch [420/651], Loss: 0.3599
Epoch [430/651], Loss: 0.3315
Epoch [440/651], Loss: 0.3672
Epoch [450/651], Loss: 0.3200
Epoch [460/651], Loss: 0.3514
Epoch [470/651], Loss: 0.3199
Epoch [480/651], Loss: 0.3943
Epoch [490/651], Loss: 0.3284
Epoch [500/651], Loss: 0.3262
Epoch [510/651], Loss: 0.3000
Epoch [520/651], Loss: 0.2892
Epoch [530/651], Loss: 0.2669
Epoch [540/651], Loss: 0.2622
Epoch [550/651], Loss: 0.2499
Epoch [560/651], Loss: 0.2361
Epoch [570/651], Loss: 0.2256
Epoch [580/651], Loss: 0.2134
Epoch [590/651], Loss: 0.2019
Epoch [600/651], Loss: 0.1893
Epoch [610/651], Loss: 0.1803
Epoch [620/651], Loss: 0.1662
Epoch [630/651], Loss: 0.1627
Epoch [640/651], Loss: 0.1559
Epoch [650/651], Loss: 0.1478
Training RMSE: 0.38306
Testing RMSE: 0.85868
DURATION: 0.00038390000008803327
SCORE: 1.1440572780572595
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 130, LEARNING_RATE = 0.001993202402671449,
NUM_EPOCHS = 834, NUM_LAYERS = 1, DROPOUT_RATE = 0.05806891380927438
Epoch [10/834], Loss: 1.5960
Epoch [20/834], Loss: 1.5663
Epoch [30/834], Loss: 1.5671
Epoch [40/834], Loss: 1.5633
Epoch [50/834], Loss: 1.5599
Epoch [60/834], Loss: 1.5566
Epoch [70/834], Loss: 1.5526
Epoch [80/834], Loss: 1.5486
Epoch [90/834], Loss: 1.5425
Epoch [100/834], Loss: 1.4937
Epoch [110/834], Loss: 1.4238
Epoch [120/834], Loss: 1.4334
Epoch [130/834], Loss: 1.3416
Epoch [140/834], Loss: 1.4073
Epoch [150/834], Loss: 1.3026
Epoch [160/834], Loss: 1.3425
Epoch [170/834], Loss: 1.5628
Epoch [180/834], Loss: 1.3266
Epoch [190/834], Loss: 1.3993
Epoch [200/834], Loss: 1.3025
Epoch [210/834], Loss: 1.3556
Epoch [220/834], Loss: 1.2577
Epoch [230/834], Loss: 1.1860
Epoch [240/834], Loss: 1.1734
Epoch [250/834], Loss: 1.2068
Epoch [260/834], Loss: 1.1557
Epoch [270/834], Loss: 1.1366
Epoch [280/834], Loss: 1.0783
Epoch [290/834], Loss: 1.0254
Epoch [300/834], Loss: 0.9856
Epoch [310/834], Loss: 0.9478
Epoch [320/834], Loss: 0.9050
Epoch [330/834], Loss: 0.8659
Epoch [340/834], Loss: 0.8242
Epoch [350/834], Loss: 0.7727
Epoch [360/834], Loss: 0.7401
Epoch [370/834], Loss: 0.8141
Epoch [380/834], Loss: 0.7748
Epoch [390/834], Loss: 0.9981
Epoch [400/834], Loss: 0.8458
Epoch [410/834], Loss: 0.8048
Epoch [420/834], Loss: 0.7890
Epoch [430/834], Loss: 0.6800
Epoch [440/834], Loss: 0.6897
Epoch [450/834], Loss: 0.6214
Epoch [460/834], Loss: 0.5982
Epoch [470/834], Loss: 0.5807
Epoch [480/834], Loss: 0.5655
Epoch [490/834], Loss: 0.5425
Epoch [500/834], Loss: 0.5279
Epoch [510/834], Loss: 0.5087
Epoch [520/834], Loss: 0.4997
Epoch [530/834], Loss: 0.4786
Epoch [540/834], Loss: 0.4599
Epoch [550/834], Loss: 0.4522
Epoch [560/834], Loss: 0.4432
Epoch [570/834], Loss: 0.4391
Epoch [580/834], Loss: 0.4435
Epoch [590/834], Loss: 0.4176
Epoch [600/834], Loss: 0.4331
Epoch [610/834], Loss: 0.4454
Epoch [620/834], Loss: 0.4556
Epoch [630/834], Loss: 0.4559
Epoch [640/834], Loss: 0.4288
Epoch [650/834], Loss: 0.4206
Epoch [660/834], Loss: 0.4677
Epoch [670/834], Loss: 0.4105
Epoch [680/834], Loss: 0.4445
Epoch [690/834], Loss: 0.3940
Epoch [700/834], Loss: 0.4150
Epoch [710/834], Loss: 0.4140
Epoch [720/834], Loss: 0.3674
Epoch [730/834], Loss: 0.4046
Epoch [740/834], Loss: 0.3605
Epoch [750/834], Loss: 0.3733
Epoch [760/834], Loss: 0.3652
Epoch [770/834], Loss: 0.3509
Epoch [780/834], Loss: 0.3687
Epoch [790/834], Loss: 0.3339
Epoch [800/834], Loss: 0.3585
Epoch [810/834], Loss: 0.3262
Epoch [820/834], Loss: 0.3549
Epoch [830/834], Loss: 0.3299
Training RMSE: 0.56683
Testing RMSE: 0.72378
DURATION: -1.1100000847363845e-05
SCORE: 0.8179601121795474
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 103, LEARNING_RATE = 0.004873466099260709,
NUM_EPOCHS = 398, NUM_LAYERS = 2, DROPOUT_RATE = 0.04912596777527102
Epoch [10/398], Loss: 1.7012
Epoch [20/398], Loss: 1.5911
Epoch [30/398], Loss: 1.5821
Epoch [40/398], Loss: 1.5721
Epoch [50/398], Loss: 1.5526
Epoch [60/398], Loss: 1.5191
Epoch [70/398], Loss: 1.4270
Epoch [80/398], Loss: 1.6130
Epoch [90/398], Loss: 1.5509
Epoch [100/398], Loss: 1.5265
Epoch [110/398], Loss: 1.4914
Epoch [120/398], Loss: 1.3974
Epoch [130/398], Loss: 1.3952
Epoch [140/398], Loss: 1.3635
Epoch [150/398], Loss: 1.3521
Epoch [160/398], Loss: 1.3474
Epoch [170/398], Loss: 1.3221
Epoch [180/398], Loss: 1.3398
Epoch [190/398], Loss: 1.4379
Epoch [200/398], Loss: 1.3523
Epoch [210/398], Loss: 1.3664
Epoch [220/398], Loss: 1.3033
Epoch [230/398], Loss: 1.4020
Epoch [240/398], Loss: 1.2780
Epoch [250/398], Loss: 1.2632
Epoch [260/398], Loss: 1.2199
Epoch [270/398], Loss: 1.1921
Epoch [280/398], Loss: 1.0956
Epoch [290/398], Loss: 1.2554
Epoch [300/398], Loss: 1.3069
Epoch [310/398], Loss: 1.2520
Epoch [320/398], Loss: 1.1927
Epoch [330/398], Loss: 1.1598
Epoch [340/398], Loss: 1.1125
Epoch [350/398], Loss: 1.0761
Epoch [360/398], Loss: 1.0455
Epoch [370/398], Loss: 1.0339
Epoch [380/398], Loss: 1.0054
Epoch [390/398], Loss: 1.1297
Training RMSE: 0.99563
Testing RMSE: 1.05549
DURATION: 0.0003195999997842591
SCORE: 1.091398384751752
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 111, LEARNING_RATE = 0.004432199863102637,
NUM_EPOCHS = 120, NUM_LAYERS = 1, DROPOUT_RATE = 0.12319722162517152
Epoch [10/120], Loss: 1.7166
Epoch [20/120], Loss: 1.6070
Epoch [30/120], Loss: 1.5822
Epoch [40/120], Loss: 1.5708
Epoch [50/120], Loss: 1.5646
Epoch [60/120], Loss: 1.5594
Epoch [70/120], Loss: 1.5540
Epoch [80/120], Loss: 1.5473
Epoch [90/120], Loss: 1.4983
Epoch [100/120], Loss: 1.6070
Epoch [110/120], Loss: 1.5796
Epoch [120/120], Loss: 1.5632
Training RMSE: 1.25074
Testing RMSE: 1.26392
DURATION: -0.00025159999859170057
SCORE: 1.2718290665720882
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 248, LEARNING_RATE = 0.008893369055711172,
NUM_EPOCHS = 988, NUM_LAYERS = 1, DROPOUT_RATE = 0.19689117614693252
Epoch [10/988], Loss: 2.0755
Epoch [20/988], Loss: 1.5937
Epoch [30/988], Loss: 1.5860
Epoch [40/988], Loss: 1.5849
Epoch [50/988], Loss: 1.5846
Epoch [60/988], Loss: 1.5844
Epoch [70/988], Loss: 1.5842
Epoch [80/988], Loss: 1.5839
Epoch [90/988], Loss: 1.5829
Epoch [100/988], Loss: 1.5783
Epoch [110/988], Loss: 1.5586
Epoch [120/988], Loss: 1.5441
Epoch [130/988], Loss: 1.5409
Epoch [140/988], Loss: 1.5335
Epoch [150/988], Loss: 1.5656
Epoch [160/988], Loss: 1.5483
Epoch [170/988], Loss: 1.4204
Epoch [180/988], Loss: 1.3731
Epoch [190/988], Loss: 1.3104
Epoch [200/988], Loss: 1.3768
Epoch [210/988], Loss: 1.3740
Epoch [220/988], Loss: 1.3534
Epoch [230/988], Loss: 1.3355
Epoch [240/988], Loss: 1.3475
Epoch [250/988], Loss: 1.3661
Epoch [260/988], Loss: 1.3442
Epoch [270/988], Loss: 1.3150
Epoch [280/988], Loss: 1.2572
Epoch [290/988], Loss: 1.2054
Epoch [300/988], Loss: 1.0772
Epoch [310/988], Loss: 1.2802
Epoch [320/988], Loss: 1.2541
Epoch [330/988], Loss: 1.1445
Epoch [340/988], Loss: 1.4449
Epoch [350/988], Loss: 1.2968
Epoch [360/988], Loss: 1.2684
Epoch [370/988], Loss: 1.1787
Epoch [380/988], Loss: 1.2856
Epoch [390/988], Loss: 1.1554
Epoch [400/988], Loss: 1.0739
Epoch [410/988], Loss: 1.0164
Epoch [420/988], Loss: 0.9848
Epoch [430/988], Loss: 0.9415
Epoch [440/988], Loss: 0.8982
Epoch [450/988], Loss: 0.8542
Epoch [460/988], Loss: 0.8025
Epoch [470/988], Loss: 0.7218
Epoch [480/988], Loss: 0.6919
Epoch [490/988], Loss: 0.5945
Epoch [500/988], Loss: 0.5229
Epoch [510/988], Loss: 0.4791
Epoch [520/988], Loss: 0.4527
Epoch [530/988], Loss: 0.4661
Epoch [540/988], Loss: 0.5233
Epoch [550/988], Loss: 0.5014
Epoch [560/988], Loss: 0.4470
Epoch [570/988], Loss: 0.4317
Epoch [580/988], Loss: 0.4211
Epoch [590/988], Loss: 0.4077
Epoch [600/988], Loss: 0.3981
Epoch [610/988], Loss: 0.3786
Epoch [620/988], Loss: 0.3719
Epoch [630/988], Loss: 0.3636
Epoch [640/988], Loss: 0.3633
Epoch [650/988], Loss: 0.3458
Epoch [660/988], Loss: 0.3374
Epoch [670/988], Loss: 0.3318
Epoch [680/988], Loss: 0.3272
Epoch [690/988], Loss: 0.3172
Epoch [700/988], Loss: 0.3036
Epoch [710/988], Loss: 0.2924
Epoch [720/988], Loss: 0.2827
Epoch [730/988], Loss: 0.2797
Epoch [740/988], Loss: 0.2744
Epoch [750/988], Loss: 0.2728
Epoch [760/988], Loss: 0.3148
Epoch [770/988], Loss: 0.3075
Epoch [780/988], Loss: 0.3017
Epoch [790/988], Loss: 0.2723
Epoch [800/988], Loss: 0.2801
Epoch [810/988], Loss: 0.2586
Epoch [820/988], Loss: 0.2624
Epoch [830/988], Loss: 0.2659
Epoch [840/988], Loss: 0.2439
Epoch [850/988], Loss: 0.2499
Epoch [860/988], Loss: 0.2532
Epoch [870/988], Loss: 0.2319
Epoch [880/988], Loss: 0.2442
Epoch [890/988], Loss: 0.2235
Epoch [900/988], Loss: 0.2328
Epoch [910/988], Loss: 0.2148
Epoch [920/988], Loss: 0.2174
Epoch [930/988], Loss: 0.2212
Epoch [940/988], Loss: 0.2000
Epoch [950/988], Loss: 0.2135
Epoch [960/988], Loss: 0.2016
Epoch [970/988], Loss: 0.1972
Epoch [980/988], Loss: 0.1996
Training RMSE: 0.44191
Testing RMSE: 1.56982
DURATION: 0.00023049999981594738
SCORE: 2.2465582504938224
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 159, LEARNING_RATE = 0.009587633755102439,
NUM_EPOCHS = 1000, NUM_LAYERS = 2, DROPOUT_RATE = 0.32791925111762543
Epoch [10/1000], Loss: 1.8882
Epoch [20/1000], Loss: 1.5823
Epoch [30/1000], Loss: 1.5580
Epoch [40/1000], Loss: 1.5498
Epoch [50/1000], Loss: 1.5379
Epoch [60/1000], Loss: 1.4143
Epoch [70/1000], Loss: 1.5149
Epoch [80/1000], Loss: 1.5617
Epoch [90/1000], Loss: 1.4903
Epoch [100/1000], Loss: 1.4120
Epoch [110/1000], Loss: 1.5318
Epoch [120/1000], Loss: 1.5225
Epoch [130/1000], Loss: 1.5185
Epoch [140/1000], Loss: 1.5011
Epoch [150/1000], Loss: 1.4899
Epoch [160/1000], Loss: 1.4486
Epoch [170/1000], Loss: 1.5212
Epoch [180/1000], Loss: 1.5050
Epoch [190/1000], Loss: 1.4898
Epoch [200/1000], Loss: 1.3800
Epoch [210/1000], Loss: 1.5176
Epoch [220/1000], Loss: 1.5097
Epoch [230/1000], Loss: 1.5016
Epoch [240/1000], Loss: 1.4994
Epoch [250/1000], Loss: 1.4979
Epoch [260/1000], Loss: 1.4940
Epoch [270/1000], Loss: 1.4887
Epoch [280/1000], Loss: 1.4821
Epoch [290/1000], Loss: 1.4839
Epoch [300/1000], Loss: 1.5529
Epoch [310/1000], Loss: 1.5062
Epoch [320/1000], Loss: 1.4936
Epoch [330/1000], Loss: 1.4758
Epoch [340/1000], Loss: 1.4337
Epoch [350/1000], Loss: 1.3564
Epoch [360/1000], Loss: 1.3916
Epoch [370/1000], Loss: 1.3991
Epoch [380/1000], Loss: 1.4309
Epoch [390/1000], Loss: 1.4069
Epoch [400/1000], Loss: 1.3909
Epoch [410/1000], Loss: 1.3742
Epoch [420/1000], Loss: 1.3646
Epoch [430/1000], Loss: 1.3548
Epoch [440/1000], Loss: 1.3729
Epoch [450/1000], Loss: 1.3096
Epoch [460/1000], Loss: 1.2928
Epoch [470/1000], Loss: 1.2965
Epoch [480/1000], Loss: 1.2160
Epoch [490/1000], Loss: 1.2242
Epoch [500/1000], Loss: 1.2371
Epoch [510/1000], Loss: 1.1726
Epoch [520/1000], Loss: 1.1619
Epoch [530/1000], Loss: 1.1520
Epoch [540/1000], Loss: 1.1385
Epoch [550/1000], Loss: 1.1052
Epoch [560/1000], Loss: 1.0641
Epoch [570/1000], Loss: 1.0281
Epoch [580/1000], Loss: 0.9803
Epoch [590/1000], Loss: 0.9021
Epoch [600/1000], Loss: 0.9944
Epoch [610/1000], Loss: 0.9613
Epoch [620/1000], Loss: 0.9266
Epoch [630/1000], Loss: 0.8750
Epoch [640/1000], Loss: 0.8272
Epoch [650/1000], Loss: 0.7597
Epoch [660/1000], Loss: 0.6953
Epoch [670/1000], Loss: 0.6757
Epoch [680/1000], Loss: 0.7009
Epoch [690/1000], Loss: 0.6159
Epoch [700/1000], Loss: 0.6299
Epoch [710/1000], Loss: 0.5849
Epoch [720/1000], Loss: 0.5371
Epoch [730/1000], Loss: 0.5035
Epoch [740/1000], Loss: 0.4698
Epoch [750/1000], Loss: 0.4394
Epoch [760/1000], Loss: 0.4232
Epoch [770/1000], Loss: 0.4148
Epoch [780/1000], Loss: 0.4734
Epoch [790/1000], Loss: 0.4186
Epoch [800/1000], Loss: 0.4271
Epoch [810/1000], Loss: 0.3840
Epoch [820/1000], Loss: 0.4350
Epoch [830/1000], Loss: 0.3933
Epoch [840/1000], Loss: 0.3617
Epoch [850/1000], Loss: 0.3714
Epoch [860/1000], Loss: 0.3524
Epoch [870/1000], Loss: 0.3352
Epoch [880/1000], Loss: 0.3303
Epoch [890/1000], Loss: 0.3445
Epoch [900/1000], Loss: 0.3340
Epoch [910/1000], Loss: 0.3174
Epoch [920/1000], Loss: 0.3080
Epoch [930/1000], Loss: 0.2986
Epoch [940/1000], Loss: 0.3077
Epoch [950/1000], Loss: 0.3211
Epoch [960/1000], Loss: 0.2821
Epoch [970/1000], Loss: 0.3369
Epoch [980/1000], Loss: 0.3010
Epoch [990/1000], Loss: 0.2708
Epoch [1000/1000], Loss: 0.2850
Training RMSE: 0.54928
Testing RMSE: 0.99313
DURATION: -0.00031010000020614825
SCORE: 1.2594396681221876
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 100, LEARNING_RATE = 1e-05,
NUM_EPOCHS = 100, NUM_LAYERS = 1, DROPOUT_RATE = 0.039916823572650924
Epoch [10/100], Loss: 2.6140
Epoch [20/100], Loss: 2.6088
Epoch [30/100], Loss: 2.6036
Epoch [40/100], Loss: 2.5984
Epoch [50/100], Loss: 2.5931
Epoch [60/100], Loss: 2.5879
Epoch [70/100], Loss: 2.5827
Epoch [80/100], Loss: 2.5774
Epoch [90/100], Loss: 2.5721
Epoch [100/100], Loss: 2.5667
Training RMSE: 1.60193
Testing RMSE: 1.59933
DURATION: -0.0002305999987584073
SCORE: 1.6034932858206323
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 141, LEARNING_RATE = 0.0019966483918312117,
NUM_EPOCHS = 759, NUM_LAYERS = 1, DROPOUT_RATE = 0.32814875387517656
Epoch [10/759], Loss: 1.6002
Epoch [20/759], Loss: 1.5675
Epoch [30/759], Loss: 1.5678
Epoch [40/759], Loss: 1.5645
Epoch [50/759], Loss: 1.5604
Epoch [60/759], Loss: 1.5575
Epoch [70/759], Loss: 1.5532
Epoch [80/759], Loss: 1.5333
Epoch [90/759], Loss: 1.4209
Epoch [100/759], Loss: 1.4960
Epoch [110/759], Loss: 1.5164
Epoch [120/759], Loss: 1.4378
Epoch [130/759], Loss: 1.4098
Epoch [140/759], Loss: 1.3908
Epoch [150/759], Loss: 1.5493
Epoch [160/759], Loss: 1.5025
Epoch [170/759], Loss: 1.4605
Epoch [180/759], Loss: 1.4368
Epoch [190/759], Loss: 1.4065
Epoch [200/759], Loss: 1.3907
Epoch [210/759], Loss: 1.3829
Epoch [220/759], Loss: 1.3665
Epoch [230/759], Loss: 1.3245
Epoch [240/759], Loss: 1.2605
Epoch [250/759], Loss: 1.4049
Epoch [260/759], Loss: 1.3482
Epoch [270/759], Loss: 1.2604
Epoch [280/759], Loss: 1.2332
Epoch [290/759], Loss: 1.1974
Epoch [300/759], Loss: 1.1718
Epoch [310/759], Loss: 1.1441
Epoch [320/759], Loss: 1.2367
Epoch [330/759], Loss: 1.2053
Epoch [340/759], Loss: 1.1719
Epoch [350/759], Loss: 1.1319
Epoch [360/759], Loss: 1.0627
Epoch [370/759], Loss: 1.0944
Epoch [380/759], Loss: 1.0157
Epoch [390/759], Loss: 1.0505
Epoch [400/759], Loss: 1.0472
Epoch [410/759], Loss: 1.0052
Epoch [420/759], Loss: 0.9162
Epoch [430/759], Loss: 0.9577
Epoch [440/759], Loss: 0.9944
Epoch [450/759], Loss: 0.9940
Epoch [460/759], Loss: 0.9692
Epoch [470/759], Loss: 0.9208
Epoch [480/759], Loss: 0.8819
Epoch [490/759], Loss: 0.8319
Epoch [500/759], Loss: 0.8570
Epoch [510/759], Loss: 0.8009
Epoch [520/759], Loss: 0.7757
Epoch [530/759], Loss: 0.7435
Epoch [540/759], Loss: 0.6952
Epoch [550/759], Loss: 0.6745
Epoch [560/759], Loss: 0.6750
Epoch [570/759], Loss: 0.6810
Epoch [580/759], Loss: 0.6270
Epoch [590/759], Loss: 0.7066
Epoch [600/759], Loss: 0.7384
Epoch [610/759], Loss: 0.7004
Epoch [620/759], Loss: 0.7119
Epoch [630/759], Loss: 0.5913
Epoch [640/759], Loss: 0.5922
Epoch [650/759], Loss: 0.5874
Epoch [660/759], Loss: 0.5354
Epoch [670/759], Loss: 0.5678
Epoch [680/759], Loss: 0.5035
Epoch [690/759], Loss: 0.5048
Epoch [700/759], Loss: 0.5247
Epoch [710/759], Loss: 0.4611
Epoch [720/759], Loss: 0.4881
Epoch [730/759], Loss: 0.4902
Epoch [740/759], Loss: 0.4480
Epoch [750/759], Loss: 0.4827
Training RMSE: 0.69197
Testing RMSE: 0.81110
DURATION: -0.0007856000011088327
SCORE: 0.8825821900995774
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 100, LEARNING_RATE = 0.0018555954659972237,
NUM_EPOCHS = 1000, NUM_LAYERS = 1, DROPOUT_RATE = 0.0
Epoch [10/1000], Loss: 1.6234
Epoch [20/1000], Loss: 1.6007
Epoch [30/1000], Loss: 1.5724
Epoch [40/1000], Loss: 1.5645
Epoch [50/1000], Loss: 1.5580
Epoch [60/1000], Loss: 1.5528
Epoch [70/1000], Loss: 1.5448
Epoch [80/1000], Loss: 1.5026
Epoch [90/1000], Loss: 1.3979
Epoch [100/1000], Loss: 1.4000
Epoch [110/1000], Loss: 1.4078
Epoch [120/1000], Loss: 1.4253
Epoch [130/1000], Loss: 1.3672
Epoch [140/1000], Loss: 1.4111
Epoch [150/1000], Loss: 1.4437
Epoch [160/1000], Loss: 1.2997
Epoch [170/1000], Loss: 1.3747
Epoch [180/1000], Loss: 1.3274
Epoch [190/1000], Loss: 1.2933
Epoch [200/1000], Loss: 1.2153
Epoch [210/1000], Loss: 1.2721
Epoch [220/1000], Loss: 1.2747
Epoch [230/1000], Loss: 1.1818
Epoch [240/1000], Loss: 1.3079
Epoch [250/1000], Loss: 1.3039
Epoch [260/1000], Loss: 1.2980
Epoch [270/1000], Loss: 1.1310
Epoch [280/1000], Loss: 1.1288
Epoch [290/1000], Loss: 1.1500
Epoch [300/1000], Loss: 1.1686
Epoch [310/1000], Loss: 1.1329
Epoch [320/1000], Loss: 1.0944
Epoch [330/1000], Loss: 1.0346
Epoch [340/1000], Loss: 1.0285
Epoch [350/1000], Loss: 0.9757
Epoch [360/1000], Loss: 1.0264
Epoch [370/1000], Loss: 0.9397
Epoch [380/1000], Loss: 0.9700
Epoch [390/1000], Loss: 0.9157
Epoch [400/1000], Loss: 0.9118
Epoch [410/1000], Loss: 0.8915
Epoch [420/1000], Loss: 0.8818
Epoch [430/1000], Loss: 0.8630
Epoch [440/1000], Loss: 0.8520
Epoch [450/1000], Loss: 0.8377
Epoch [460/1000], Loss: 0.8236
Epoch [470/1000], Loss: 0.8035
Epoch [480/1000], Loss: 0.7746
Epoch [490/1000], Loss: 0.7318
Epoch [500/1000], Loss: 0.7389
Epoch [510/1000], Loss: 0.8134
Epoch [520/1000], Loss: 0.9137
Epoch [530/1000], Loss: 0.7263
Epoch [540/1000], Loss: 0.7394
Epoch [550/1000], Loss: 0.8326
Epoch [560/1000], Loss: 0.7055
Epoch [570/1000], Loss: 0.6474
Epoch [580/1000], Loss: 0.6604
Epoch [590/1000], Loss: 0.6403
Epoch [600/1000], Loss: 0.5833
Epoch [610/1000], Loss: 0.5945
Epoch [620/1000], Loss: 0.5637
Epoch [630/1000], Loss: 0.5868
Epoch [640/1000], Loss: 0.5360
Epoch [650/1000], Loss: 0.5707
Epoch [660/1000], Loss: 0.5131
Epoch [670/1000], Loss: 0.5537
Epoch [680/1000], Loss: 0.5005
Epoch [690/1000], Loss: 0.5207
Epoch [700/1000], Loss: 0.4784
Epoch [710/1000], Loss: 0.4992
Epoch [720/1000], Loss: 0.4640
Epoch [730/1000], Loss: 0.4788
Epoch [740/1000], Loss: 0.4465
Epoch [750/1000], Loss: 0.4687
Epoch [760/1000], Loss: 0.4399
Epoch [770/1000], Loss: 0.4462
Epoch [780/1000], Loss: 0.4420
Epoch [790/1000], Loss: 0.4314
Epoch [800/1000], Loss: 0.4453
Epoch [810/1000], Loss: 0.4015
Epoch [820/1000], Loss: 0.4362
Epoch [830/1000], Loss: 0.4030
Epoch [840/1000], Loss: 0.4068
Epoch [850/1000], Loss: 0.4305
Epoch [860/1000], Loss: 0.3893
Epoch [870/1000], Loss: 0.3941
Epoch [880/1000], Loss: 0.4029
Epoch [890/1000], Loss: 0.3735
Epoch [900/1000], Loss: 0.3694
Epoch [910/1000], Loss: 0.3978
Epoch [920/1000], Loss: 0.3613
Epoch [930/1000], Loss: 0.3678
Epoch [940/1000], Loss: 0.3862
Epoch [950/1000], Loss: 0.3527
Epoch [960/1000], Loss: 0.3583
Epoch [970/1000], Loss: 0.3588
Epoch [980/1000], Loss: 0.3379
Epoch [990/1000], Loss: 0.3560
Epoch [1000/1000], Loss: 0.3262
Training RMSE: 0.58551
Testing RMSE: 0.69868
DURATION: -0.002369800000451505
SCORE: 0.7665882494709837
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 249, LEARNING_RATE = 0.0029530940666112793,
NUM_EPOCHS = 1000, NUM_LAYERS = 1, DROPOUT_RATE = 0.0
Epoch [10/1000], Loss: 1.8832
Epoch [20/1000], Loss: 1.5742
Epoch [30/1000], Loss: 1.5749
Epoch [40/1000], Loss: 1.5704
Epoch [50/1000], Loss: 1.5660
Epoch [60/1000], Loss: 1.5607
Epoch [70/1000], Loss: 1.5556
Epoch [80/1000], Loss: 1.5474
Epoch [90/1000], Loss: 1.4897
Epoch [100/1000], Loss: 1.5517
Epoch [110/1000], Loss: 1.4628
Epoch [120/1000], Loss: 1.4135
Epoch [130/1000], Loss: 1.4138
Epoch [140/1000], Loss: 1.3915
Epoch [150/1000], Loss: 1.3792
Epoch [160/1000], Loss: 1.3491
Epoch [170/1000], Loss: 1.3711
Epoch [180/1000], Loss: 1.5679
Epoch [190/1000], Loss: 1.5156
Epoch [200/1000], Loss: 1.3798
Epoch [210/1000], Loss: 1.4061
Epoch [220/1000], Loss: 1.3778
Epoch [230/1000], Loss: 1.4038
Epoch [240/1000], Loss: 1.2864
Epoch [250/1000], Loss: 1.3394
Epoch [260/1000], Loss: 1.2759
Epoch [270/1000], Loss: 1.2100
Epoch [280/1000], Loss: 1.2091
Epoch [290/1000], Loss: 1.2885
Epoch [300/1000], Loss: 1.2561
Epoch [310/1000], Loss: 1.2243
Epoch [320/1000], Loss: 1.2519
Epoch [330/1000], Loss: 1.0967
Epoch [340/1000], Loss: 0.9655
Epoch [350/1000], Loss: 0.8874
Epoch [360/1000], Loss: 0.8813
Epoch [370/1000], Loss: 0.8437
Epoch [380/1000], Loss: 0.8175
Epoch [390/1000], Loss: 0.7787
Epoch [400/1000], Loss: 0.6986
Epoch [410/1000], Loss: 0.9042
Epoch [420/1000], Loss: 0.9377
Epoch [430/1000], Loss: 0.8680
Epoch [440/1000], Loss: 0.7458
Epoch [450/1000], Loss: 0.6636
Epoch [460/1000], Loss: 0.5952
Epoch [470/1000], Loss: 0.5441
Epoch [480/1000], Loss: 0.4900
Epoch [490/1000], Loss: 0.4423
Epoch [500/1000], Loss: 0.4230
Epoch [510/1000], Loss: 0.3862
Epoch [520/1000], Loss: 0.3808
Epoch [530/1000], Loss: 0.3658
Epoch [540/1000], Loss: 0.3707
Epoch [550/1000], Loss: 0.4932
Epoch [560/1000], Loss: 0.6025
Epoch [570/1000], Loss: 0.4683
Epoch [580/1000], Loss: 0.4672
Epoch [590/1000], Loss: 0.4149
Epoch [600/1000], Loss: 0.3659
Epoch [610/1000], Loss: 0.3534
Epoch [620/1000], Loss: 0.3587
Epoch [630/1000], Loss: 0.3498
Epoch [640/1000], Loss: 0.3584
Epoch [650/1000], Loss: 0.3690
Epoch [660/1000], Loss: 0.3697
Epoch [670/1000], Loss: 0.3275
Epoch [680/1000], Loss: 0.3621
Epoch [690/1000], Loss: 0.3411
Epoch [700/1000], Loss: 0.4063
Epoch [710/1000], Loss: 0.3357
Epoch [720/1000], Loss: 0.3443
Epoch [730/1000], Loss: 0.3523
Epoch [740/1000], Loss: 0.2999
Epoch [750/1000], Loss: 0.3388
Epoch [760/1000], Loss: 0.3462
Epoch [770/1000], Loss: 0.2971
Epoch [780/1000], Loss: 0.3137
Epoch [790/1000], Loss: 0.3358
Epoch [800/1000], Loss: 0.3008
Epoch [810/1000], Loss: 0.2876
Epoch [820/1000], Loss: 0.3064
Epoch [830/1000], Loss: 0.2846
Epoch [840/1000], Loss: 0.2734
Epoch [850/1000], Loss: 0.2817
Epoch [860/1000], Loss: 0.2600
Epoch [870/1000], Loss: 0.2799
Epoch [880/1000], Loss: 0.2515
Epoch [890/1000], Loss: 0.2677
Epoch [900/1000], Loss: 0.2620
Epoch [910/1000], Loss: 0.2533
Epoch [920/1000], Loss: 0.2395
Epoch [930/1000], Loss: 0.2373
Epoch [940/1000], Loss: 0.2317
Epoch [950/1000], Loss: 0.2301
Epoch [960/1000], Loss: 0.2186
Epoch [970/1000], Loss: 0.2110
Epoch [980/1000], Loss: 0.2097
Epoch [990/1000], Loss: 0.2068
Epoch [1000/1000], Loss: 0.2009
Training RMSE: 0.45063
Testing RMSE: 0.70069
DURATION: -0.015626000000338536
SCORE: 0.8507269467960761
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 165, LEARNING_RATE = 0.0016249001630866402,
NUM_EPOCHS = 1000, NUM_LAYERS = 1, DROPOUT_RATE = 0.0
Epoch [10/1000], Loss: 1.6052
Epoch [20/1000], Loss: 1.5692
Epoch [30/1000], Loss: 1.5730
Epoch [40/1000], Loss: 1.5645
Epoch [50/1000], Loss: 1.5604
Epoch [60/1000], Loss: 1.5558
Epoch [70/1000], Loss: 1.5440
Epoch [80/1000], Loss: 1.4245
Epoch [90/1000], Loss: 1.4337
Epoch [100/1000], Loss: 1.4364
Epoch [110/1000], Loss: 1.8018
Epoch [120/1000], Loss: 1.5050
Epoch [130/1000], Loss: 1.4095
Epoch [140/1000], Loss: 1.5119
Epoch [150/1000], Loss: 1.4863
Epoch [160/1000], Loss: 1.4887
Epoch [170/1000], Loss: 1.3846
Epoch [180/1000], Loss: 1.3891
Epoch [190/1000], Loss: 1.3768
Epoch [200/1000], Loss: 1.4545
Epoch [210/1000], Loss: 1.3208
Epoch [220/1000], Loss: 1.3266
Epoch [230/1000], Loss: 1.3500
Epoch [240/1000], Loss: 1.3311
Epoch [250/1000], Loss: 1.3209
Epoch [260/1000], Loss: 1.2951
Epoch [270/1000], Loss: 1.2316
Epoch [280/1000], Loss: 1.1874
Epoch [290/1000], Loss: 1.3276
Epoch [300/1000], Loss: 1.1929
Epoch [310/1000], Loss: 1.2411
Epoch [320/1000], Loss: 1.2560
Epoch [330/1000], Loss: 1.2244
Epoch [340/1000], Loss: 1.2328
Epoch [350/1000], Loss: 1.2494
Epoch [360/1000], Loss: 1.1730
Epoch [370/1000], Loss: 1.2293
Epoch [380/1000], Loss: 1.2071
Epoch [390/1000], Loss: 1.0937
Epoch [400/1000], Loss: 1.1139
Epoch [410/1000], Loss: 1.0236
Epoch [420/1000], Loss: 1.0072
Epoch [430/1000], Loss: 0.9706
Epoch [440/1000], Loss: 0.9501
Epoch [450/1000], Loss: 0.9282
Epoch [460/1000], Loss: 0.9100
Epoch [470/1000], Loss: 0.8923
Epoch [480/1000], Loss: 0.8907
Epoch [490/1000], Loss: 0.8782
Epoch [500/1000], Loss: 0.8548
Epoch [510/1000], Loss: 0.8206
Epoch [520/1000], Loss: 0.8100
Epoch [530/1000], Loss: 0.8610
Epoch [540/1000], Loss: 1.0091
Epoch [550/1000], Loss: 0.8781
Epoch [560/1000], Loss: 0.8100
Epoch [570/1000], Loss: 0.8201
Epoch [580/1000], Loss: 0.7871
Epoch [590/1000], Loss: 0.6386
Epoch [600/1000], Loss: 0.6707
Epoch [610/1000], Loss: 0.7018
Epoch [620/1000], Loss: 0.6059
Epoch [630/1000], Loss: 0.6158
Epoch [640/1000], Loss: 0.6187
Epoch [650/1000], Loss: 0.5522
Epoch [660/1000], Loss: 0.6029
Epoch [670/1000], Loss: 0.5109
Epoch [680/1000], Loss: 0.5332
Epoch [690/1000], Loss: 0.4773
Epoch [700/1000], Loss: 0.4901
Epoch [710/1000], Loss: 0.4416
Epoch [720/1000], Loss: 0.4518
Epoch [730/1000], Loss: 0.4451
Epoch [740/1000], Loss: 0.4109
Epoch [750/1000], Loss: 0.4361
Epoch [760/1000], Loss: 0.4164
Epoch [770/1000], Loss: 0.3987
Epoch [780/1000], Loss: 0.4305
Epoch [790/1000], Loss: 0.3803
Epoch [800/1000], Loss: 0.4357
Epoch [810/1000], Loss: 0.3928
Epoch [820/1000], Loss: 0.4372
Epoch [830/1000], Loss: 0.3769
Epoch [840/1000], Loss: 0.4360
Epoch [850/1000], Loss: 0.3992
Epoch [860/1000], Loss: 0.3629
Epoch [870/1000], Loss: 0.4042
Epoch [880/1000], Loss: 0.3809
Epoch [890/1000], Loss: 0.3708
Epoch [900/1000], Loss: 0.3984
Epoch [910/1000], Loss: 0.3713
Epoch [920/1000], Loss: 0.3454
Epoch [930/1000], Loss: 0.3870
Epoch [940/1000], Loss: 0.3728
Epoch [950/1000], Loss: 0.3462
Epoch [960/1000], Loss: 0.3511
Epoch [970/1000], Loss: 0.3575
Epoch [980/1000], Loss: 0.3295
Epoch [990/1000], Loss: 0.3235
Epoch [1000/1000], Loss: 0.3412
Training RMSE: 0.56905
Testing RMSE: 0.64216
DURATION: 8.650000017951243e-05
SCORE: 0.6860194283100715
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 300, LEARNING_RATE = 0.00046561129149826563,
NUM_EPOCHS = 1000, NUM_LAYERS = 1, DROPOUT_RATE = 0.0
Epoch [10/1000], Loss: 1.8837
Epoch [20/1000], Loss: 1.6339
Epoch [30/1000], Loss: 1.5829
Epoch [40/1000], Loss: 1.5732
Epoch [50/1000], Loss: 1.5681
Epoch [60/1000], Loss: 1.5649
Epoch [70/1000], Loss: 1.5623
Epoch [80/1000], Loss: 1.5595
Epoch [90/1000], Loss: 1.5530
Epoch [100/1000], Loss: 1.4995
Epoch [110/1000], Loss: 1.4252
Epoch [120/1000], Loss: 1.4101
Epoch [130/1000], Loss: 1.5805
Epoch [140/1000], Loss: 1.5634
Epoch [150/1000], Loss: 1.5325
Epoch [160/1000], Loss: 1.4845
Epoch [170/1000], Loss: 1.4100
Epoch [180/1000], Loss: 1.4090
Epoch [190/1000], Loss: 1.3966
Epoch [200/1000], Loss: 1.4953
Epoch [210/1000], Loss: 1.4140
Epoch [220/1000], Loss: 1.5403
Epoch [230/1000], Loss: 1.3955
Epoch [240/1000], Loss: 1.4356
Epoch [250/1000], Loss: 1.3092
Epoch [260/1000], Loss: 1.4940
Epoch [270/1000], Loss: 1.3144
Epoch [280/1000], Loss: 1.4138
Epoch [290/1000], Loss: 1.3275
Epoch [300/1000], Loss: 1.2640
Epoch [310/1000], Loss: 1.4930
Epoch [320/1000], Loss: 1.3312
Epoch [330/1000], Loss: 1.3001
Epoch [340/1000], Loss: 1.3013
Epoch [350/1000], Loss: 1.2474
Epoch [360/1000], Loss: 1.2946
Epoch [370/1000], Loss: 1.4049
Epoch [380/1000], Loss: 1.2026
Epoch [390/1000], Loss: 1.2218
Epoch [400/1000], Loss: 1.3476
Epoch [410/1000], Loss: 1.1411
Epoch [420/1000], Loss: 1.2631
Epoch [430/1000], Loss: 1.2990
Epoch [440/1000], Loss: 1.1461
Epoch [450/1000], Loss: 1.2141
Epoch [460/1000], Loss: 1.1326
Epoch [470/1000], Loss: 1.1303
Epoch [480/1000], Loss: 1.1958
Epoch [490/1000], Loss: 1.1716
Epoch [500/1000], Loss: 1.0560
Epoch [510/1000], Loss: 1.0467
Epoch [520/1000], Loss: 1.1113
Epoch [530/1000], Loss: 1.0602
Epoch [540/1000], Loss: 1.0615
Epoch [550/1000], Loss: 1.0341
Epoch [560/1000], Loss: 1.0688
Epoch [570/1000], Loss: 1.0267
Epoch [580/1000], Loss: 1.0239
Epoch [590/1000], Loss: 1.0215
Epoch [600/1000], Loss: 0.9969
Epoch [610/1000], Loss: 0.9810
Epoch [620/1000], Loss: 0.9628
Epoch [630/1000], Loss: 0.9494
Epoch [640/1000], Loss: 0.9425
Epoch [650/1000], Loss: 0.9360
Epoch [660/1000], Loss: 0.9235
Epoch [670/1000], Loss: 0.9166
Epoch [680/1000], Loss: 0.9053
Epoch [690/1000], Loss: 0.8950
Epoch [700/1000], Loss: 0.8824
Epoch [710/1000], Loss: 0.8662
Epoch [720/1000], Loss: 0.8412
Epoch [730/1000], Loss: 0.8829
Epoch [740/1000], Loss: 1.0123
Epoch [750/1000], Loss: 0.8615
Epoch [760/1000], Loss: 0.9210
Epoch [770/1000], Loss: 0.9549
Epoch [780/1000], Loss: 0.8757
Epoch [790/1000], Loss: 0.8440
Epoch [800/1000], Loss: 0.8173
Epoch [810/1000], Loss: 0.8060
Epoch [820/1000], Loss: 0.8768
Epoch [830/1000], Loss: 0.8175
Epoch [840/1000], Loss: 0.7992
Epoch [850/1000], Loss: 0.8507
Epoch [860/1000], Loss: 0.8000
Epoch [870/1000], Loss: 0.7537
Epoch [880/1000], Loss: 0.7456
Epoch [890/1000], Loss: 0.7443
Epoch [900/1000], Loss: 0.7730
Epoch [910/1000], Loss: 0.7770
Epoch [920/1000], Loss: 0.7438
Epoch [930/1000], Loss: 0.7221
Epoch [940/1000], Loss: 0.6967
Epoch [950/1000], Loss: 0.6655
Epoch [960/1000], Loss: 0.6536
Epoch [970/1000], Loss: 0.6513
Epoch [980/1000], Loss: 0.6478
Epoch [990/1000], Loss: 0.6536
Epoch [1000/1000], Loss: 0.6804
Training RMSE: 0.80848
Testing RMSE: 0.92913
DURATION: -0.0008336999999301042
SCORE: 1.0015133549704076
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 272, LEARNING_RATE = 1e-05,
NUM_EPOCHS = 1000, NUM_LAYERS = 2, DROPOUT_RATE = 0.030803419746356604
Epoch [10/1000], Loss: 2.6212
Epoch [20/1000], Loss: 2.6022
Epoch [30/1000], Loss: 2.5832
Epoch [40/1000], Loss: 2.5639
Epoch [50/1000], Loss: 2.5443
Epoch [60/1000], Loss: 2.5240
Epoch [70/1000], Loss: 2.5026
Epoch [80/1000], Loss: 2.4800
Epoch [90/1000], Loss: 2.4557
Epoch [100/1000], Loss: 2.4292
Epoch [110/1000], Loss: 2.3998
Epoch [120/1000], Loss: 2.3670
Epoch [130/1000], Loss: 2.3295
Epoch [140/1000], Loss: 2.2863
Epoch [150/1000], Loss: 2.2355
Epoch [160/1000], Loss: 2.1752
Epoch [170/1000], Loss: 2.1031
Epoch [180/1000], Loss: 2.0174
Epoch [190/1000], Loss: 1.9193
Epoch [200/1000], Loss: 1.8182
Epoch [210/1000], Loss: 1.7377
Epoch [220/1000], Loss: 1.7013
Epoch [230/1000], Loss: 1.6872
Epoch [240/1000], Loss: 1.6719
Epoch [250/1000], Loss: 1.6593
Epoch [260/1000], Loss: 1.6491
Epoch [270/1000], Loss: 1.6400
Epoch [280/1000], Loss: 1.6321
Epoch [290/1000], Loss: 1.6251
Epoch [300/1000], Loss: 1.6190
Epoch [310/1000], Loss: 1.6136
Epoch [320/1000], Loss: 1.6088
Epoch [330/1000], Loss: 1.6045
Epoch [340/1000], Loss: 1.6008
Epoch [350/1000], Loss: 1.5974
Epoch [360/1000], Loss: 1.5945
Epoch [370/1000], Loss: 1.5918
Epoch [380/1000], Loss: 1.5895
Epoch [390/1000], Loss: 1.5874
Epoch [400/1000], Loss: 1.5855
Epoch [410/1000], Loss: 1.5838
Epoch [420/1000], Loss: 1.5823
Epoch [430/1000], Loss: 1.5809
Epoch [440/1000], Loss: 1.5796
Epoch [450/1000], Loss: 1.5784
Epoch [460/1000], Loss: 1.5773
Epoch [470/1000], Loss: 1.5762
Epoch [480/1000], Loss: 1.5751
Epoch [490/1000], Loss: 1.5740
Epoch [500/1000], Loss: 1.5729
Epoch [510/1000], Loss: 1.5717
Epoch [520/1000], Loss: 1.5705
Epoch [530/1000], Loss: 1.5692
Epoch [540/1000], Loss: 1.5677
Epoch [550/1000], Loss: 1.5662
Epoch [560/1000], Loss: 1.5644
Epoch [570/1000], Loss: 1.5625
Epoch [580/1000], Loss: 1.5602
Epoch [590/1000], Loss: 1.5575
Epoch [600/1000], Loss: 1.5543
Epoch [610/1000], Loss: 1.5505
Epoch [620/1000], Loss: 1.5458
Epoch [630/1000], Loss: 1.5399
Epoch [640/1000], Loss: 1.5324
Epoch [650/1000], Loss: 1.5227
Epoch [660/1000], Loss: 1.5101
Epoch [670/1000], Loss: 1.4937
Epoch [680/1000], Loss: 1.4733
Epoch [690/1000], Loss: 1.4503
Epoch [700/1000], Loss: 1.4228
Epoch [710/1000], Loss: 1.3993
Epoch [720/1000], Loss: 1.3837
Epoch [730/1000], Loss: 1.3773
Epoch [740/1000], Loss: 1.3762
Epoch [750/1000], Loss: 1.3802
Epoch [760/1000], Loss: 1.3776
Epoch [770/1000], Loss: 1.3581
Epoch [780/1000], Loss: 1.3457
Epoch [790/1000], Loss: 1.4154
Epoch [800/1000], Loss: 1.4443
Epoch [810/1000], Loss: 1.3496
Epoch [820/1000], Loss: 1.4252
Epoch [830/1000], Loss: 1.4247
Epoch [840/1000], Loss: 1.4743
Epoch [850/1000], Loss: 1.3943
Epoch [860/1000], Loss: 1.3570
Epoch [870/1000], Loss: 1.3431
Epoch [880/1000], Loss: 1.3632
Epoch [890/1000], Loss: 1.3936
Epoch [900/1000], Loss: 1.3742
Epoch [910/1000], Loss: 1.3673
Epoch [920/1000], Loss: 1.3424
Epoch [930/1000], Loss: 1.3394
Epoch [940/1000], Loss: 1.3505
Epoch [950/1000], Loss: 1.3408
Epoch [960/1000], Loss: 1.3620
Epoch [970/1000], Loss: 1.3609
Epoch [980/1000], Loss: 1.3545
Epoch [990/1000], Loss: 1.3326
Epoch [1000/1000], Loss: 1.3230
Training RMSE: 1.15133
Testing RMSE: 1.23002
DURATION: 0.0006950999995751772
SCORE: 1.277236255088954
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 100, LEARNING_RATE = 0.01,
NUM_EPOCHS = 100, NUM_LAYERS = 2, DROPOUT_RATE = 0.4680103890402622
Epoch [10/100], Loss: 1.7643
Epoch [20/100], Loss: 1.5959
Epoch [30/100], Loss: 1.5844
Epoch [40/100], Loss: 1.5582
Epoch [50/100], Loss: 1.5064
Epoch [60/100], Loss: 1.5511
Epoch [70/100], Loss: 1.4117
Epoch [80/100], Loss: 1.4249
Epoch [90/100], Loss: 1.3978
Epoch [100/100], Loss: 1.3621
Training RMSE: 1.15450
Testing RMSE: 1.21252
DURATION: 0.0005208000002312474
SCORE: 1.2473373281235196
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 300, LEARNING_RATE = 0.003793294000798966,
NUM_EPOCHS = 100, NUM_LAYERS = 2, DROPOUT_RATE = 0.05447521084754153
Epoch [10/100], Loss: 1.5823
Epoch [20/100], Loss: 1.5887
Epoch [30/100], Loss: 1.5721
Epoch [40/100], Loss: 1.5524
Epoch [50/100], Loss: 1.5179
Epoch [60/100], Loss: 1.5769
Epoch [70/100], Loss: 1.5284
Epoch [80/100], Loss: 1.4309
Epoch [90/100], Loss: 1.5345
Epoch [100/100], Loss: 1.5166
Training RMSE: 1.23246
Testing RMSE: 1.27185
DURATION: -0.00010140000085812062
SCORE: 1.2954862778723741
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 299, LEARNING_RATE = 0.005963672361607049,
NUM_EPOCHS = 1000, NUM_LAYERS = 2, DROPOUT_RATE = 0.18383291956684836
Epoch [10/1000], Loss: 1.8157
Epoch [20/1000], Loss: 1.5641
Epoch [30/1000], Loss: 1.5602
Epoch [40/1000], Loss: 1.5513
Epoch [50/1000], Loss: 1.5350
Epoch [60/1000], Loss: 1.5710
Epoch [70/1000], Loss: 1.5619
Epoch [80/1000], Loss: 1.5749
Epoch [90/1000], Loss: 1.5692
Epoch [100/1000], Loss: 1.5679
Epoch [110/1000], Loss: 1.5662
Epoch [120/1000], Loss: 1.5626
Epoch [130/1000], Loss: 1.5592
Epoch [140/1000], Loss: 1.5452
Epoch [150/1000], Loss: 1.5152
Epoch [160/1000], Loss: 1.5684
Epoch [170/1000], Loss: 1.5377
Epoch [180/1000], Loss: 1.5140
Epoch [190/1000], Loss: 1.5081
Epoch [200/1000], Loss: 1.5151
Epoch [210/1000], Loss: 1.4957
Epoch [220/1000], Loss: 1.4993
Epoch [230/1000], Loss: 1.4886
Epoch [240/1000], Loss: 1.4638
Epoch [250/1000], Loss: 1.4513
Epoch [260/1000], Loss: 1.4474
Epoch [270/1000], Loss: 1.4377
Epoch [280/1000], Loss: 1.4312
Epoch [290/1000], Loss: 1.4851
Epoch [300/1000], Loss: 1.4661
Epoch [310/1000], Loss: 1.3980
Epoch [320/1000], Loss: 1.4037
Epoch [330/1000], Loss: 1.3607
Epoch [340/1000], Loss: 1.3257
Epoch [350/1000], Loss: 1.2850
Epoch [360/1000], Loss: 1.3894
Epoch [370/1000], Loss: 1.3201
Epoch [380/1000], Loss: 1.2474
Epoch [390/1000], Loss: 1.1643
Epoch [400/1000], Loss: 1.0939
Epoch [410/1000], Loss: 1.0529
Epoch [420/1000], Loss: 1.0295
Epoch [430/1000], Loss: 0.9957
Epoch [440/1000], Loss: 0.9604
Epoch [450/1000], Loss: 0.9206
Epoch [460/1000], Loss: 0.8795
Epoch [470/1000], Loss: 0.8582
Epoch [480/1000], Loss: 0.8319
Epoch [490/1000], Loss: 0.8891
Epoch [500/1000], Loss: 0.7963
Epoch [510/1000], Loss: 0.7997
Epoch [520/1000], Loss: 0.7888
Epoch [530/1000], Loss: 0.7767
Epoch [540/1000], Loss: 0.7667
Epoch [550/1000], Loss: 0.7535
Epoch [560/1000], Loss: 0.7351
Epoch [570/1000], Loss: 0.7139
Epoch [580/1000], Loss: 0.7098
Epoch [590/1000], Loss: 0.6785
Epoch [600/1000], Loss: 0.6451
Epoch [610/1000], Loss: 0.6752
Epoch [620/1000], Loss: 0.6153
Epoch [630/1000], Loss: 0.5866
Epoch [640/1000], Loss: 0.5672
Epoch [650/1000], Loss: 0.5371
Epoch [660/1000], Loss: 0.5521
Epoch [670/1000], Loss: 0.5291
Epoch [680/1000], Loss: 0.5307
Epoch [690/1000], Loss: 0.5099
Epoch [700/1000], Loss: 0.4951
Epoch [710/1000], Loss: 0.4770
Epoch [720/1000], Loss: 0.4534
Epoch [730/1000], Loss: 0.4456
Epoch [740/1000], Loss: 0.4889
Epoch [750/1000], Loss: 0.4333
Epoch [760/1000], Loss: 0.4293
Epoch [770/1000], Loss: 0.4121
Epoch [780/1000], Loss: 0.4039
Epoch [790/1000], Loss: 0.3958
Epoch [800/1000], Loss: 0.3832
Epoch [810/1000], Loss: 0.3624
Epoch [820/1000], Loss: 0.3431
Epoch [830/1000], Loss: 0.3579
Epoch [840/1000], Loss: 0.3460
Epoch [850/1000], Loss: 0.3379
Epoch [860/1000], Loss: 0.3274
Epoch [870/1000], Loss: 0.3193
Epoch [880/1000], Loss: 0.3049
Epoch [890/1000], Loss: 0.3043
Epoch [900/1000], Loss: 0.2972
Epoch [910/1000], Loss: 0.2863
Epoch [920/1000], Loss: 0.2915
Epoch [930/1000], Loss: 0.3011
Epoch [940/1000], Loss: 0.2760
Epoch [950/1000], Loss: 0.2899
Epoch [960/1000], Loss: 0.2801
Epoch [970/1000], Loss: 0.2687
Epoch [980/1000], Loss: 0.2626
Epoch [990/1000], Loss: 0.2604
Epoch [1000/1000], Loss: 0.2573
Training RMSE: 0.50030
Testing RMSE: 1.37989
DURATION: -0.0001894999986689072
SCORE: 1.907649294339076
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 158, LEARNING_RATE = 0.0004878993828026078,
NUM_EPOCHS = 973, NUM_LAYERS = 1, DROPOUT_RATE = 0.0
Epoch [10/973], Loss: 2.3083
Epoch [20/973], Loss: 1.6575
Epoch [30/973], Loss: 1.6054
Epoch [40/973], Loss: 1.5695
Epoch [50/973], Loss: 1.5674
Epoch [60/973], Loss: 1.5656
Epoch [70/973], Loss: 1.5607
Epoch [80/973], Loss: 1.5502
Epoch [90/973], Loss: 1.5217
Epoch [100/973], Loss: 1.4506
Epoch [110/973], Loss: 1.3978
Epoch [120/973], Loss: 1.4981
Epoch [130/973], Loss: 1.7914
Epoch [140/973], Loss: 1.6257
Epoch [150/973], Loss: 1.5365
Epoch [160/973], Loss: 1.4778
Epoch [170/973], Loss: 1.4039
Epoch [180/973], Loss: 1.4283
Epoch [190/973], Loss: 1.3723
Epoch [200/973], Loss: 1.3777
Epoch [210/973], Loss: 1.4466
Epoch [220/973], Loss: 1.4568
Epoch [230/973], Loss: 1.4310
Epoch [240/973], Loss: 1.3222
Epoch [250/973], Loss: 1.3024
Epoch [260/973], Loss: 1.4818
Epoch [270/973], Loss: 1.4381
Epoch [280/973], Loss: 1.3532
Epoch [290/973], Loss: 1.3448
Epoch [300/973], Loss: 1.3367
Epoch [310/973], Loss: 1.3594
Epoch [320/973], Loss: 1.3678
Epoch [330/973], Loss: 1.3038
Epoch [340/973], Loss: 1.3136
Epoch [350/973], Loss: 1.3641
Epoch [360/973], Loss: 1.2835
Epoch [370/973], Loss: 1.3581
Epoch [380/973], Loss: 1.3192
Epoch [390/973], Loss: 1.2559
Epoch [400/973], Loss: 1.3592
Epoch [410/973], Loss: 1.2905
Epoch [420/973], Loss: 1.2074
Epoch [430/973], Loss: 1.3480
Epoch [440/973], Loss: 1.2227
Epoch [450/973], Loss: 1.2003
Epoch [460/973], Loss: 1.3066
Epoch [470/973], Loss: 1.1708
Epoch [480/973], Loss: 1.2669
Epoch [490/973], Loss: 1.1589
Epoch [500/973], Loss: 1.2665
Epoch [510/973], Loss: 1.0995
Epoch [520/973], Loss: 1.2249
Epoch [530/973], Loss: 1.1201
Epoch [540/973], Loss: 1.1327
Epoch [550/973], Loss: 1.2032
Epoch [560/973], Loss: 1.0798
Epoch [570/973], Loss: 1.0564
Epoch [580/973], Loss: 1.1620
Epoch [590/973], Loss: 1.0755
Epoch [600/973], Loss: 1.0252
Epoch [610/973], Loss: 1.0791
Epoch [620/973], Loss: 1.0908
Epoch [630/973], Loss: 1.0466
Epoch [640/973], Loss: 1.0048
Epoch [650/973], Loss: 0.9848
Epoch [660/973], Loss: 0.9758
Epoch [670/973], Loss: 0.9669
Epoch [680/973], Loss: 0.9590
Epoch [690/973], Loss: 0.9541
Epoch [700/973], Loss: 0.9459
Epoch [710/973], Loss: 0.9319
Epoch [720/973], Loss: 0.9250
Epoch [730/973], Loss: 0.9132
Epoch [740/973], Loss: 0.9035
Epoch [750/973], Loss: 0.8920
Epoch [760/973], Loss: 0.8795
Epoch [770/973], Loss: 0.8639
Epoch [780/973], Loss: 0.8452
Epoch [790/973], Loss: 0.8202
Epoch [800/973], Loss: 0.7907
Epoch [810/973], Loss: 0.7896
Epoch [820/973], Loss: 1.0682
Epoch [830/973], Loss: 0.8693
Epoch [840/973], Loss: 0.9363
Epoch [850/973], Loss: 0.9556
Epoch [860/973], Loss: 0.8772
Epoch [870/973], Loss: 0.8147
Epoch [880/973], Loss: 0.7982
Epoch [890/973], Loss: 0.7616
Epoch [900/973], Loss: 0.7192
Epoch [910/973], Loss: 0.7469
Epoch [920/973], Loss: 0.7749
Epoch [930/973], Loss: 0.7402
Epoch [940/973], Loss: 0.7142
Epoch [950/973], Loss: 0.6944
Epoch [960/973], Loss: 0.6864
Epoch [970/973], Loss: 0.6913
Training RMSE: 0.81575
Testing RMSE: 0.99997
DURATION: 0.0007402000010188203
SCORE: 1.110494383365904
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 300, LEARNING_RATE = 0.001838905868692176,
NUM_EPOCHS = 1000, NUM_LAYERS = 1, DROPOUT_RATE = 0.0
Epoch [10/1000], Loss: 1.8418
Epoch [20/1000], Loss: 1.6321
Epoch [30/1000], Loss: 1.5807
Epoch [40/1000], Loss: 1.5688
Epoch [50/1000], Loss: 1.5624
Epoch [60/1000], Loss: 1.5567
Epoch [70/1000], Loss: 1.5438
Epoch [80/1000], Loss: 1.3903
Epoch [90/1000], Loss: 1.5453
Epoch [100/1000], Loss: 1.6078
Epoch [110/1000], Loss: 1.5736
Epoch [120/1000], Loss: 1.5599
Epoch [130/1000], Loss: 1.5486
Epoch [140/1000], Loss: 1.5172
Epoch [150/1000], Loss: 1.3956
Epoch [160/1000], Loss: 1.3859
Epoch [170/1000], Loss: 1.3951
Epoch [180/1000], Loss: 1.3998
Epoch [190/1000], Loss: 1.3946
Epoch [200/1000], Loss: 1.3726
Epoch [210/1000], Loss: 1.3627
Epoch [220/1000], Loss: 1.3370
Epoch [230/1000], Loss: 1.3882
Epoch [240/1000], Loss: 1.4486
Epoch [250/1000], Loss: 1.3561
Epoch [260/1000], Loss: 1.3509
Epoch [270/1000], Loss: 1.3917
Epoch [280/1000], Loss: 1.3786
Epoch [290/1000], Loss: 1.2748
Epoch [300/1000], Loss: 1.3156
Epoch [310/1000], Loss: 1.2310
Epoch [320/1000], Loss: 1.3310
Epoch [330/1000], Loss: 1.2608
Epoch [340/1000], Loss: 1.1313
Epoch [350/1000], Loss: 1.1745
Epoch [360/1000], Loss: 1.1151
Epoch [370/1000], Loss: 1.1033
Epoch [380/1000], Loss: 1.0293
Epoch [390/1000], Loss: 0.9430
Epoch [400/1000], Loss: 0.8796
Epoch [410/1000], Loss: 0.8147
Epoch [420/1000], Loss: 0.8492
Epoch [430/1000], Loss: 0.8156
Epoch [440/1000], Loss: 0.7250
Epoch [450/1000], Loss: 0.6520
Epoch [460/1000], Loss: 0.5916
Epoch [470/1000], Loss: 0.5924
Epoch [480/1000], Loss: 0.5278
Epoch [490/1000], Loss: 0.4832
Epoch [500/1000], Loss: 0.5459
Epoch [510/1000], Loss: 0.4936
Epoch [520/1000], Loss: 0.4458
Epoch [530/1000], Loss: 0.4574
Epoch [540/1000], Loss: 0.4548
Epoch [550/1000], Loss: 0.4738
Epoch [560/1000], Loss: 0.4575
Epoch [570/1000], Loss: 0.4289
Epoch [580/1000], Loss: 0.4147
Epoch [590/1000], Loss: 0.3891
Epoch [600/1000], Loss: 0.3672
Epoch [610/1000], Loss: 0.3956
Epoch [620/1000], Loss: 0.3804
Epoch [630/1000], Loss: 0.3953
Epoch [640/1000], Loss: 0.3494
Epoch [650/1000], Loss: 0.3826
Epoch [660/1000], Loss: 0.3345
Epoch [670/1000], Loss: 0.3650
Epoch [680/1000], Loss: 0.3399
Epoch [690/1000], Loss: 0.3128
Epoch [700/1000], Loss: 0.3379
Epoch [710/1000], Loss: 0.3072
Epoch [720/1000], Loss: 0.3177
Epoch [730/1000], Loss: 0.2922
Epoch [740/1000], Loss: 0.2875
Epoch [750/1000], Loss: 0.2822
Epoch [760/1000], Loss: 0.2750
Epoch [770/1000], Loss: 0.2645
Epoch [780/1000], Loss: 0.2607
Epoch [790/1000], Loss: 0.2555
Epoch [800/1000], Loss: 0.2495
Epoch [810/1000], Loss: 0.2435
Epoch [820/1000], Loss: 0.2399
Epoch [830/1000], Loss: 0.2347
Epoch [840/1000], Loss: 0.2291
Epoch [850/1000], Loss: 0.2214
Epoch [860/1000], Loss: 0.2185
Epoch [870/1000], Loss: 0.2212
Epoch [880/1000], Loss: 0.2223
Epoch [890/1000], Loss: 0.2298
Epoch [900/1000], Loss: 0.2086
Epoch [910/1000], Loss: 0.2242
Epoch [920/1000], Loss: 0.2155
Epoch [930/1000], Loss: 0.2116
Epoch [940/1000], Loss: 0.2059
Epoch [950/1000], Loss: 0.1932
Epoch [960/1000], Loss: 0.1940
Epoch [970/1000], Loss: 0.1915
Epoch [980/1000], Loss: 0.1837
Epoch [990/1000], Loss: 0.2072
Epoch [1000/1000], Loss: 0.2018
Training RMSE: 0.44067
Testing RMSE: 0.74005
DURATION: -7.950000508571975e-05
SCORE: 0.9196878295281781
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 100, LEARNING_RATE = 0.00478450733247406,
NUM_EPOCHS = 807, NUM_LAYERS = 2, DROPOUT_RATE = 0.0594359589791357
Epoch [10/807], Loss: 1.5972
Epoch [20/807], Loss: 1.5718
Epoch [30/807], Loss: 1.5633
Epoch [40/807], Loss: 1.5503
Epoch [50/807], Loss: 1.5392
Epoch [60/807], Loss: 1.4048
Epoch [70/807], Loss: 1.5232
Epoch [80/807], Loss: 1.4015
Epoch [90/807], Loss: 1.4374
Epoch [100/807], Loss: 1.3832
Epoch [110/807], Loss: 1.3509
Epoch [120/807], Loss: 1.3202
Epoch [130/807], Loss: 1.3253
Epoch [140/807], Loss: 1.5491
Epoch [150/807], Loss: 1.3219
Epoch [160/807], Loss: 1.4454
Epoch [170/807], Loss: 1.4199
Epoch [180/807], Loss: 1.3731
Epoch [190/807], Loss: 1.3210
Epoch [200/807], Loss: 1.2986
Epoch [210/807], Loss: 1.3156
Epoch [220/807], Loss: 1.2852
Epoch [230/807], Loss: 1.1990
Epoch [240/807], Loss: 1.1783
Epoch [250/807], Loss: 1.0630
Epoch [260/807], Loss: 0.9994
Epoch [270/807], Loss: 1.0274
Epoch [280/807], Loss: 0.9303
Epoch [290/807], Loss: 0.9059
Epoch [300/807], Loss: 0.8328
Epoch [310/807], Loss: 0.8333
Epoch [320/807], Loss: 0.6737
Epoch [330/807], Loss: 0.5973
Epoch [340/807], Loss: 0.6038
Epoch [350/807], Loss: 0.5376
Epoch [360/807], Loss: 0.5144
Epoch [370/807], Loss: 0.5123
Epoch [380/807], Loss: 0.5035
Epoch [390/807], Loss: 0.4784
Epoch [400/807], Loss: 0.4497
Epoch [410/807], Loss: 0.4885
Epoch [420/807], Loss: 0.4352
Epoch [430/807], Loss: 0.4158
Epoch [440/807], Loss: 0.3968
Epoch [450/807], Loss: 0.3791
Epoch [460/807], Loss: 0.3868
Epoch [470/807], Loss: 0.3881
Epoch [480/807], Loss: 0.3640
Epoch [490/807], Loss: 0.3573
Epoch [500/807], Loss: 0.3628
Epoch [510/807], Loss: 0.3983
Epoch [520/807], Loss: 0.3756
Epoch [530/807], Loss: 0.4127
Epoch [540/807], Loss: 0.3665
Epoch [550/807], Loss: 0.4080
Epoch [560/807], Loss: 0.3899
Epoch [570/807], Loss: 0.3327
Epoch [580/807], Loss: 0.3204
Epoch [590/807], Loss: 0.3169
Epoch [600/807], Loss: 0.2973
Epoch [610/807], Loss: 0.2900
Epoch [620/807], Loss: 0.2862
Epoch [630/807], Loss: 0.2681
Epoch [640/807], Loss: 0.2732
Epoch [650/807], Loss: 0.2655
Epoch [660/807], Loss: 0.2598
Epoch [670/807], Loss: 0.2479
Epoch [680/807], Loss: 0.2473
Epoch [690/807], Loss: 0.2601
Epoch [700/807], Loss: 0.2707
Epoch [710/807], Loss: 0.2441
Epoch [720/807], Loss: 0.2653
Epoch [730/807], Loss: 0.3031
Epoch [740/807], Loss: 0.2886
Epoch [750/807], Loss: 0.2868
Epoch [760/807], Loss: 0.2681
Epoch [770/807], Loss: 0.2750
Epoch [780/807], Loss: 0.2719
Epoch [790/807], Loss: 0.2415
Epoch [800/807], Loss: 0.2280
Training RMSE: 0.49779
Testing RMSE: 0.70067
DURATION: -0.0004302000015741214
SCORE: 0.8224066768795268
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 237, LEARNING_RATE = 0.0021920122124044206,
NUM_EPOCHS = 820, NUM_LAYERS = 1, DROPOUT_RATE = 0.0784748661329657
Epoch [10/820], Loss: 1.6713
Epoch [20/820], Loss: 1.5788
Epoch [30/820], Loss: 1.5740
Epoch [40/820], Loss: 1.5678
Epoch [50/820], Loss: 1.5622
Epoch [60/820], Loss: 1.5587
Epoch [70/820], Loss: 1.5543
Epoch [80/820], Loss: 1.5400
Epoch [90/820], Loss: 1.4441
Epoch [100/820], Loss: 1.6026
Epoch [110/820], Loss: 1.5478
Epoch [120/820], Loss: 1.5197
Epoch [130/820], Loss: 1.4081
Epoch [140/820], Loss: 1.4042
Epoch [150/820], Loss: 1.3599
Epoch [160/820], Loss: 1.3469
Epoch [170/820], Loss: 1.3283
Epoch [180/820], Loss: 1.2782
Epoch [190/820], Loss: 1.2631
Epoch [200/820], Loss: 1.2410
Epoch [210/820], Loss: 1.2373
Epoch [220/820], Loss: 1.2229
Epoch [230/820], Loss: 1.1204
Epoch [240/820], Loss: 1.1117
Epoch [250/820], Loss: 1.0625
Epoch [260/820], Loss: 1.0409
Epoch [270/820], Loss: 1.0038
Epoch [280/820], Loss: 0.9393
Epoch [290/820], Loss: 0.9098
Epoch [300/820], Loss: 0.9538
Epoch [310/820], Loss: 0.9452
Epoch [320/820], Loss: 0.8864
Epoch [330/820], Loss: 0.8873
Epoch [340/820], Loss: 0.8332
Epoch [350/820], Loss: 0.7881
Epoch [360/820], Loss: 0.7353
Epoch [370/820], Loss: 0.6976
Epoch [380/820], Loss: 0.6602
Epoch [390/820], Loss: 0.6823
Epoch [400/820], Loss: 0.6882
Epoch [410/820], Loss: 0.8375
Epoch [420/820], Loss: 0.7379
Epoch [430/820], Loss: 0.7962
Epoch [440/820], Loss: 0.7070
Epoch [450/820], Loss: 0.6882
Epoch [460/820], Loss: 0.6156
Epoch [470/820], Loss: 0.5755
Epoch [480/820], Loss: 0.6026
Epoch [490/820], Loss: 0.5823
Epoch [500/820], Loss: 0.5686
Epoch [510/820], Loss: 0.5358
Epoch [520/820], Loss: 0.5257
Epoch [530/820], Loss: 0.5111
Epoch [540/820], Loss: 0.4968
Epoch [550/820], Loss: 0.4950
Epoch [560/820], Loss: 0.4684
Epoch [570/820], Loss: 0.4290
Epoch [580/820], Loss: 0.3908
Epoch [590/820], Loss: 0.3700
Epoch [600/820], Loss: 0.5192
Epoch [610/820], Loss: 0.4945
Epoch [620/820], Loss: 0.9611
Epoch [630/820], Loss: 0.5754
Epoch [640/820], Loss: 0.5543
Epoch [650/820], Loss: 0.4533
Epoch [660/820], Loss: 0.3942
Epoch [670/820], Loss: 0.3636
Epoch [680/820], Loss: 0.3456
Epoch [690/820], Loss: 0.3317
Epoch [700/820], Loss: 0.3194
Epoch [710/820], Loss: 0.3095
Epoch [720/820], Loss: 0.3017
Epoch [730/820], Loss: 0.2983
Epoch [740/820], Loss: 0.2913
Epoch [750/820], Loss: 0.2845
Epoch [760/820], Loss: 0.2774
Epoch [770/820], Loss: 0.2708
Epoch [780/820], Loss: 0.3344
Epoch [790/820], Loss: 0.3576
Epoch [800/820], Loss: 0.3437
Epoch [810/820], Loss: 0.4741
Epoch [820/820], Loss: 0.4059
Training RMSE: 0.64770
Testing RMSE: 0.74302
DURATION: -0.00018899999849963933
SCORE: 0.8002158168065322
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 100, LEARNING_RATE = 0.0011911652899656581,
NUM_EPOCHS = 824, NUM_LAYERS = 2, DROPOUT_RATE = 0.1333868930379805
Epoch [10/824], Loss: 1.5877
Epoch [20/824], Loss: 1.6110
Epoch [30/824], Loss: 1.5760
Epoch [40/824], Loss: 1.5658
Epoch [50/824], Loss: 1.5608
Epoch [60/824], Loss: 1.5574
Epoch [70/824], Loss: 1.5538
Epoch [80/824], Loss: 1.5460
Epoch [90/824], Loss: 1.4977
Epoch [100/824], Loss: 1.4272
Epoch [110/824], Loss: 1.3178
Epoch [120/824], Loss: 1.3487
Epoch [130/824], Loss: 1.3404
Epoch [140/824], Loss: 1.3896
Epoch [150/824], Loss: 1.2982
Epoch [160/824], Loss: 1.4698
Epoch [170/824], Loss: 1.3849
Epoch [180/824], Loss: 1.3031
Epoch [190/824], Loss: 1.3526
Epoch [200/824], Loss: 1.2616
Epoch [210/824], Loss: 1.3206
Epoch [220/824], Loss: 1.2241
Epoch [230/824], Loss: 1.1407
Epoch [240/824], Loss: 1.1146
Epoch [250/824], Loss: 1.1300
Epoch [260/824], Loss: 1.1544
Epoch [270/824], Loss: 1.1036
Epoch [280/824], Loss: 1.0541
Epoch [290/824], Loss: 1.1444
Epoch [300/824], Loss: 1.1327
Epoch [310/824], Loss: 1.0650
Epoch [320/824], Loss: 1.0291
Epoch [330/824], Loss: 1.0354
Epoch [340/824], Loss: 1.0791
Epoch [350/824], Loss: 1.0393
Epoch [360/824], Loss: 0.9839
Epoch [370/824], Loss: 0.9746
Epoch [380/824], Loss: 1.0161
Epoch [390/824], Loss: 1.0272
Epoch [400/824], Loss: 0.9925
Epoch [410/824], Loss: 0.9415
Epoch [420/824], Loss: 0.9494
Epoch [430/824], Loss: 0.9624
Epoch [440/824], Loss: 0.9133
Epoch [450/824], Loss: 0.9259
Epoch [460/824], Loss: 0.9079
Epoch [470/824], Loss: 0.8815
Epoch [480/824], Loss: 0.8396
Epoch [490/824], Loss: 0.8380
Epoch [500/824], Loss: 0.8370
Epoch [510/824], Loss: 0.9379
Epoch [520/824], Loss: 0.9547
Epoch [530/824], Loss: 0.7989
Epoch [540/824], Loss: 0.8273
Epoch [550/824], Loss: 0.7701
Epoch [560/824], Loss: 0.7318
Epoch [570/824], Loss: 0.7954
Epoch [580/824], Loss: 0.8352
Epoch [590/824], Loss: 0.7607
Epoch [600/824], Loss: 0.7101
Epoch [610/824], Loss: 0.7392
Epoch [620/824], Loss: 0.7582
Epoch [630/824], Loss: 0.7114
Epoch [640/824], Loss: 0.6445
Epoch [650/824], Loss: 0.6338
Epoch [660/824], Loss: 0.6595
Epoch [670/824], Loss: 0.6441
Epoch [680/824], Loss: 0.5991
Epoch [690/824], Loss: 0.6028
Epoch [700/824], Loss: 0.6234
Epoch [710/824], Loss: 0.5884
Epoch [720/824], Loss: 0.5605
Epoch [730/824], Loss: 0.6036
Epoch [740/824], Loss: 0.5954
Epoch [750/824], Loss: 0.5694
Epoch [760/824], Loss: 0.5810
Epoch [770/824], Loss: 0.5800
Epoch [780/824], Loss: 0.5777
Epoch [790/824], Loss: 0.5317
Epoch [800/824], Loss: 0.5409
Epoch [810/824], Loss: 0.5849
Epoch [820/824], Loss: 0.5960
Training RMSE: 0.73982
Testing RMSE: 0.82746
DURATION: -0.0002855000057024881
SCORE: 0.8800495170750071
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 121, LEARNING_RATE = 0.004269256770533288,
NUM_EPOCHS = 781, NUM_LAYERS = 1, DROPOUT_RATE = 0.13622308809292943
Epoch [10/781], Loss: 1.7766
Epoch [20/781], Loss: 1.6161
Epoch [30/781], Loss: 1.5806
Epoch [40/781], Loss: 1.5683
Epoch [50/781], Loss: 1.5622
Epoch [60/781], Loss: 1.5555
Epoch [70/781], Loss: 1.5344
Epoch [80/781], Loss: 1.4094
Epoch [90/781], Loss: 1.4229
Epoch [100/781], Loss: 1.5813
Epoch [110/781], Loss: 1.5680
Epoch [120/781], Loss: 1.5589
Epoch [130/781], Loss: 1.5536
Epoch [140/781], Loss: 1.5492
Epoch [150/781], Loss: 1.5434
Epoch [160/781], Loss: 1.5219
Epoch [170/781], Loss: 1.3635
Epoch [180/781], Loss: 1.4289
Epoch [190/781], Loss: 1.6267
Epoch [200/781], Loss: 1.5761
Epoch [210/781], Loss: 1.5576
Epoch [220/781], Loss: 1.5479
Epoch [230/781], Loss: 1.5375
Epoch [240/781], Loss: 1.4613
Epoch [250/781], Loss: 1.6595
Epoch [260/781], Loss: 1.4447
Epoch [270/781], Loss: 1.3976
Epoch [280/781], Loss: 1.3531
Epoch [290/781], Loss: 1.3299
Epoch [300/781], Loss: 1.3127
Epoch [310/781], Loss: 1.2766
Epoch [320/781], Loss: 1.3660
Epoch [330/781], Loss: 1.3180
Epoch [340/781], Loss: 1.2646
Epoch [350/781], Loss: 1.1875
Epoch [360/781], Loss: 1.2248
Epoch [370/781], Loss: 1.2056
Epoch [380/781], Loss: 1.1476
Epoch [390/781], Loss: 1.1197
Epoch [400/781], Loss: 1.1358
Epoch [410/781], Loss: 1.1102
Epoch [420/781], Loss: 1.0297
Epoch [430/781], Loss: 0.9872
Epoch [440/781], Loss: 0.9288
Epoch [450/781], Loss: 0.8940
Epoch [460/781], Loss: 0.8669
Epoch [470/781], Loss: 0.8309
Epoch [480/781], Loss: 0.8140
Epoch [490/781], Loss: 0.8059
Epoch [500/781], Loss: 0.9562
Epoch [510/781], Loss: 0.7855
Epoch [520/781], Loss: 0.9825
Epoch [530/781], Loss: 0.9146
Epoch [540/781], Loss: 0.7528
Epoch [550/781], Loss: 0.8221
Epoch [560/781], Loss: 0.6958
Epoch [570/781], Loss: 0.6272
Epoch [580/781], Loss: 0.5793
Epoch [590/781], Loss: 0.5623
Epoch [600/781], Loss: 0.5176
Epoch [610/781], Loss: 0.4695
Epoch [620/781], Loss: 0.4513
Epoch [630/781], Loss: 0.4220
Epoch [640/781], Loss: 0.4025
Epoch [650/781], Loss: 0.3895
Epoch [660/781], Loss: 0.3757
Epoch [670/781], Loss: 0.3665
Epoch [680/781], Loss: 0.3581
Epoch [690/781], Loss: 0.3465
Epoch [700/781], Loss: 0.3408
Epoch [710/781], Loss: 0.3661
Epoch [720/781], Loss: 0.5312
Epoch [730/781], Loss: 0.6163
Epoch [740/781], Loss: 0.5355
Epoch [750/781], Loss: 0.5242
Epoch [760/781], Loss: 0.5216
Epoch [770/781], Loss: 0.4534
Epoch [780/781], Loss: 0.4428
Training RMSE: 0.62918
Testing RMSE: 0.83213
DURATION: 0.0003104999996139668
SCORE: 0.9539018926234503
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 300, LEARNING_RATE = 0.0001199285236069628,
NUM_EPOCHS = 826, NUM_LAYERS = 2, DROPOUT_RATE = 0.0016248279379209123
Epoch [10/826], Loss: 2.3951
Epoch [20/826], Loss: 2.0229
Epoch [30/826], Loss: 1.7213
Epoch [40/826], Loss: 1.6146
Epoch [50/826], Loss: 1.5795
Epoch [60/826], Loss: 1.5713
Epoch [70/826], Loss: 1.5660
Epoch [80/826], Loss: 1.5580
Epoch [90/826], Loss: 1.5452
Epoch [100/826], Loss: 1.5171
Epoch [110/826], Loss: 1.4551
Epoch [120/826], Loss: 1.4000
Epoch [130/826], Loss: 1.3598
Epoch [140/826], Loss: 1.3517
Epoch [150/826], Loss: 1.3722
Epoch [160/826], Loss: 1.4559
Epoch [170/826], Loss: 1.3647
Epoch [180/826], Loss: 1.3883
Epoch [190/826], Loss: 1.4263
Epoch [200/826], Loss: 1.3960
Epoch [210/826], Loss: 1.3710
Epoch [220/826], Loss: 1.3798
Epoch [230/826], Loss: 1.3502
Epoch [240/826], Loss: 1.3400
Epoch [250/826], Loss: 1.3416
Epoch [260/826], Loss: 1.2819
Epoch [270/826], Loss: 1.3268
Epoch [280/826], Loss: 1.2725
Epoch [290/826], Loss: 1.2437
Epoch [300/826], Loss: 1.2730
Epoch [310/826], Loss: 1.2912
Epoch [320/826], Loss: 1.3167
Epoch [330/826], Loss: 1.2633
Epoch [340/826], Loss: 1.2069
Epoch [350/826], Loss: 1.3356
Epoch [360/826], Loss: 1.2217
Epoch [370/826], Loss: 1.2597
Epoch [380/826], Loss: 1.1632
Epoch [390/826], Loss: 1.2617
Epoch [400/826], Loss: 1.2511
Epoch [410/826], Loss: 1.1324
Epoch [420/826], Loss: 1.2268
Epoch [430/826], Loss: 1.1777
Epoch [440/826], Loss: 1.2709
Epoch [450/826], Loss: 1.1065
Epoch [460/826], Loss: 1.2272
Epoch [470/826], Loss: 1.1449
Epoch [480/826], Loss: 1.2544
Epoch [490/826], Loss: 1.0963
Epoch [500/826], Loss: 1.2135
Epoch [510/826], Loss: 1.1341
Epoch [520/826], Loss: 1.2305
Epoch [530/826], Loss: 1.0909
Epoch [540/826], Loss: 1.1893
Epoch [550/826], Loss: 1.1340
Epoch [560/826], Loss: 1.2064
Epoch [570/826], Loss: 1.0954
Epoch [580/826], Loss: 1.1584
Epoch [590/826], Loss: 1.1449
Epoch [600/826], Loss: 1.1719
Epoch [610/826], Loss: 1.1172
Epoch [620/826], Loss: 1.1212
Epoch [630/826], Loss: 1.1618
Epoch [640/826], Loss: 1.1290
Epoch [650/826], Loss: 1.1480
Epoch [660/826], Loss: 1.0910
Epoch [670/826], Loss: 1.1773
Epoch [680/826], Loss: 1.0865
Epoch [690/826], Loss: 1.1672
Epoch [700/826], Loss: 1.0821
Epoch [710/826], Loss: 1.1771
Epoch [720/826], Loss: 1.0679
Epoch [730/826], Loss: 1.1513
Epoch [740/826], Loss: 1.1013
Epoch [750/826], Loss: 1.1496
Epoch [760/826], Loss: 1.0928
Epoch [770/826], Loss: 1.1052
Epoch [780/826], Loss: 1.1370
Epoch [790/826], Loss: 1.0975
Epoch [800/826], Loss: 1.1383
Epoch [810/826], Loss: 1.0717
Epoch [820/826], Loss: 1.1598
Training RMSE: 1.07107
Testing RMSE: 1.09703
DURATION: -0.0002015999998548068
SCORE: 1.1126054120873587
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 300, LEARNING_RATE = 7.566858347210171e-05,
NUM_EPOCHS = 800, NUM_LAYERS = 1, DROPOUT_RATE = 0.270920778466529
Epoch [10/800], Loss: 2.5047
Epoch [20/800], Loss: 2.4114
Epoch [30/800], Loss: 2.2870
Epoch [40/800], Loss: 2.0767
Epoch [50/800], Loss: 1.7178
Epoch [60/800], Loss: 1.6634
Epoch [70/800], Loss: 1.6273
Epoch [80/800], Loss: 1.5961
Epoch [90/800], Loss: 1.5851
Epoch [100/800], Loss: 1.5793
Epoch [110/800], Loss: 1.5747
Epoch [120/800], Loss: 1.5717
Epoch [130/800], Loss: 1.5690
Epoch [140/800], Loss: 1.5659
Epoch [150/800], Loss: 1.5617
Epoch [160/800], Loss: 1.5550
Epoch [170/800], Loss: 1.5436
Epoch [180/800], Loss: 1.5219
Epoch [190/800], Loss: 1.4781
Epoch [200/800], Loss: 1.4198
Epoch [210/800], Loss: 1.4064
Epoch [220/800], Loss: 1.3782
Epoch [230/800], Loss: 1.4021
Epoch [240/800], Loss: 1.3959
Epoch [250/800], Loss: 1.5794
Epoch [260/800], Loss: 1.3999
Epoch [270/800], Loss: 1.6325
Epoch [280/800], Loss: 1.5285
Epoch [290/800], Loss: 1.4769
Epoch [300/800], Loss: 1.4224
Epoch [310/800], Loss: 1.3764
Epoch [320/800], Loss: 1.3702
Epoch [330/800], Loss: 1.3860
Epoch [340/800], Loss: 1.3691
Epoch [350/800], Loss: 1.4234
Epoch [360/800], Loss: 1.4524
Epoch [370/800], Loss: 1.4818
Epoch [380/800], Loss: 1.4159
Epoch [390/800], Loss: 1.4361
Epoch [400/800], Loss: 1.3525
Epoch [410/800], Loss: 1.3802
Epoch [420/800], Loss: 1.3288
Epoch [430/800], Loss: 1.3496
Epoch [440/800], Loss: 1.3263
Epoch [450/800], Loss: 1.3157
Epoch [460/800], Loss: 1.3580
Epoch [470/800], Loss: 1.3283
Epoch [480/800], Loss: 1.3584
Epoch [490/800], Loss: 1.3352
Epoch [500/800], Loss: 1.4026
Epoch [510/800], Loss: 1.3612
Epoch [520/800], Loss: 1.3686
Epoch [530/800], Loss: 1.3609
Epoch [540/800], Loss: 1.3709
Epoch [550/800], Loss: 1.2970
Epoch [560/800], Loss: 1.2992
Epoch [570/800], Loss: 1.2462
Epoch [580/800], Loss: 1.2538
Epoch [590/800], Loss: 1.2950
Epoch [600/800], Loss: 1.2917
Epoch [610/800], Loss: 1.3013
Epoch [620/800], Loss: 1.1938
Epoch [630/800], Loss: 1.2100
Epoch [640/800], Loss: 1.3281
Epoch [650/800], Loss: 1.2076
Epoch [660/800], Loss: 1.2046
Epoch [670/800], Loss: 1.2980
Epoch [680/800], Loss: 1.3022
Epoch [690/800], Loss: 1.3374
Epoch [700/800], Loss: 1.1668
Epoch [710/800], Loss: 1.2146
Epoch [720/800], Loss: 1.2024
Epoch [730/800], Loss: 1.1910
Epoch [740/800], Loss: 1.2944
Epoch [750/800], Loss: 1.2339
Epoch [760/800], Loss: 1.2239
Epoch [770/800], Loss: 1.1172
Epoch [780/800], Loss: 1.1407
Epoch [790/800], Loss: 1.2388
Epoch [800/800], Loss: 1.2124
Training RMSE: 1.08189
Testing RMSE: 1.13854
DURATION: 0.0018221999998786487
SCORE: 1.1725243263039642
--------------EXPERIMENT END--------------



-------------EXPERIMENT START-------------
CONDITIONS:
HIDDEN_DIM = 100, LEARNING_RATE = 0.007029491926807,
NUM_EPOCHS = 1000, NUM_LAYERS = 2, DROPOUT_RATE = 0.5
Epoch [10/1000], Loss: 1.7577
Epoch [20/1000], Loss: 1.6029
Epoch [30/1000], Loss: 1.5853
Epoch [40/1000], Loss: 1.5666
Epoch [50/1000], Loss: 1.5125
Epoch [60/1000], Loss: 1.4507
Epoch [70/1000], Loss: 1.5421
Epoch [80/1000], Loss: 1.4803
Epoch [90/1000], Loss: 1.4388
Epoch [100/1000], Loss: 1.4151
Epoch [110/1000], Loss: 1.4036
Epoch [120/1000], Loss: 1.4039
Epoch [130/1000], Loss: 1.3802
Epoch [140/1000], Loss: 1.3808
Epoch [150/1000], Loss: 1.3688
Epoch [160/1000], Loss: 1.3308
Epoch [170/1000], Loss: 1.2702
Epoch [180/1000], Loss: 1.2424
Epoch [190/1000], Loss: 1.2865
Epoch [200/1000], Loss: 1.2137
Epoch [210/1000], Loss: 1.3016
Epoch [220/1000], Loss: 1.2677
Epoch [230/1000], Loss: 1.2739
Epoch [240/1000], Loss: 1.2318
Epoch [250/1000], Loss: 1.1917
Epoch [260/1000], Loss: 1.1506
Epoch [270/1000], Loss: 1.1539
Epoch [280/1000], Loss: 1.1252
Epoch [290/1000], Loss: 1.0591
Epoch [300/1000], Loss: 1.1391
Epoch [310/1000], Loss: 1.1238
Epoch [320/1000], Loss: 1.0311
Epoch [330/1000], Loss: 1.0318
Epoch [340/1000], Loss: 0.9730
Epoch [350/1000], Loss: 0.9228
Epoch [360/1000], Loss: 0.8847
Epoch [370/1000], Loss: 0.8740
Epoch [380/1000], Loss: 0.8468
Epoch [390/1000], Loss: 0.8157
Epoch [400/1000], Loss: 0.8024
Epoch [410/1000], Loss: 0.7619
Epoch [420/1000], Loss: 0.7900
Epoch [430/1000], Loss: 0.7524
Epoch [440/1000], Loss: 0.6868
Epoch [450/1000], Loss: 0.6199
Epoch [460/1000], Loss: 0.5833
Epoch [470/1000], Loss: 0.5801
Epoch [480/1000], Loss: 0.6662
Epoch [490/1000], Loss: 0.5836
Epoch [500/1000], Loss: 0.5444
Epoch [510/1000], Loss: 0.4952
Epoch [520/1000], Loss: 0.4818
Epoch [530/1000], Loss: 0.4487
Epoch [540/1000], Loss: 0.4098
Epoch [550/1000], Loss: 0.3898
Epoch [560/1000], Loss: 0.3675
Epoch [570/1000], Loss: 0.3528
Epoch [580/1000], Loss: 0.3351
Epoch [590/1000], Loss: 0.3876
Epoch [600/1000], Loss: 0.4770
Epoch [610/1000], Loss: 0.4454
Epoch [620/1000], Loss: 0.4311
Epoch [630/1000], Loss: 0.4274
Epoch [640/1000], Loss: 0.4224
Epoch [650/1000], Loss: 0.3495
Epoch [660/1000], Loss: 0.3355
Epoch [670/1000], Loss: 0.3194
Epoch [680/1000], Loss: 0.2985
Epoch [690/1000], Loss: 0.2934
Epoch [700/1000], Loss: 0.2798
Epoch [710/1000], Loss: 0.2781
Epoch [720/1000], Loss: 0.2836
Epoch [730/1000], Loss: 0.2631
Epoch [740/1000], Loss: 0.3200
Epoch [750/1000], Loss: 0.3308
Epoch [760/1000], Loss: 0.3830
Epoch [770/1000], Loss: 0.2966
Epoch [780/1000], Loss: 0.3183
Epoch [790/1000], Loss: 0.2973
Epoch [800/1000], Loss: 0.2882
Epoch [810/1000], Loss: 0.2762
Epoch [820/1000], Loss: 0.2712
Epoch [830/1000], Loss: 0.2629
Epoch [840/1000], Loss: 0.2627
Epoch [850/1000], Loss: 0.2639
Epoch [860/1000], Loss: 0.2658
Epoch [870/1000], Loss: 0.2474
Epoch [880/1000], Loss: 0.2377
Epoch [890/1000], Loss: 0.2360
Epoch [900/1000], Loss: 0.2295
Epoch [910/1000], Loss: 0.2235
Epoch [920/1000], Loss: 0.2173
Epoch [930/1000], Loss: 0.2132
Epoch [940/1000], Loss: 0.2082
Epoch [950/1000], Loss: 0.2033
Epoch [960/1000], Loss: 0.1983
Epoch [970/1000], Loss: 0.1935
Epoch [980/1000], Loss: 0.1884
Epoch [990/1000], Loss: 0.1825
Epoch [1000/1000], Loss: 0.1788
Training RMSE: 0.41880
Testing RMSE: 0.79344
DURATION: -0.000454399996669963
SCORE: 1.0182210648477938
--------------EXPERIMENT END--------------



          fun: 0.6860194283100715
            x: [np.int64(165), 0.0016249001630866402, np.int64(1000), np.str_('1'), 0.0]
    func_vals: [ 1.144e+00  8.180e-01 ...  1.173e+00  1.018e+00]
      x_iters: [[np.int64(200), 0.008180206026754532, np.int64(651), '2', 0.4303348856888437], [np.int64(130), 0.001993202402671449, np.int64(834), '1', 0.05806891380927438], [np.int64(103), 0.004873466099260709, np.int64(398), '2', 0.04912596777527102], [np.int64(111), 0.004432199863102637, np.int64(120), '1', 0.12319722162517152], [np.int64(248), 0.008893369055711172, np.int64(988), '1', 0.19689117614693252], [np.int64(159), 0.009587633755102439, np.int64(1000), np.str_('2'), 0.32791925111762543], [np.int64(100), 1e-05, np.int64(100), np.str_('1'), 0.039916823572650924], [np.int64(141), 0.0019966483918312117, np.int64(759), np.str_('1'), 0.32814875387517656], [np.int64(100), 0.0018555954659972237, np.int64(1000), np.str_('1'), 0.0], [np.int64(249), 0.0029530940666112793, np.int64(1000), np.str_('1'), 0.0], [np.int64(165), 0.0016249001630866402, np.int64(1000), np.str_('1'), 0.0], [np.int64(300), 0.00046561129149826563, np.int64(1000), np.str_('1'), 0.0], [np.int64(272), 1e-05, np.int64(1000), np.str_('2'), 0.030803419746356604], [np.int64(100), 0.01, np.int64(100), np.str_('2'), 0.4680103890402622], [np.int64(300), 0.003793294000798966, np.int64(100), np.str_('2'), 0.05447521084754153], [np.int64(299), 0.005963672361607049, np.int64(1000), np.str_('2'), 0.18383291956684836], [np.int64(158), 0.0004878993828026078, np.int64(973), np.str_('1'), 0.0], [np.int64(300), 0.001838905868692176, np.int64(1000), np.str_('1'), 0.0], [np.int64(100), 0.00478450733247406, np.int64(807), np.str_('2'), 0.0594359589791357], [np.int64(237), 0.0021920122124044206, np.int64(820), np.str_('1'), 0.0784748661329657], [np.int64(100), 0.0011911652899656581, np.int64(824), np.str_('2'), 0.1333868930379805], [np.int64(121), 0.004269256770533288, np.int64(781), np.str_('1'), 0.13622308809292943], [np.int64(300), 0.0001199285236069628, np.int64(826), np.str_('2'), 0.0016248279379209123], [np.int64(300), 7.566858347210171e-05, np.int64(800), np.str_('1'), 0.270920778466529], [np.int64(100), 0.007029491926807, np.int64(1000), np.str_('2'), 0.5]]
       models: [GaussianProcessRegressor(kernel=1**2 * Matern(length_scale=[1, 1, 1, 1, 1], nu=2.5) + WhiteKernel(noise_level=1),
                                        n_restarts_optimizer=2, noise='gaussian',
                                        normalize_y=True, random_state=822569775), GaussianProcessRegressor(kernel=1**2 * Matern(length_scale=[1, 1, 1, 1, 1], nu=2.5) + WhiteKernel(noise_level=1),
                                        n_restarts_optimizer=2, noise='gaussian',
                                        normalize_y=True, random_state=822569775), GaussianProcessRegressor(kernel=1**2 * Matern(length_scale=[1, 1, 1, 1, 1], nu=2.5) + WhiteKernel(noise_level=1),
                                        n_restarts_optimizer=2, noise='gaussian',
                                        normalize_y=True, random_state=822569775), GaussianProcessRegressor(kernel=1**2 * Matern(length_scale=[1, 1, 1, 1, 1], nu=2.5) + WhiteKernel(noise_level=1),
                                        n_restarts_optimizer=2, noise='gaussian',
                                        normalize_y=True, random_state=822569775), GaussianProcessRegressor(kernel=1**2 * Matern(length_scale=[1, 1, 1, 1, 1], nu=2.5) + WhiteKernel(noise_level=1),
                                        n_restarts_optimizer=2, noise='gaussian',
                                        normalize_y=True, random_state=822569775), GaussianProcessRegressor(kernel=1**2 * Matern(length_scale=[1, 1, 1, 1, 1], nu=2.5) + WhiteKernel(noise_level=1),
                                        n_restarts_optimizer=2, noise='gaussian',
                                        normalize_y=True, random_state=822569775), GaussianProcessRegressor(kernel=1**2 * Matern(length_scale=[1, 1, 1, 1, 1], nu=2.5) + WhiteKernel(noise_level=1),
                                        n_restarts_optimizer=2, noise='gaussian',
                                        normalize_y=True, random_state=822569775), GaussianProcessRegressor(kernel=1**2 * Matern(length_scale=[1, 1, 1, 1, 1], nu=2.5) + WhiteKernel(noise_level=1),
                                        n_restarts_optimizer=2, noise='gaussian',
                                        normalize_y=True, random_state=822569775), GaussianProcessRegressor(kernel=1**2 * Matern(length_scale=[1, 1, 1, 1, 1], nu=2.5) + WhiteKernel(noise_level=1),
                                        n_restarts_optimizer=2, noise='gaussian',
                                        normalize_y=True, random_state=822569775), GaussianProcessRegressor(kernel=1**2 * Matern(length_scale=[1, 1, 1, 1, 1], nu=2.5) + WhiteKernel(noise_level=1),
                                        n_restarts_optimizer=2, noise='gaussian',
                                        normalize_y=True, random_state=822569775), GaussianProcessRegressor(kernel=1**2 * Matern(length_scale=[1, 1, 1, 1, 1], nu=2.5) + WhiteKernel(noise_level=1),
                                        n_restarts_optimizer=2, noise='gaussian',
                                        normalize_y=True, random_state=822569775), GaussianProcessRegressor(kernel=1**2 * Matern(length_scale=[1, 1, 1, 1, 1], nu=2.5) + WhiteKernel(noise_level=1),
                                        n_restarts_optimizer=2, noise='gaussian',
                                        normalize_y=True, random_state=822569775), GaussianProcessRegressor(kernel=1**2 * Matern(length_scale=[1, 1, 1, 1, 1], nu=2.5) + WhiteKernel(noise_level=1),
                                        n_restarts_optimizer=2, noise='gaussian',
                                        normalize_y=True, random_state=822569775), GaussianProcessRegressor(kernel=1**2 * Matern(length_scale=[1, 1, 1, 1, 1], nu=2.5) + WhiteKernel(noise_level=1),
                                        n_restarts_optimizer=2, noise='gaussian',
                                        normalize_y=True, random_state=822569775), GaussianProcessRegressor(kernel=1**2 * Matern(length_scale=[1, 1, 1, 1, 1], nu=2.5) + WhiteKernel(noise_level=1),
                                        n_restarts_optimizer=2, noise='gaussian',
                                        normalize_y=True, random_state=822569775), GaussianProcessRegressor(kernel=1**2 * Matern(length_scale=[1, 1, 1, 1, 1], nu=2.5) + WhiteKernel(noise_level=1),
                                        n_restarts_optimizer=2, noise='gaussian',
                                        normalize_y=True, random_state=822569775), GaussianProcessRegressor(kernel=1**2 * Matern(length_scale=[1, 1, 1, 1, 1], nu=2.5) + WhiteKernel(noise_level=1),
                                        n_restarts_optimizer=2, noise='gaussian',
                                        normalize_y=True, random_state=822569775), GaussianProcessRegressor(kernel=1**2 * Matern(length_scale=[1, 1, 1, 1, 1], nu=2.5) + WhiteKernel(noise_level=1),
                                        n_restarts_optimizer=2, noise='gaussian',
                                        normalize_y=True, random_state=822569775), GaussianProcessRegressor(kernel=1**2 * Matern(length_scale=[1, 1, 1, 1, 1], nu=2.5) + WhiteKernel(noise_level=1),
                                        n_restarts_optimizer=2, noise='gaussian',
                                        normalize_y=True, random_state=822569775), GaussianProcessRegressor(kernel=1**2 * Matern(length_scale=[1, 1, 1, 1, 1], nu=2.5) + WhiteKernel(noise_level=1),
                                        n_restarts_optimizer=2, noise='gaussian',
                                        normalize_y=True, random_state=822569775), GaussianProcessRegressor(kernel=1**2 * Matern(length_scale=[1, 1, 1, 1, 1], nu=2.5) + WhiteKernel(noise_level=1),
                                        n_restarts_optimizer=2, noise='gaussian',
                                        normalize_y=True, random_state=822569775)]
        space: Space([Integer(low=100, high=300, prior='uniform', transform='normalize'),
                      Real(low=1e-05, high=0.01, prior='uniform', transform='normalize'),
                      Integer(low=100, high=1000, prior='uniform', transform='normalize'),
                      Categorical(categories=('1', '2'), prior=None),
                      Real(low=0.0, high=0.5, prior='uniform', transform='normalize')])
 random_state: RandomState(MT19937)
        specs:     args:                    func: <function f at 0x0000025E6D079BC0>
                                      dimensions: Space([Integer(low=100, high=300, prior='uniform', transform='normalize'),
                                                         Real(low=1e-05, high=0.01, prior='uniform', transform='normalize'),
                                                         Integer(low=100, high=1000, prior='uniform', transform='normalize'),
                                                         Categorical(categories=('1', '2'), prior=None),
                                                         Real(low=0.0, high=0.5, prior='uniform', transform='normalize')])
                                  base_estimator: GaussianProcessRegressor(kernel=1**2 * Matern(length_scale=[1, 1, 1, 1, 1], nu=2.5),
                                                                           n_restarts_optimizer=2, noise='gaussian',
                                                                           normalize_y=True, random_state=822569775)
                                         n_calls: 25
                                 n_random_starts: None
                                n_initial_points: 5
                         initial_point_generator: random
                                        acq_func: EI
                                   acq_optimizer: auto
                                              x0: None
                                              y0: None
                                    random_state: RandomState(MT19937)
                                         verbose: False
                                        callback: None
                                        n_points: 10000
                            n_restarts_optimizer: 5
                                              xi: 0.01
                                           kappa: 1.96
                                          n_jobs: 1
                                model_queue_size: None
                                space_constraint: None
               function: base_minimize

# print best result from optimisation and hyperparameter values
print(f'COMPARISON SCORE = {bayesian_result.fun}\n')
print(f'HIDDEN_DIM = {bayesian_result.x[0]}')
print(f'LEARNING_RATE = {bayesian_result.x[1]}')
print(f'NUM_EPOCHS = {bayesian_result.x[2]}')
print(f'NUM_LAYERS = {bayesian_result.x[3]}')
print(f'DROPOUT_RATE = {bayesian_result.x[4]}')

COMPARISON SCORE = 0.6860194283100715

HIDDEN_DIM = 165
LEARNING_RATE = 0.0016249001630866402
NUM_EPOCHS = 1000
NUM_LAYERS = 1
DROPOUT_RATE = 0.0

6c | Visualise Covergence

# values given by the bayesian optimisation function
func_vals = bayesian_result.func_vals

# empty list for conergence plot values
convergence = list()

# 
for i in range(0,len(func_vals)):
    if i == 0:
        convergence.append(func_vals[i])
    elif func_vals[i] < convergence[-1]:
        convergence.append(func_vals[i])
    else:
        convergence.append(convergence[-1])

# plot minimum value of combined score by number of calls
fig, ax = plt.subplots(1, 1, figsize=(5,3.5))

### enumerate number of calls
x_vals = range(1, len(convergence)+1, 1)

### data
ax.plot(x_vals, convergence, linewidth=2, c=WEB_COLOR, zorder=3)
ax.scatter(x_vals, convergence, c=WEB_COLOR, zorder=3)

offset = (convergence[0]-convergence[-1])*0.05

# fill below data
ax.fill_between(
    x=x_vals, y1=convergence, y2=convergence[-1]-offset,
    color=WEB_COLOR, alpha=0.3, ec=None, zorder=2
)

### chart labels
ax.set_xlabel('Number of calls, $n$', fontsize=11)
ax.set_ylabel('Min. Combined Score after $n$ calls', fontsize=11)
#plt.title(
#    f'Epoch vs. Loss \n RNN (LSTM) Model  |  Epochs: {NUM_EPOCHS:.0f}', 
#    fontweight='bold', fontsize=11
#)

# gridlines
ax.spines[['right','top']].set_visible(False)
ax.grid(axis='y', zorder=1)

fig.tight_layout()
plt.show()

6d | Dimensional Reduction of Sampled Points in Search Space

https://www.geeksforgeeks.org/machine-learning/implementing-pca-in-python-with-scikit-learn/

# use PCA to display all sampled points in the search space
x_iters = pd.DataFrame(bayesian_result.x_iters)

scaler = StandardScaler()
x_iters_scaled = scaler.fit_transform(x_iters)

pca = PCA(n_components=2)
x_iters_pca = pca.fit_transform(x_iters_scaled)

print(pca.explained_variance_ratio_)

[0.43779812 0.22531471]

# get position of minimum in 'func_vals' array
minimum_pos = np.where(bayesian_result.func_vals==min(bayesian_result.func_vals))
minimum_pos = minimum_pos[0].tolist()           # convert to list from tuple of arrays
minimum_pos


# plot PCA data
plt.figure(figsize=(5,4))

plt.scatter(
    x_iters_pca[:, 0], x_iters_pca[:, 1], 
    c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40, zorder=3
)

plt.colorbar(label="Combined Score")

#plt.plot(
#    x_iters_pca[:, 0], x_iters_pca[:, 1], c='lightgrey', zorder=2
#)

# plot thicker edge for minima
for m in minimum_pos:
    plt.scatter(
        x_iters_pca[m][0], x_iters_pca[m][1], 
        c='k', edgecolor='k', s=100, zorder=3
    )

plt.xlabel("Principal Component 1")
plt.ylabel("Principal Component 2")
plt.show()

"""
fig, ax = plt.subplots(5,5, figsize=(9,9))

### ax[x,y]

ax[0,4].hist(x_iters[0], color=WEB_COLOR)
ax[1,3].hist(x_iters[1], color=WEB_COLOR)
ax[2,2].hist(x_iters[2], color=WEB_COLOR)
ax[3,1].hist(x_iters[3], color=WEB_COLOR)
ax[4,0].hist(x_iters[4], color=WEB_COLOR)

ax[0,0].scatter(x_iters[4], x_iters[0], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)
ax[0,1].scatter(x_iters[3], x_iters[0], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)
ax[0,2].scatter(x_iters[2], x_iters[0], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)
ax[0,3].scatter(x_iters[1], x_iters[0], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)

ax[1,0].scatter(x_iters[4], x_iters[1], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)
ax[1,1].scatter(x_iters[3], x_iters[1], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)
ax[1,2].scatter(x_iters[2], x_iters[1], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)
ax[1,4].scatter(x_iters[0], x_iters[1], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)

ax[2,0].scatter(x_iters[4], x_iters[2], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)
ax[2,1].scatter(x_iters[3], x_iters[2], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)
ax[2,3].scatter(x_iters[1], x_iters[2], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)
ax[2,4].scatter(x_iters[0], x_iters[2], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)

ax[3,0].scatter(x_iters[4], x_iters[3], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)
ax[3,2].scatter(x_iters[2], x_iters[3], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)
ax[3,3].scatter(x_iters[1], x_iters[3], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)
ax[3,4].scatter(x_iters[0], x_iters[3], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)

ax[4,1].scatter(x_iters[3], x_iters[4], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)
ax[4,2].scatter(x_iters[2], x_iters[4], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)
ax[4,3].scatter(x_iters[1], x_iters[4], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)
ax[4,4].scatter(x_iters[0], x_iters[4], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)

ax[4,0].set_ylabel('DROPOUT_RATE', fontsize=9)
ax[3,0].set_ylabel('NUM_LAYERS', fontsize=9)
ax[2,0].set_ylabel('NUM_EPOCHS', fontsize=9)
ax[1,0].set_ylabel('LEARNING_RATE', fontsize=9)
ax[0,0].set_ylabel('HIDDEN_DIM', fontsize=9)

ax[4,0].set_xlabel('DROPOUT_RATE', fontsize=9)
ax[4,1].set_xlabel('NUM_LAYERS', fontsize=9)
ax[4,2].set_xlabel('NUM_EPOCHS', fontsize=9)
ax[4,3].set_xlabel('LEARNING_RATE', fontsize=9)
ax[4,4].set_xlabel('HIDDEN_DIM', fontsize=9)

remove_y = ([4,0], [4,1], [4,2], [4,3], [4,4])

remove_xy = (
    [0,1], [0,2], [0,3], [0,4], [1,1], [1,2], [1,3], [1,4], 
    [2,1], [2,2], [2,3], [2,4], [3,1], [3,2], [3,3], [3,4],
    [0,0], [1,0], [2,0], [3,0],
)

ax[0,1].set_xlim(-0.3,1.3)
ax[1,1].set_xlim(-0.3,1.3)
ax[2,1].set_xlim(-0.3,1.3)
ax[3,1].set_xlim(-0.3,1.3)
ax[4,1].set_xlim(-0.3,1.3)

ax[3,0].set_ylim(-0.3,1.3)
#ax[3,1].set_ylim(-0.3,1.3)
ax[3,2].set_ylim(-0.3,1.3)
ax[3,3].set_ylim(-0.3,1.3)
ax[3,4].set_ylim(-0.3,1.3)

for vals in remove_y: ax[vals[0],vals[1]].tick_params(labelleft=False)
for vals in remove_xy: ax[vals[0],vals[1]].tick_params(labelleft=False, labelbottom=False)

#fig.tight_layout()
"""

"fig, ax = plt.subplots(5,5, figsize=(9,9))\n\n### ax[x,y]\n\nax[0,4].hist(x_iters[0], color=WEB_COLOR)\nax[1,3].hist(x_iters[1], color=WEB_COLOR)\nax[2,2].hist(x_iters[2], color=WEB_COLOR)\nax[3,1].hist(x_iters[3], color=WEB_COLOR)\nax[4,0].hist(x_iters[4], color=WEB_COLOR)\n\nax[0,0].scatter(x_iters[4], x_iters[0], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)\nax[0,1].scatter(x_iters[3], x_iters[0], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)\nax[0,2].scatter(x_iters[2], x_iters[0], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)\nax[0,3].scatter(x_iters[1], x_iters[0], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)\n\nax[1,0].scatter(x_iters[4], x_iters[1], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)\nax[1,1].scatter(x_iters[3], x_iters[1], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)\nax[1,2].scatter(x_iters[2], x_iters[1], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)\nax[1,4].scatter(x_iters[0], x_iters[1], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)\n\nax[2,0].scatter(x_iters[4], x_iters[2], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)\nax[2,1].scatter(x_iters[3], x_iters[2], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)\nax[2,3].scatter(x_iters[1], x_iters[2], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)\nax[2,4].scatter(x_iters[0], x_iters[2], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)\n\nax[3,0].scatter(x_iters[4], x_iters[3], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)\nax[3,2].scatter(x_iters[2], x_iters[3], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)\nax[3,3].scatter(x_iters[1], x_iters[3], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)\nax[3,4].scatter(x_iters[0], x_iters[3], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)\n\nax[4,1].scatter(x_iters[3], x_iters[4], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)\nax[4,2].scatter(x_iters[2], x_iters[4], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)\nax[4,3].scatter(x_iters[1], x_iters[4], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)\nax[4,4].scatter(x_iters[0], x_iters[4], c=bayesian_result.func_vals, cmap='plasma_r', edgecolor='k', s=40)\n\nax[4,0].set_ylabel('DROPOUT_RATE', fontsize=9)\nax[3,0].set_ylabel('NUM_LAYERS', fontsize=9)\nax[2,0].set_ylabel('NUM_EPOCHS', fontsize=9)\nax[1,0].set_ylabel('LEARNING_RATE', fontsize=9)\nax[0,0].set_ylabel('HIDDEN_DIM', fontsize=9)\n\nax[4,0].set_xlabel('DROPOUT_RATE', fontsize=9)\nax[4,1].set_xlabel('NUM_LAYERS', fontsize=9)\nax[4,2].set_xlabel('NUM_EPOCHS', fontsize=9)\nax[4,3].set_xlabel('LEARNING_RATE', fontsize=9)\nax[4,4].set_xlabel('HIDDEN_DIM', fontsize=9)\n\nremove_y = ([4,0], [4,1], [4,2], [4,3], [4,4])\n\nremove_xy = (\n    [0,1], [0,2], [0,3], [0,4], [1,1], [1,2], [1,3], [1,4], \n    [2,1], [2,2], [2,3], [2,4], [3,1], [3,2], [3,3], [3,4],\n    [0,0], [1,0], [2,0], [3,0],\n)\n\nax[0,1].set_xlim(-0.3,1.3)\nax[1,1].set_xlim(-0.3,1.3)\nax[2,1].set_xlim(-0.3,1.3)\nax[3,1].set_xlim(-0.3,1.3)\nax[4,1].set_xlim(-0.3,1.3)\n\nax[3,0].set_ylim(-0.3,1.3)\n#ax[3,1].set_ylim(-0.3,1.3)\nax[3,2].set_ylim(-0.3,1.3)\nax[3,3].set_ylim(-0.3,1.3)\nax[3,4].set_ylim(-0.3,1.3)\n\nfor vals in remove_y: ax[vals[0],vals[1]].tick_params(labelleft=False)\nfor vals in remove_xy: ax[vals[0],vals[1]].tick_params(labelleft=False, labelbottom=False)\n\n#fig.tight_layout()\n"

7 | Repeat Best Result

7a | Run Experiment

# run experiment and save result
model, rmse_train, rmse_test, loss_by_epoch = run_experiment(
    #train, trainX, trainY_true, test, testX, testY_true,
    hidden_dim = int(bayesian_result.x[0]), 
    learning_rate = bayesian_result.x[1], 
    num_epochs = bayesian_result.x[2], 
    layer_dim = int(bayesian_result.x[3]),
    dropout_rate= bayesian_result.x[4],
)

torch.save(model.state_dict(),'logP_RNN_v3_BO.pt')

Epoch [10/1000], Loss: 1.6112
Epoch [20/1000], Loss: 1.5707
Epoch [30/1000], Loss: 1.5727
Epoch [40/1000], Loss: 1.5625
Epoch [50/1000], Loss: 1.5462
Epoch [60/1000], Loss: 1.4552
Epoch [70/1000], Loss: 1.4742
Epoch [80/1000], Loss: 1.4678
Epoch [90/1000], Loss: 1.5865
Epoch [100/1000], Loss: 1.5608
Epoch [110/1000], Loss: 1.4886
Epoch [120/1000], Loss: 1.3747
Epoch [130/1000], Loss: 1.3767
Epoch [140/1000], Loss: 1.3594
Epoch [150/1000], Loss: 1.3526
Epoch [160/1000], Loss: 1.4640
Epoch [170/1000], Loss: 1.3565
Epoch [180/1000], Loss: 1.3613
Epoch [190/1000], Loss: 1.4018
Epoch [200/1000], Loss: 1.3512
Epoch [210/1000], Loss: 1.3480
Epoch [220/1000], Loss: 1.3365
Epoch [230/1000], Loss: 1.3336
Epoch [240/1000], Loss: 1.3282
Epoch [250/1000], Loss: 1.3317
Epoch [260/1000], Loss: 1.3198
Epoch [270/1000], Loss: 1.2889
Epoch [280/1000], Loss: 1.2367
Epoch [290/1000], Loss: 1.2746
Epoch [300/1000], Loss: 1.3243
Epoch [310/1000], Loss: 1.5609
Epoch [320/1000], Loss: 1.3332
Epoch [330/1000], Loss: 1.2662
Epoch [340/1000], Loss: 1.2827
Epoch [350/1000], Loss: 1.1998
Epoch [360/1000], Loss: 1.1782
Epoch [370/1000], Loss: 1.2369
Epoch [380/1000], Loss: 1.3370
Epoch [390/1000], Loss: 1.1166
Epoch [400/1000], Loss: 1.2180
Epoch [410/1000], Loss: 1.0639
Epoch [420/1000], Loss: 1.1936
Epoch [430/1000], Loss: 1.1108
Epoch [440/1000], Loss: 1.0422
Epoch [450/1000], Loss: 1.0083
Epoch [460/1000], Loss: 0.9708
Epoch [470/1000], Loss: 0.9513
Epoch [480/1000], Loss: 0.9385
Epoch [490/1000], Loss: 0.9043
Epoch [500/1000], Loss: 0.8797
Epoch [510/1000], Loss: 0.8514
Epoch [520/1000], Loss: 1.2544
Epoch [530/1000], Loss: 0.8366
Epoch [540/1000], Loss: 0.7950
Epoch [550/1000], Loss: 0.7344
Epoch [560/1000], Loss: 0.7087
Epoch [570/1000], Loss: 0.6898
Epoch [580/1000], Loss: 0.7346
Epoch [590/1000], Loss: 0.6426
Epoch [600/1000], Loss: 0.7311
Epoch [610/1000], Loss: 0.5981
Epoch [620/1000], Loss: 0.6616
Epoch [630/1000], Loss: 0.6551
Epoch [640/1000], Loss: 0.5664
Epoch [650/1000], Loss: 0.5495
Epoch [660/1000], Loss: 0.5305
Epoch [670/1000], Loss: 0.4935
Epoch [680/1000], Loss: 0.4675
Epoch [690/1000], Loss: 0.4463
Epoch [700/1000], Loss: 0.4311
Epoch [710/1000], Loss: 0.4139
Epoch [720/1000], Loss: 0.4009
Epoch [730/1000], Loss: 0.3908
Epoch [740/1000], Loss: 0.3838
Epoch [750/1000], Loss: 0.3755
Epoch [760/1000], Loss: 0.3857
Epoch [770/1000], Loss: 0.3737
Epoch [780/1000], Loss: 0.4031
Epoch [790/1000], Loss: 0.3829
Epoch [800/1000], Loss: 0.3987
Epoch [810/1000], Loss: 0.4168
Epoch [820/1000], Loss: 0.3916
Epoch [830/1000], Loss: 0.3456
Epoch [840/1000], Loss: 0.3600
Epoch [850/1000], Loss: 0.3691
Epoch [860/1000], Loss: 0.3366
Epoch [870/1000], Loss: 0.3469
Epoch [880/1000], Loss: 0.3673
Epoch [890/1000], Loss: 0.3260
Epoch [900/1000], Loss: 0.3594
Epoch [910/1000], Loss: 0.3102
Epoch [920/1000], Loss: 0.3474
Epoch [930/1000], Loss: 0.3133
Epoch [940/1000], Loss: 0.3148
Epoch [950/1000], Loss: 0.3294
Epoch [960/1000], Loss: 0.2909
Epoch [970/1000], Loss: 0.3156
Epoch [980/1000], Loss: 0.2882
Epoch [990/1000], Loss: 0.2926
Epoch [1000/1000], Loss: 0.2974
Training RMSE: 0.53295
Testing RMSE: 0.60789

# score experiment
score = score_experiment(rmse_train, rmse_test, 1.1)
print(f'COMPARISON SCORE: {score}')

COMPARISON SCORE: 0.6528530597242058

7b | Visualise Loss per Epoch

# plot loss values per epoch
fig, ax = plt.subplots(1, 1, figsize=(5,3.5))

### enumerate epochs
x_vals = range(0, len(loss_by_epoch), 1)

### data
ax.plot(x_vals, loss_by_epoch, linewidth=2, c=WEB_COLOR)

# fill below data
ax.fill_between(
    x=x_vals, y1=loss_by_epoch, y2=0.2,
    color=WEB_COLOR, alpha=0.3, ec=None,
)

### chart labels
ax.set_xlabel('Number of Epochs', fontsize=11)
ax.set_ylabel('MSE Loss', fontsize=11)
#plt.title(
#    f'Epoch vs. Loss \n RNN (LSTM) Model  |  Epochs: {NUM_EPOCHS:.0f}', 
#    fontweight='bold', fontsize=11
#)

# gridlines
ax.spines[['right','top']].set_visible(False)

fig.tight_layout()
plt.show()

7c | Visualise RMSE between Predicted and True Datasets

# plot true vs. predicted log(P) values for training and testing data 
fig, ax = plt.subplots(1, 2, figsize=(8,4))
ax[0].grid(True, color='lightgrey', zorder=1)
ax[1].grid(True, color='lightgrey', zorder=1)

### data
ax[0].scatter(train['ClogP'], train['ClogP (Predicted)'], c='#1E4DAA', s=5, alpha=0.15, label='Training Data', zorder=2)
ax[1].scatter(test['ClogP'], test['ClogP (Predicted)'], c=WEB_COLOR, s=5, alpha=0.15, label='Test Data', zorder=2)

### ideal fit
ax[0].plot(np.linspace(-3.75,6.25,50),np.linspace(-3.75,6.25,50), c='grey', linestyle=":", zorder=3)
ax[1].plot(np.linspace(-3.75,6.25,50),np.linspace(-3.75,6.25,50), c='grey', linestyle=":", zorder=3)

### chart size
ax[0].set_xlim(-3.75, 6.25)
ax[0].set_ylim(-3.75, 6.25)
ax[1].set_xlim(-3.75, 6.25)
ax[1].set_ylim(-3.75, 6.25)

### legends
ax[0].legend(fontsize=10)
ax[1].legend(fontsize=10)

### chart labels
fig.supxlabel('True Clog(P)')
fig.supylabel('Predicted Clog(P)')
fig.suptitle(
    f'HIDDEN_DIM: {HIDDEN_DIM} | LEARNING_RATE: {LEARNING_RATE} | NUM_LAYERS: {NUM_LAYERS} | NUM_EPOCHS: {NUM_EPOCHS} | DROPOUT_RATE: {DROPOUT_RATE}', 
    fontsize=9, fontweight='bold', 
)

### annotate with RMSEs
ax[0].annotate(f'RMSE: {rmse_train:.3f}', xy=(3,-3.5), fontsize=10)
ax[1].annotate(f'RMSE: {rmse_test:.3f}', xy=(3,-3.5), fontsize=10)

fig.tight_layout()
plt.show()

7d | Visualise Molecules with Largest and Smallest Errors

# add absolute error to the 'test' DataFrame
test['Absolute Error'] = abs(test['Error'])

# set cutoff to print all molecules above
ABSOLUTE_ERROR_CUTOFF = 1.8

# create list of SMILES above 
SMILES_list = test[test['Absolute Error']>=ABSOLUTE_ERROR_CUTOFF]['SMILES'].values.tolist()
print(f'Size of SMILES_list: {len(SMILES_list)}')

# get 'test' DataFrame indices corresponding to each SMILES and convert to strings
legends_list = test.index[test['Absolute Error']>=ABSOLUTE_ERROR_CUTOFF].tolist()
legends_list = [str(x) for x in legends_list]

# create Mol objects from SMILES
ms = [Chem.MolFromSmiles(smi) for smi in SMILES_list]   ### 'SMILES_list' sliced to visualise selected values only

# visualise Mol objects
opts = Draw.MolDrawOptions()
opts.drawMolsSameScale = False     # undoes the default behaviour of scaling everything to fit the largest molecule
opts.fixedBondLength = 30          # sets an upper limit for the max bond length
opts.bondLineWidth = 3             # sets a slightly thicker bond width (default is 2)
img = Draw.MolsToGridImage(
    ms, molsPerRow=5, subImgSize=(200,200), legends=legends_list, drawOptions=opts
)
display(img)

Size of SMILES_list: 18

# add absolute error to the 'test' DataFrame
test['Absolute Error'] = abs(test['Error'])

# set cutoff to print all molecules above
ABSOLUTE_ERROR_CUTOFF = 0.01

# create list of SMILES above 
SMILES_list = test[test['Absolute Error']<=ABSOLUTE_ERROR_CUTOFF]['SMILES'].values.tolist()
print(f'Size of SMILES_list: {len(SMILES_list)}')

# get 'test' DataFrame indices corresponding to each SMILES and convert to strings
legends_list = test.index[test['Absolute Error']<=ABSOLUTE_ERROR_CUTOFF].tolist()
legends_list = [str(x) for x in legends_list]

# create Mol objects from SMILES
ms = [Chem.MolFromSmiles(smi) for smi in SMILES_list]   ### 'SMILES_list' sliced to visualise selected values only

# visualise Mol objects
opts = Draw.MolDrawOptions()
opts.drawMolsSameScale = False     # undoes the default behaviour of scaling everything to fit the largest molecule
opts.fixedBondLength = 30          # sets an upper limit for the max bond length
opts.bondLineWidth = 3             # sets a slightly thicker bond width (default is 2)
img = Draw.MolsToGridImage(
    ms, molsPerRow=5, subImgSize=(200,200), legends=legends_list, drawOptions=opts
)
display(img)

Size of SMILES_list: 19

test[test.index==6850]

	SMILES	ClogP	Tokenize	Tokenized Length	ClogP (Predicted)	Error	Absolute Error
6850	C1C[CH+]CC[CH-]1	3.4	[tensor(1), tensor(4), tensor(17), tensor(4), ...	18	1.488867	-1.911133	1.911133