EfficientNet B0 Model does not predict right class while testing

Question

I trained an effectivenetB0 model using a training dataset of about 400 images of army personals, the general public, and army vehicles specifically tanks of each category. After training it on the training set I get an accuracy of about 98% and on the testing set, I get pretty good accuracy.

The confusion matrix on the testing set is: [Army, general, vehicle]

[[52 35 7]
[ 5 93 15]
[ 5 5 86]]

But when I try to predict a separate image it does not predict it well. I tried different solutions from StackOverflow but could not make anything work.

import numpy as np
import tensorflow as tf

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import os

dataset_path = os.listdir('combined')

print (dataset_path) #what kinds of classes are in this dataset

print("Types of classes labels found: ", len(dataset_path))

class_labels = []

for item in dataset_path:
    all_classes = os.listdir('combined' + '/' +item)
    for room in all_classes:
        class_labels.append((item, str('dataset_path' + '/' +item) + '/' + room))


print(class_labels)

df = pd.DataFrame(data=class_labels, columns=['Labels', 'image'])
print(df.head())
print(df.tail())

print("Total number of images in the dataset: ", len(df))

label_count = df['Labels'].value_counts()
print(label_count)

import cv2
path = 'combined/'
dataset_path = os.listdir('combined')

im_size = 224

images = []
labels = []

for i in dataset_path:
    data_path = path + str(I)
    filenames = [i for i in os.listdir(data_path) ]
    print(data_path)
    for f in filenames:
        img = cv2.imread(data_path + '/' + f)
        img = cv2.resize(img, (im_size, im_size))
        images.append(img)
        labels.append(i)


images = np.array(images)
print(images.shape)
images = images.astype('float32') / 255.0

images = preprocess_input(images)
print(images.shape)

from sklearn.preprocessing import LabelEncoder , OneHotEncoder
y=df['Labels'].values
print(y)
print(len(y))
print(list(set(y)))
y_labelencoder = LabelEncoder ()
y = y_labelencoder.fit_transform (y)
print(y)
print(list(set(y)))

y=y.reshape(-1,1)

from sklearn.compose import ColumnTransformer
ct = ColumnTransformer([('my_ohe', OneHotEncoder(), [0])], remainder='passthrough')
Y = ct.fit_transform(y) #.toarray()
print(Y[:5])
print(Y[35:])

from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split

images, Y = shuffle(images, Y, random_state=1)

train_x, test_x, train_y, test_y = train_test_split(images, Y, test_size=0.2, random_state=0)

from tensorflow.keras import layers
from tensorflow.keras.applications import EfficientNetB0

NUM_CLASSES = 3
IMG_SIZE = 224
size = (IMG_SIZE, IMG_SIZE)

inputs = layers.Input(shape=(IMG_SIZE, IMG_SIZE, 3))

outputs = EfficientNetB0(include_top=True, weights=None, classes=NUM_CLASSES)(inputs)

model = tf.keras.Model(inputs, outputs)

model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"] )

model.summary()

hist = model.fit(train_x, train_y, epochs=30, verbose=2)




preds = model.evaluate(test_x, test_y)
print ("Loss = " + str(preds[0]))
print ("Test Accuracy = " + str(preds[1]))




from matplotlib.pyplot import imread
from matplotlib.pyplot import imshow
from tensorflow.keras.preprocessing import image
from tensorflow.keras.applications.imagenet_utils import decode_predictions
from tensorflow.keras.applications.imagenet_utils import preprocess_input

img_path = '/content/combined/vehicles/al_khalid_l1.jpg'

img = cv2.imread(img_path)
img = cv2.resize(img, (224, 224))
img_array = image.img_to_array(img)

img_batch = np.expand_dims(img_array, axis=0)

img_preprocessed = preprocess_input(img_batch)

my_image = imread(img_path)
imshow(my_image)

preds=model.predict(img_preprocessed)

joined = preds[0]
joined[np.where(joined==np.max(joined)) ] = 1
joined[np.where(joined!=np.max(joined)) ] = 0
joined = list(map(int,joined))
print(joined)
label = ['vehicle', 'army', 'general']
zipped = zip(joined, label)
for i in list(zipped):
    if i[0] == 1:
        print(i[1])

Predicting results are: [0, 1, 0] army