I am trying to train a YoloV4 model with my own data. I get the following error when i try to fit the model with a custom loss function:
ValueError: Cannot reshape a tensor with 2945760 elements to shape [2,76,76,3,36] (1247616 elements)
the parameters are:
NETWORK_W = 608
NETWORK_H = 608
NB_BOX = 3
NB_CLASS = len(labels)
OBJ_THRESHOLD = 0.3
NMS_THRESHOLD = 0.3
grids = [(76,76), (38,38), (19,19)]
anchors = [ [12, 16, 19, 36, 40, 28],[36, 75, 76, 55, 72, 146],[142, 110, 192, 243, 459, 401]]
scales_x_y = [1.2, 1.1, 1.05]
NO_OBJECT_SCALE = 1.0
OBJECT_SCALE = 5.0
COORD_SCALE = 1.0
CLASS_SCALE = 1.0
BATCH_SIZE = 2
TRUE_BOX_BUFFER = 50
the loss function looks like this:
def custom_loss(y_true, y_pred):
grid_h, grid_w = y_pred.shape[1:3]
if grid_h == grids[0][0]:
anchor = anchors[0]
elif grid_h == grids[1][0]:
anchor = anchors[1]
else:
anchor = anchors[2]
print("anchor",anchor)
mask_shape = tf.shape(y_true)[:4]
cell_x = tf.cast((tf.reshape(tf.tile(tf.range(grid_w), [grid_h]), (1, grid_h, grid_w, 1, 1))),dtype=tf.float32)
cell_y = tf.transpose(cell_x, (0,2,1,3,4))
cell_grid = tf.tile(tf.concat([cell_x,cell_y], -1), [BATCH_SIZE, 1, 1, NB_BOX, 1])
###### prediction
y_pred = tf.reshape(y_pred, (BATCH_SIZE, grid_h, grid_w, NB_BOX, NB_CLASS+5))
print ("prediction", y_pred.shape)
### adjust x and y
pred_box_xy = tf.sigmoid(y_pred[..., :2]) # x, y)
pred_box_xy = pred_box_xy + cell_grid
### adjust w and h
pred_box_wh = tf.exp(y_pred[..., 2:4]) * np.reshape(anchor, [1,1,1,NB_BOX,2]) / np.full((1,1,1,NB_BOX, 2), [NETWORK_W, NETWORK_H])
### adjust objectness
pred_box_obj = tf.sigmoid(y_pred[..., 4])
### adjust class probabilities
pred_box_class = tf.sigmoid(y_pred[..., 5:])
###### true
y_true = tf.reshape(y_true, (BATCH_SIZE, grid_h, grid_w, NB_BOX, NB_CLASS+5))
print ("true", y_true.shape)
### adjust x and y
true_box_xy = y_true[..., :2] # x, y
### adjust w and h
true_box_wh = y_true[..., 2:4]
### adjust objectness
true_wh_half = true_box_wh / 2.
true_mins = true_box_xy - true_wh_half
true_maxes = true_box_xy + true_wh_half
pred_wh_half = pred_box_wh / 2.
pred_mins = pred_box_xy - pred_wh_half
pred_maxes = pred_box_xy + pred_wh_half
intersect_mins = tf.maximum(pred_mins, true_mins)
intersect_maxes = tf.minimum(pred_maxes, true_maxes)
intersect_wh = tf.maximum(intersect_maxes - intersect_mins, 0.)
intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]
true_areas = true_box_wh[..., 0] * true_box_wh[..., 1]
pred_areas = pred_box_wh[..., 0] * pred_box_wh[..., 1]
union_areas = pred_areas + true_areas - intersect_areas + 1e-10
iou_scores = tf.truediv(intersect_areas, union_areas)
true_box_obj = iou_scores * y_true[..., 4]
### adjust class probabilities
true_box_class = tf.argmax(y_true[..., 5:], -1)
###### coefficients
### coordinate mask: simply the position of the ground truth boxes (the predictors)
### is 1 when there is an object in the cell i, else 0.
coord_mask = tf.zeros(mask_shape)
coord_mask = tf.expand_dims(y_true[..., 4], axis=-1) * COORD_SCALE
### objectness mask: penelize predictors + penalize boxes with low IOU
# penalize the confidence of the boxes, which have IOU with some ground truth box < 0.6
for i in range(BATCH_SIZE):
bd = y_true[i,:,:,:,:4]
nozero = tf.not_equal(bd, tf.zeros((grid_h, grid_w, NB_BOX, 4)))
bdd = tf.boolean_mask(bd, nozero)
s=tf.squeeze(tf.size(bdd)//4)
c= tf.zeros((50-s,4))
bdd=tf.reshape(bdd, (s,4))
bdd = tf.concat([bdd,c],axis=0)
bdd = tf.expand_dims(bdd,0)
bdd = tf.expand_dims(bdd,0)
bdd = tf.expand_dims(bdd,0)
bdd = tf.expand_dims(bdd,0)
if (i==0):
true_boxes =bdd
else:
true_boxes = tf.concat([true_boxes,bdd], axis=0)
true_xy = true_boxes[..., 0:2]
true_wh = true_boxes[..., 2:4]
true_wh_half = true_wh / 2.
true_mins = true_xy - true_wh_half
true_maxes = true_xy + true_wh_half
pred_xy = tf.expand_dims(pred_box_xy, 4)
pred_wh = tf.expand_dims(pred_box_wh, 4)
pred_wh_half = pred_wh / 2.
pred_mins = pred_xy - pred_wh_half
pred_maxes = pred_xy + pred_wh_half
intersect_mins = tf.maximum(pred_mins, true_mins)
intersect_maxes = tf.minimum(pred_maxes, true_maxes)
intersect_wh = tf.maximum(intersect_maxes - intersect_mins, 0.)
intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]
true_areas = true_wh[..., 0] * true_wh[..., 1]
pred_areas = pred_wh[..., 0] * pred_wh[..., 1]
union_areas = pred_areas + true_areas - intersect_areas
iou_scores = tf.truediv(intersect_areas, union_areas)
best_ious = tf.reduce_max(iou_scores, axis=4)
obj_mask = tf.zeros(mask_shape)
obj_mask = tf.cast((best_ious < 0.6),dtype=tf.float32) * (1 - y_true[..., 4]) * NO_OBJECT_SCALE
obj_mask = obj_mask + y_true[..., 4] * OBJECT_SCALE
### class mask: simply the position of the ground truth boxes (the predictors)
### is 1 when there is a particular class is predicted, else 0.
class_mask = tf.zeros(mask_shape)
class_weights = np.ones(NB_CLASS, dtype='float32')
class_mask = y_true[..., 4] * tf.gather(class_weights, true_box_class) * CLASS_SCALE
nb_coord_box = tf.reduce_sum(tf.cast((coord_mask > 0.0),dtype=tf.float32))
nb_obj_box = tf.reduce_sum(tf.cast((obj_mask > 0.0),dtype=tf.float32))
nb_class_box = tf.reduce_sum(tf.cast((class_mask > 0.0),dtype=tf.float32))
### loss
loss_xy = tf.reduce_sum(coord_mask * tf.square(true_box_xy - pred_box_xy)) / (nb_coord_box + 1e-6) / 2.
loss_wh = tf.reduce_sum(coord_mask * tf.square(tf.sqrt(tf.abs(true_box_wh)) - tf.sqrt(tf.abs(pred_box_wh)))) / (nb_coord_box + 1e-6) / 2.
loss_obj = tf.reduce_sum(obj_mask * tf.square(true_box_obj-pred_box_obj)) / (nb_obj_box + 1e-6) / 2.
loss_class = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=true_box_class, logits=pred_box_class)
loss_class = tf.reduce_sum(class_mask * loss_class) / (nb_class_box + 1e-6)
loss = loss_xy + loss_wh + loss_obj + loss_class
print()
return loss
I changed the
y_pred = tf.reshape(y_pred, (BATCH_SIZE, grid_h, grid_w, NB_BOX, NB_CLASS+5))
to:
y_pred = tf.reshape(y_pred, (BATCH_SIZE, grid_h, grid_w, NB_BOX, -1))
But i get the following error :
ValueError: Dimensions must be equal, but are 255 and 108 for '{{node SquaredDifference}} = SquaredDifference[T=DT_FLOAT](Yolo_v4/BN_138/FusedBatchNormV3, IteratorGetNext:1)' with input shapes: [2,76,76,255], [2,76,76,108].
Now i'm not sure where it goes wrong, my data should be fine and i believe the the problem is in the custom loss, but i dont know where.
Any tips would be appreciated!
