Я пишу более быструю R-CNN сеть с resnet 50 в качестве магистрали. И теперь я продолжаю получать следующую ошибку:
Exception: Incompatible shapes: [1,38,50,9] vs. [1,37,50,9]
Я начинаю с изображения размером 600x800. Так что для модели re snet 50 изображение будет разделено пополам в 4 раза. Что дает мне 600/16=37.5
Теперь я действительно не могу понять, почему он округляется в rpn_model (re snet 50 модель) до 38 и почему он округляется в calc_rpn до 37 Ошибка может быть либо в calc_rpn, либо в модели re snet. Я проверил и перепроверил модель resnet, но не смог найти ничего плохого. Так что я действительно понятия не имею, как это решить. Может быть, будет идея вырезать кусок изображения перед тем, как поместить его в модель resnet, чтобы он получал изюминку внешних изображений. Или, может быть, просто вырезать один пиксель в конце модели. Кто-нибудь знает, решит ли это мою проблему.
Код начала re snet 50
Это первые 4 блока, которые обычно используются между rpn_layer и слой классификатора
def nn_base(input_tensor=None, trainable=False):
input_shape = (None, None, 3)
classes = 1
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
bn_axis = 3
# Zero-Padding
X = ZeroPadding2D((3, 3))(img_input)
# Stage 1
X = Conv2D(64, (7, 7), strides=(2, 2), name='conv1', kernel_initializer=glorot_uniform(seed=0))(X)
X = BatchNormalization(axis=3, name='bn_conv1')(X)
X = Activation('relu')(X)
X = MaxPooling2D((3, 3), strides=(2, 2))(X)
# Stage 2
X = convolutional_block(X, f=3, filters=[64, 64, 256], stage=2, block='a', s=1)
X = identity_block(X, 3, [64, 64, 256], stage=2, block='b')
X = identity_block(X, 3, [64, 64, 256], stage=2, block='c')
### START CODE HERE ###
# Stage 3 (≈4 lines)
X = convolutional_block(X, f = 3, filters = [128, 128, 512], stage = 3, block='a', s = 2)
X = identity_block(X, 3, [128, 128, 512], stage=3, block='b')
X = identity_block(X, 3, [128, 128, 512], stage=3, block='c')
X = identity_block(X, 3, [128, 128, 512], stage=3, block='d')
# Stage 4 (≈6 lines)
X = convolutional_block(X, f = 3, filters = [256, 256, 1024], stage = 4, block='a', s = 2)
X = identity_block(X, 3, [256, 256, 1024], stage=4, block='b')
X = identity_block(X, 3, [256, 256, 1024], stage=4, block='c')
X = identity_block(X, 3, [256, 256, 1024], stage=4, block='d')
X = identity_block(X, 3, [256, 256, 1024], stage=4, block='e')
X = identity_block(X, 3, [256, 256, 1024], stage=4, block='f')
return X
Код rpn_layer
def rpn_layer(base_layers, num_anchors):
x = Conv2D(512, (3, 3), padding='same', activation='relu', kernel_initializer='normal', name='rpn_conv1')(base_layers)
x_class = Conv2D(num_anchors, (1, 1), activation='sigmoid', kernel_initializer='uniform', name='rpn_out_class')(x)
x_regr = Conv2D(num_anchors * 4, (1, 1), activation='linear', kernel_initializer='zero', name='rpn_out_regress')(x)
return [x_class, x_regr, base_layers]
Вывод rpn_model
Мне не хватило символов для поместите все это здесь, так что я просто добавил части, где размер меняется
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
input_13 (InputLayer) (None, 600, 800, 3) 0
__________________________________________________________________________________________________
zero_padding2d_7 (ZeroPadding2D (None, 606, 806, 3) 0 input_13[0][0]
__________________________________________________________________________________________________
conv1 (Conv2D) (None, 300, 400, 64) 9472 zero_padding2d_7[0][0]
__________________________________________________________________________________________________
bn_conv1 (BatchNormalization) (None, 300, 400, 64) 256 conv1[0][0]
__________________________________________________________________________________________________
activation_322 (Activation) (None, 300, 400, 64) 0 bn_conv1[0][0]
__________________________________________________________________________________________________
max_pooling2d_7 (MaxPooling2D) (None, 149, 199, 64) 0 activation_322[0][0]
__________________________________________________________________________________________________
res2a_branch2a (Conv2D) (None, 149, 199, 64) 4160 max_pooling2d_7[0][0]
__________________________________________________________________________________________________
bn2a_branch2a (BatchNormalizati (None, 149, 199, 64) 256 res2a_branch2a[0][0] ______
bn2c_branch2c (BatchNormalizati (None, 149, 199, 256 1024 res2c_branch2c[0][0]
__________________________________________________________________________________________________
add_108 (Add) (None, 149, 199, 256 0 bn2c_branch2c[0][0]
activation_328[0][0]
__________________________________________________________________________________________________
activation_331 (Activation) (None, 149, 199, 256 0 add_108[0][0]
__________________________________________________________________________________________________
res3a_branch2a (Conv2D) (None, 75, 100, 128) 32896 activation_331[0][0]
__________________________________________________________________________________________________
bn3a_branch2a (BatchNormalizati (None, 75, 100, 128) 512 res3a_branch2a[0][0]
________________________________________________________________________________________________
__________________________________________________________________________________________________
add_112 (Add) (None, 75, 100, 512) 0 bn3d_branch2c[0][0]
activation_340[0][0]
__________________________________________________________________________________________________
activation_343 (Activation) (None, 75, 100, 512) 0 add_112[0][0]
__________________________________________________________________________________________________
res4a_branch2a (Conv2D) (None, 38, 50, 256) 131328 activation_343[0][0]
__________________________________________________________________________________________________
bn4a_branch2a (BatchNormalizati (None, 38, 50, 256) 1024 res4a_branch2a[0][0]
__________________________________________________________________________________________________
__________________________________________________________________________________________________
rpn_conv1 (Conv2D) (None, 38, 50, 512) 4719104 activation_361[0][0]
__________________________________________________________________________________________________
rpn_out_class (Conv2D) (None, 38, 50, 9) 4617 rpn_conv1[0][0]
__________________________________________________________________________________________________
rpn_out_regress (Conv2D) (None, 38, 50, 36) 18468 rpn_conv1[0][0]
==================================================================================================
Total params: 13,331,373
Trainable params: 13,300,781
Non-trainable params: 30,592
Код calc_rpn
, который должен дать мне карту возможностей 38x50, но дает мне 37x50 карту объектов. Я предполагаю, что ошибка здесь, но я не могу ее найти.
def calc_rpn(C, img_data, width, height, resized_width, resized_height, img_length_calc_function):
"""(Important part!) Calculate the rpn for all anchors
If feature map has shape 38x50=1900, there are 1900x9=17100 potential anchors
Args:
C: config
img_data: augmented image data
width: original image width (e.g. 600)
height: original image height (e.g. 800)
resized_width: resized image width according to C.im_size (e.g. 300)
resized_height: resized image height according to C.im_size (e.g. 400)
img_length_calc_function: function to calculate final layer's feature map (of base model) size according to input image size
Returns:
y_rpn_cls: list(num_bboxes, y_is_box_valid + y_rpn_overlap)
y_is_box_valid: 0 or 1 (0 means the box is invalid, 1 means the box is valid)
y_rpn_overlap: 0 or 1 (0 means the box is not an object, 1 means the box is an object)
y_rpn_regr: list(num_bboxes, 4*y_rpn_overlap + y_rpn_regr)
y_rpn_regr: x1,y1,x2,y2 bunding boxes coordinates
"""
downscale = float(C.rpn_stride)
anchor_sizes = C.anchor_box_scales # 128, 256, 512
anchor_ratios = C.anchor_box_ratios # 1:1, 1:2*sqrt(2), 2*sqrt(2):1
num_anchors = len(anchor_sizes) * len(anchor_ratios) # 3x3=9
# calculate the output map size based on the network architecture
(output_width, output_height) = img_length_calc_function(resized_width, resized_height)
n_anchratios = len(anchor_ratios) # 3
# initialise empty output objectives
y_rpn_overlap = np.zeros((output_height, output_width, num_anchors))
y_is_box_valid = np.zeros((output_height, output_width, num_anchors))
y_rpn_regr = np.zeros((output_height, output_width, num_anchors * 4))
num_bboxes = len(img_data['bboxes'])
num_anchors_for_bbox = np.zeros(num_bboxes).astype(int)
best_anchor_for_bbox = -1*np.ones((num_bboxes, 4)).astype(int)
best_iou_for_bbox = np.zeros(num_bboxes).astype(np.float32)
best_x_for_bbox = np.zeros((num_bboxes, 4)).astype(int)
best_dx_for_bbox = np.zeros((num_bboxes, 4)).astype(np.float32)
# get the GT box coordinates, and resize to account for image resizing
gta = np.zeros((num_bboxes, 4))
for bbox_num, bbox in enumerate(img_data['bboxes']):
# get the GT box coordinates, and resize to account for image resizing
gta[bbox_num, 0] = bbox['x1'] * (resized_width / float(width))
gta[bbox_num, 1] = bbox['x2'] * (resized_width / float(width))
gta[bbox_num, 2] = bbox['y1'] * (resized_height / float(height))
gta[bbox_num, 3] = bbox['y2'] * (resized_height / float(height))
# rpn ground truth
for anchor_size_idx in range(len(anchor_sizes)):
for anchor_ratio_idx in range(n_anchratios):
anchor_x = anchor_sizes[anchor_size_idx] * anchor_ratios[anchor_ratio_idx][0]
anchor_y = anchor_sizes[anchor_size_idx] * anchor_ratios[anchor_ratio_idx][1]
for ix in range(output_width):
# x-coordinates of the current anchor box
x1_anc = downscale * (ix + 0.5) - anchor_x / 2
x2_anc = downscale * (ix + 0.5) + anchor_x / 2
# ignore boxes that go across image boundaries
if x1_anc < 0 or x2_anc > resized_width:
continue
for jy in range(output_height):
# y-coordinates of the current anchor box
y1_anc = downscale * (jy + 0.5) - anchor_y / 2
y2_anc = downscale * (jy + 0.5) + anchor_y / 2
# ignore boxes that go across image boundaries
if y1_anc < 0 or y2_anc > resized_height:
continue
# bbox_type indicates whether an anchor should be a target
# Initialize with 'negative'
bbox_type = 'neg'
# this is the best IOU for the (x,y) coord and the current anchor
# note that this is different from the best IOU for a GT bbox
best_iou_for_loc = 0.0
for bbox_num in range(num_bboxes):
# get IOU of the current GT box and the current anchor box
curr_iou = iou([gta[bbox_num, 0], gta[bbox_num, 2], gta[bbox_num, 1], gta[bbox_num, 3]], [x1_anc, y1_anc, x2_anc, y2_anc])
# calculate the regression targets if they will be needed
if curr_iou > best_iou_for_bbox[bbox_num] or curr_iou > C.rpn_max_overlap:
cx = (gta[bbox_num, 0] + gta[bbox_num, 1]) / 2.0
cy = (gta[bbox_num, 2] + gta[bbox_num, 3]) / 2.0
cxa = (x1_anc + x2_anc)/2.0
cya = (y1_anc + y2_anc)/2.0
# x,y are the center point of ground-truth bbox
# xa,ya are the center point of anchor bbox (xa=downscale * (ix + 0.5); ya=downscale * (iy+0.5))
# w,h are the width and height of ground-truth bbox
# wa,ha are the width and height of anchor bboxe
# tx = (x - xa) / wa
# ty = (y - ya) / ha
# tw = log(w / wa)
# th = log(h / ha)
tx = (cx - cxa) / (x2_anc - x1_anc)
ty = (cy - cya) / (y2_anc - y1_anc)
tw = np.log((gta[bbox_num, 1] - gta[bbox_num, 0]) / (x2_anc - x1_anc))
th = np.log((gta[bbox_num, 3] - gta[bbox_num, 2]) / (y2_anc - y1_anc))
if img_data['bboxes'][bbox_num]['class'] != 'bg':
# all GT boxes should be mapped to an anchor box, so we keep track of which anchor box was best
if curr_iou > best_iou_for_bbox[bbox_num]:
best_anchor_for_bbox[bbox_num] = [jy, ix, anchor_ratio_idx, anchor_size_idx]
best_iou_for_bbox[bbox_num] = curr_iou
best_x_for_bbox[bbox_num,:] = [x1_anc, x2_anc, y1_anc, y2_anc]
best_dx_for_bbox[bbox_num,:] = [tx, ty, tw, th]
# we set the anchor to positive if the IOU is >0.7 (it does not matter if there was another better box, it just indicates overlap)
if curr_iou > C.rpn_max_overlap:
bbox_type = 'pos'
num_anchors_for_bbox[bbox_num] += 1
# we update the regression layer target if this IOU is the best for the current (x,y) and anchor position
if curr_iou > best_iou_for_loc:
best_iou_for_loc = curr_iou
best_regr = (tx, ty, tw, th)
# if the IOU is >0.3 and <0.7, it is ambiguous and no included in the objective
if C.rpn_min_overlap < curr_iou < C.rpn_max_overlap:
# gray zone between neg and pos
if bbox_type != 'pos':
bbox_type = 'neutral'
# turn on or off outputs depending on IOUs
if bbox_type == 'neg':
y_is_box_valid[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 1
y_rpn_overlap[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 0
elif bbox_type == 'neutral':
y_is_box_valid[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 0
y_rpn_overlap[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 0
elif bbox_type == 'pos':
y_is_box_valid[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 1
y_rpn_overlap[jy, ix, anchor_ratio_idx + n_anchratios * anchor_size_idx] = 1
start = 4 * (anchor_ratio_idx + n_anchratios * anchor_size_idx)
y_rpn_regr[jy, ix, start:start+4] = best_regr
# we ensure that every bbox has at least one positive RPN region
for idx in range(num_anchors_for_bbox.shape[0]):
if num_anchors_for_bbox[idx] == 0:
# no box with an IOU greater than zero ...
if best_anchor_for_bbox[idx, 0] == -1:
continue
y_is_box_valid[
best_anchor_for_bbox[idx,0], best_anchor_for_bbox[idx,1], best_anchor_for_bbox[idx,2] + n_anchratios *
best_anchor_for_bbox[idx,3]] = 1
y_rpn_overlap[
best_anchor_for_bbox[idx,0], best_anchor_for_bbox[idx,1], best_anchor_for_bbox[idx,2] + n_anchratios *
best_anchor_for_bbox[idx,3]] = 1
start = 4 * (best_anchor_for_bbox[idx,2] + n_anchratios * best_anchor_for_bbox[idx,3])
y_rpn_regr[
best_anchor_for_bbox[idx,0], best_anchor_for_bbox[idx,1], start:start+4] = best_dx_for_bbox[idx, :]
y_rpn_overlap = np.transpose(y_rpn_overlap, (2, 0, 1))
y_rpn_overlap = np.expand_dims(y_rpn_overlap, axis=0)
y_is_box_valid = np.transpose(y_is_box_valid, (2, 0, 1))
y_is_box_valid = np.expand_dims(y_is_box_valid, axis=0)
y_rpn_regr = np.transpose(y_rpn_regr, (2, 0, 1))
y_rpn_regr = np.expand_dims(y_rpn_regr, axis=0)
pos_locs = np.where(np.logical_and(y_rpn_overlap[0, :, :, :] == 1, y_is_box_valid[0, :, :, :] == 1))
neg_locs = np.where(np.logical_and(y_rpn_overlap[0, :, :, :] == 0, y_is_box_valid[0, :, :, :] == 1))
num_pos = len(pos_locs[0])
# one issue is that the RPN has many more negative than positive regions, so we turn off some of the negative
# regions. We also limit it to 256 regions.
num_regions = 256
if len(pos_locs[0]) > num_regions/2:
val_locs = random.sample(range(len(pos_locs[0])), len(pos_locs[0]) - num_regions/2)
y_is_box_valid[0, pos_locs[0][val_locs], pos_locs[1][val_locs], pos_locs[2][val_locs]] = 0
num_pos = num_regions/2
if len(neg_locs[0]) + num_pos > num_regions:
val_locs = random.sample(range(len(neg_locs[0])), len(neg_locs[0]) - num_pos)
y_is_box_valid[0, neg_locs[0][val_locs], neg_locs[1][val_locs], neg_locs[2][val_locs]] = 0
y_rpn_cls = np.concatenate([y_is_box_valid, y_rpn_overlap], axis=1)
y_rpn_regr = np.concatenate([np.repeat(y_rpn_overlap, 4, axis=1), y_rpn_regr], axis=1)
return np.copy(y_rpn_cls), np.copy(y_rpn_regr), num_pos