Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added support for Evaluation using bigger Batch Size and Multiple GPUs #688

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Added support for Evaluation using bigger Batch Size and Multiple GPUs #688

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5d7baee
Support multiple GPUs & larger batch size
siddharthgorayan Aug 27, 2021
ae5ecfb
added preprocess_eval for multi gpu
siddharthgorayan Aug 27, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
155 changes: 105 additions & 50 deletions coco_eval.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@
put weights here /path/to/your/weights/*.pth
change compound_coef

ADDITIONAL INSTRUCTIONS AFTER INTRODUCING BATCH WISE AND MULTI GPU EVALUATION:
change the batch_size
change the number of GPUs in yml file

"""

import json
Expand All @@ -16,13 +20,17 @@
import argparse
import torch
import yaml
import math
from tqdm import tqdm
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval

from backbone import EfficientDetBackbone
from efficientdet.utils import BBoxTransform, ClipBoxes
from utils.utils import preprocess, invert_affine, postprocess, boolean_string
from utils.utils import preprocess_eval, invert_affine, postprocess, boolean_string

from utils.utils import replace_w_sync_bn, CustomDataParallel
from utils.sync_batchnorm import patch_replication_callback

ap = argparse.ArgumentParser()
ap.add_argument('-p', '--project', type=str, default='coco', help='project file that contains parameters')
Expand All @@ -33,8 +41,16 @@
ap.add_argument('--device', type=int, default=0)
ap.add_argument('--float16', type=boolean_string, default=False)
ap.add_argument('--override', type=boolean_string, default=True, help='override previous bbox results file if exists')
ap.add_argument('--batch_size', type=int, default=12, help='The number of images per batch among all devices')
args = ap.parse_args()

class Params:
def __init__(self, project_file):
self.params = yaml.safe_load(open(project_file).read())

def __getattr__(self, item):
return self.params.get(item, None)

compound_coef = args.compound_coef
nms_threshold = args.nms_threshold
use_cuda = args.cuda
Expand All @@ -46,72 +62,103 @@

print(f'running coco-style evaluation on project {project_name}, weights {weights_path}...')

params = yaml.safe_load(open(f'projects/{project_name}.yml'))
obj_list = params['obj_list']
params = Params(f'projects/{args.project}.yml')
obj_list = params.obj_list

input_sizes = [512, 640, 768, 896, 1024, 1280, 1280, 1536, 1536]


def evaluate_coco(img_path, set_name, image_ids, coco, model, threshold=0.05):

if params.num_gpus == 0:
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'

if torch.cuda.is_available():
torch.cuda.manual_seed(42)
else:
torch.manual_seed(42)

results = []

no_of_batches = math.ceil(len(image_ids)/args.batch_size)
batches = no_of_batches*[0]

regressBoxes = BBoxTransform()
clipBoxes = ClipBoxes()

for image_id in tqdm(image_ids):
image_info = coco.loadImgs(image_id)[0]
image_path = img_path + image_info['file_name']
for batch in tqdm(range(no_of_batches)):
batches[batch] = []
start = batch*args.batch_size
if batch == no_of_batches-1:
end = len(image_ids)
else:
end = (batch*args.batch_size) + args.batch_size
for image_id in range(start, end):
image_info = coco.loadImgs(image_id)[0]
image_path = img_path + image_info['file_name']
batches[batch].append(image_path)

ori_imgs, framed_imgs, framed_metas = preprocess(image_path, max_size=input_sizes[compound_coef], mean=params['mean'], std=params['std'])
x = torch.from_numpy(framed_imgs[0])
ori_imgs, framed_imgs, framed_metas = preprocess_eval(batches[batch], max_size=input_sizes[compound_coef], mean=params.mean, std=params.std)
x = torch.tensor(framed_imgs[0:args.batch_size])

if use_cuda:
if params.num_gpus == 1:
x = x.cuda(gpu)
if params.num_gpus > 1:
x = CustomDataParallel(x, params.num_gpus)
if use_sync_bn:
patch_replication_callback(x)

if use_float16:
x = x.half()
else:
x = x.float()
else:
x = x.float()

x = x.unsqueeze(0).permute(0, 3, 1, 2)
x = x.permute(0, 3, 1, 2)
features, regression, classification, anchors = model(x)

preds = postprocess(x,
anchors, regression, classification,
regressBoxes, clipBoxes,
threshold, nms_threshold)

if not preds:
continue

preds = invert_affine(framed_metas, preds)[0]

scores = preds['scores']
class_ids = preds['class_ids']
rois = preds['rois']

if rois.shape[0] > 0:
# x1,y1,x2,y2 -> x1,y1,w,h
rois[:, 2] -= rois[:, 0]
rois[:, 3] -= rois[:, 1]

bbox_score = scores

for roi_id in range(rois.shape[0]):
score = float(bbox_score[roi_id])
label = int(class_ids[roi_id])
box = rois[roi_id, :]

image_result = {
'image_id': image_id,
'category_id': label + 1,
'score': float(score),
'bbox': box.tolist(),
}

results.append(image_result)

preds = invert_affine(framed_metas, preds)[0:args.batch_size]

for image in range(args.batch_size):
image_id = (batch*args.batch_size) + image

if batch == no_of_batches-1 and image_id == len(image_ids):
break

scores = preds[image]['scores']
class_ids = preds[image]['class_ids']
rois = preds[image]['rois']

if rois.shape[0] > 0:
# x1,y1,x2,y2 -> x1,y1,w,h
rois[:, 2] -= rois[:, 0]
rois[:, 3] -= rois[:, 1]

bbox_score = scores

for roi_id in range(rois.shape[0]):
score = float(bbox_score[roi_id])
label = int(class_ids[roi_id])
box = rois[roi_id, :]

image_result = {
'image_id': image_id,
'category_id': label + 1,
'score': float(score),
'bbox': box.tolist(),
}

results.append(image_result)

if not len(results):
raise Exception('the model does not provide any valid output, check model architecture and the data input')

Expand All @@ -127,7 +174,7 @@ def _eval(coco_gt, image_ids, pred_json_path):
coco_pred = coco_gt.loadRes(pred_json_path)

# run COCO evaluation
print('BBox')
print('BBox \n\nOverall:\n')
coco_eval = COCOeval(coco_gt, coco_pred, 'bbox')
coco_eval.params.imgIds = image_ids
coco_eval.evaluate()
Expand All @@ -136,25 +183,33 @@ def _eval(coco_gt, image_ids, pred_json_path):


if __name__ == '__main__':
SET_NAME = params['val_set']
VAL_GT = f'datasets/{params["project_name"]}/annotations/instances_{SET_NAME}.json'
VAL_IMGS = f'datasets/{params["project_name"]}/{SET_NAME}/'
SET_NAME = params.val_set
VAL_GT = f'datasets/{params.project_name}/annotations/instances_{SET_NAME}.json'
VAL_IMGS = f'datasets/{params.project_name}/{SET_NAME}/'
MAX_IMAGES = 10000
coco_gt = COCO(VAL_GT)
image_ids = coco_gt.getImgIds()[:MAX_IMAGES]

if override_prev_results or not os.path.exists(f'{SET_NAME}_bbox_results.json'):
model = EfficientDetBackbone(compound_coef=compound_coef, num_classes=len(obj_list),
ratios=eval(params['anchors_ratios']), scales=eval(params['anchors_scales']))
ratios=eval(params.anchors_ratios), scales=eval(params.anchors_scales))
model.load_state_dict(torch.load(weights_path, map_location=torch.device('cpu')))
model.requires_grad_(False)
model.eval()

if use_cuda:
model.cuda(gpu)
if params.num_gpus > 1 and args.batch_size // params.num_gpus < 4:
model.apply(replace_w_sync_bn)
use_sync_bn = True
else:
use_sync_bn = False

if use_float16:
model.half()
if params.num_gpus > 0:
model = model.cuda(gpu)
if params.num_gpus > 1:
model = CustomDataParallel(model, params.num_gpus)
if use_sync_bn:
patch_replication_callback(model)

model.requires_grad_(False)
model.eval()

evaluate_coco(VAL_IMGS, SET_NAME, image_ids, coco_gt, model)

Expand Down
10 changes: 10 additions & 0 deletions utils/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,16 @@ def preprocess(*image_path, max_size=512, mean=(0.485, 0.456, 0.406), std=(0.229

return ori_imgs, framed_imgs, framed_metas

def preprocess_eval(image_path, max_size=512, mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)):
ori_imgs = [cv2.imread(img_path) for img_path in image_path]
normalized_imgs = [(img[..., ::-1] / 255 - mean) / std for img in ori_imgs]
imgs_meta = [aspectaware_resize_padding(img, max_size, max_size,
means=None) for img in normalized_imgs]
framed_imgs = [img_meta[0] for img_meta in imgs_meta]
framed_metas = [img_meta[1:] for img_meta in imgs_meta]

return ori_imgs, framed_imgs, framed_metas


def preprocess_video(*frame_from_video, max_size=512, mean=(0.406, 0.456, 0.485), std=(0.225, 0.224, 0.229)):
ori_imgs = frame_from_video
Expand Down