# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import copy import math import random import numpy as np from copy import deepcopy from typing import List, Tuple from collections import defaultdict from .chip_box_utils import nms, transform_chip_boxes2image_boxes from .chip_box_utils import find_chips_to_cover_overlaped_boxes from .chip_box_utils import transform_chip_box from .chip_box_utils import intersection_over_box class AnnoCropper(object): def __init__(self, image_target_sizes: List[int], valid_box_ratio_ranges: List[List[float]], chip_target_size: int, chip_target_stride: int, use_neg_chip: bool=False, max_neg_num_per_im: int=8, max_per_img: int=-1, nms_thresh: int=0.5): """ Generate chips by chip_target_size and chip_target_stride. These two parameters just like kernel_size and stride in cnn. Each image has its raw size. After resizing, then get its target size. The resizing scale = target_size / raw_size. So are chips of the image. box_ratio = box_raw_size / image_raw_size = box_target_size / image_target_size The 'size' above mentioned is the size of long-side of image, box or chip. :param image_target_sizes: [2000, 1000] :param valid_box_ratio_ranges: [[-1, 0.1],[0.08, -1]] :param chip_target_size: 500 :param chip_target_stride: 200 """ self.target_sizes = image_target_sizes self.valid_box_ratio_ranges = valid_box_ratio_ranges assert len(self.target_sizes) == len(self.valid_box_ratio_ranges) self.scale_num = len(self.target_sizes) self.chip_target_size = chip_target_size # is target size self.chip_target_stride = chip_target_stride # is target stride self.use_neg_chip = use_neg_chip self.max_neg_num_per_im = max_neg_num_per_im self.max_per_img = max_per_img self.nms_thresh = nms_thresh def crop_anno_records(self, records: List[dict]): """ The main logic: # foreach record(image): # foreach scale: # 1 generate chips by chip size and stride for each scale # 2 get pos chips # - validate boxes: current scale; h,w >= 1 # - find pos chips greedily by valid gt boxes in each scale # - for every valid gt box, find its corresponding pos chips in each scale # 3 get neg chips # - If given proposals, find neg boxes in them which are not in pos chips # - If got neg boxes in last step, we find neg chips and assign neg boxes to neg chips such as 2. # 4 sample neg chips if too much each image # transform this image-scale annotations to chips(pos chips&neg chips) annotations :param records, standard coco_record but with extra key `proposals`(Px4), which are predicted by stage1 model and maybe have neg boxes in them. :return: new_records, list of dict like { 'im_file': 'fake_image1.jpg', 'im_id': np.array([1]), # new _global_chip_id as im_id 'h': h, # chip height 'w': w, # chip width 'is_crowd': is_crowd, # Nx1 -> Mx1 'gt_class': gt_class, # Nx1 -> Mx1 'gt_bbox': gt_bbox, # Nx4 -> Mx4, 4 represents [x1,y1,x2,y2] 'gt_poly': gt_poly, # [None]xN -> [None]xM 'chip': [x1, y1, x2, y2] # added } Attention: ------------------------------>x | | (x1,y1)------ | | | | | | | | | | | | | | | | ---------- | (x2,y2) | ↓ y If we use [x1, y1, x2, y2] to represent boxes or chips, (x1,y1) is the left-top point which is in the box, but (x2,y2) is the right-bottom point which is not in the box. So x1 in [0, w-1], x2 in [1, w], y1 in [0, h-1], y2 in [1,h]. And you can use x2-x1 to get width, and you can use image[y1:y2, x1:x2] to get the box area. """ self.chip_records = [] self._global_chip_id = 1 for r in records: self._cur_im_pos_chips = [ ] # element: (chip, boxes_idx), chip is [x1, y1, x2, y2], boxes_ids is List[int] self._cur_im_neg_chips = [] # element: (chip, neg_box_num) for scale_i in range(self.scale_num): self._get_current_scale_parameters(scale_i, r) # Cx4 chips = self._create_chips(r['h'], r['w'], self._cur_scale) # # dict: chipid->[box_id, ...] pos_chip2boxes_idx = self._get_valid_boxes_and_pos_chips( r['gt_bbox'], chips) # dict: chipid->neg_box_num neg_chip2box_num = self._get_neg_boxes_and_chips( chips, list(pos_chip2boxes_idx.keys()), r.get('proposals', None)) self._add_to_cur_im_chips(chips, pos_chip2boxes_idx, neg_chip2box_num) cur_image_records = self._trans_all_chips2annotations(r) self.chip_records.extend(cur_image_records) return self.chip_records def _add_to_cur_im_chips(self, chips, pos_chip2boxes_idx, neg_chip2box_num): for pos_chipid, boxes_idx in pos_chip2boxes_idx.items(): chip = np.array(chips[pos_chipid]) # copy chips slice self._cur_im_pos_chips.append((chip, boxes_idx)) if neg_chip2box_num is None: return for neg_chipid, neg_box_num in neg_chip2box_num.items(): chip = np.array(chips[neg_chipid]) self._cur_im_neg_chips.append((chip, neg_box_num)) def _trans_all_chips2annotations(self, r): gt_bbox = r['gt_bbox'] im_file = r['im_file'] is_crowd = r['is_crowd'] gt_class = r['gt_class'] # gt_poly = r['gt_poly'] # [None]xN # remaining keys: im_id, h, w chip_records = self._trans_pos_chips2annotations(im_file, gt_bbox, is_crowd, gt_class) if not self.use_neg_chip: return chip_records sampled_neg_chips = self._sample_neg_chips() neg_chip_records = self._trans_neg_chips2annotations(im_file, sampled_neg_chips) chip_records.extend(neg_chip_records) return chip_records def _trans_pos_chips2annotations(self, im_file, gt_bbox, is_crowd, gt_class): chip_records = [] for chip, boxes_idx in self._cur_im_pos_chips: chip_bbox, final_boxes_idx = transform_chip_box(gt_bbox, boxes_idx, chip) x1, y1, x2, y2 = chip chip_h = y2 - y1 chip_w = x2 - x1 rec = { 'im_file': im_file, 'im_id': np.array([self._global_chip_id]), 'h': chip_h, 'w': chip_w, 'gt_bbox': chip_bbox, 'is_crowd': is_crowd[final_boxes_idx].copy(), 'gt_class': gt_class[final_boxes_idx].copy(), # 'gt_poly': [None] * len(final_boxes_idx), 'chip': chip } self._global_chip_id += 1 chip_records.append(rec) return chip_records def _sample_neg_chips(self): pos_num = len(self._cur_im_pos_chips) neg_num = len(self._cur_im_neg_chips) sample_num = min(pos_num + 2, self.max_neg_num_per_im) assert sample_num >= 1 if neg_num <= sample_num: return self._cur_im_neg_chips candidate_num = int(sample_num * 1.5) candidate_neg_chips = sorted( self._cur_im_neg_chips, key=lambda x: -x[1])[:candidate_num] random.shuffle(candidate_neg_chips) sampled_neg_chips = candidate_neg_chips[:sample_num] return sampled_neg_chips def _trans_neg_chips2annotations(self, im_file: str, sampled_neg_chips: List[Tuple]): chip_records = [] for chip, neg_box_num in sampled_neg_chips: x1, y1, x2, y2 = chip chip_h = y2 - y1 chip_w = x2 - x1 rec = { 'im_file': im_file, 'im_id': np.array([self._global_chip_id]), 'h': chip_h, 'w': chip_w, 'gt_bbox': np.zeros( (0, 4), dtype=np.float32), 'is_crowd': np.zeros( (0, 1), dtype=np.int32), 'gt_class': np.zeros( (0, 1), dtype=np.int32), # 'gt_poly': [], 'chip': chip } self._global_chip_id += 1 chip_records.append(rec) return chip_records def _get_current_scale_parameters(self, scale_i, r): im_size = max(r['h'], r['w']) im_target_size = self.target_sizes[scale_i] self._cur_im_size, self._cur_im_target_size = im_size, im_target_size self._cur_scale = self._get_current_scale(im_target_size, im_size) self._cur_valid_ratio_range = self.valid_box_ratio_ranges[scale_i] def _get_current_scale(self, im_target_size, im_size): return im_target_size / im_size def _create_chips(self, h: int, w: int, scale: float): """ Generate chips by chip_target_size and chip_target_stride. These two parameters just like kernel_size and stride in cnn. :return: chips, Cx4, xy in raw size dimension """ chip_size = self.chip_target_size # omit target for simplicity stride = self.chip_target_stride width = int(scale * w) height = int(scale * h) min_chip_location_diff = 20 # in target size assert chip_size >= stride chip_overlap = chip_size - stride if (width - chip_overlap ) % stride > min_chip_location_diff: # 不能被stride整除的部分比较大，则保留 w_steps = max(1, int(math.ceil((width - chip_overlap) / stride))) else: # 不能被stride整除的部分比较小，则丢弃 w_steps = max(1, int(math.floor((width - chip_overlap) / stride))) if (height - chip_overlap) % stride > min_chip_location_diff: h_steps = max(1, int(math.ceil((height - chip_overlap) / stride))) else: h_steps = max(1, int(math.floor((height - chip_overlap) / stride))) chips = list() for j in range(h_steps): for i in range(w_steps): x1 = i * stride y1 = j * stride x2 = min(x1 + chip_size, width) y2 = min(y1 + chip_size, height) chips.append([x1, y1, x2, y2]) # check chip size for item in chips: if item[2] - item[0] > chip_size * 1.1 or item[3] - item[ 1] > chip_size * 1.1: raise ValueError(item) chips = np.array(chips, dtype=np.float32) raw_size_chips = chips / scale return raw_size_chips def _get_valid_boxes_and_pos_chips(self, gt_bbox, chips): valid_ratio_range = self._cur_valid_ratio_range im_size = self._cur_im_size scale = self._cur_scale # Nx4 N valid_boxes, valid_boxes_idx = self._validate_boxes( valid_ratio_range, im_size, gt_bbox, scale) # dict: chipid->[box_id, ...] pos_chip2boxes_idx = self._find_pos_chips(chips, valid_boxes, valid_boxes_idx) return pos_chip2boxes_idx def _validate_boxes(self, valid_ratio_range: List[float], im_size: int, gt_boxes: 'np.array of Nx4', scale: float): """ :return: valid_boxes: Nx4, valid_boxes_idx: N """ ws = (gt_boxes[:, 2] - gt_boxes[:, 0]).astype(np.int32) hs = (gt_boxes[:, 3] - gt_boxes[:, 1]).astype(np.int32) maxs = np.maximum(ws, hs) box_ratio = maxs / im_size mins = np.minimum(ws, hs) target_mins = mins * scale low = valid_ratio_range[0] if valid_ratio_range[0] > 0 else 0 high = valid_ratio_range[1] if valid_ratio_range[1] > 0 else np.finfo( np.float32).max valid_boxes_idx = np.nonzero((low <= box_ratio) & (box_ratio < high) & ( target_mins >= 2))[0] valid_boxes = gt_boxes[valid_boxes_idx] return valid_boxes, valid_boxes_idx def _find_pos_chips(self, chips: 'Cx4', valid_boxes: 'Bx4', valid_boxes_idx: 'B'): """ :return: pos_chip2boxes_idx, dict: chipid->[box_id, ...] """ iob = intersection_over_box(chips, valid_boxes) # overlap, CxB iob_threshold_to_find_chips = 1. pos_chip_ids, _ = self._find_chips_to_cover_overlaped_boxes( iob, iob_threshold_to_find_chips) pos_chip_ids = set(pos_chip_ids) iob_threshold_to_assign_box = 0.5 pos_chip2boxes_idx = self._assign_boxes_to_pos_chips( iob, iob_threshold_to_assign_box, pos_chip_ids, valid_boxes_idx) return pos_chip2boxes_idx def _find_chips_to_cover_overlaped_boxes(self, iob, overlap_threshold): return find_chips_to_cover_overlaped_boxes(iob, overlap_threshold) def _assign_boxes_to_pos_chips(self, iob, overlap_threshold, pos_chip_ids, valid_boxes_idx): chip_ids, box_ids = np.nonzero(iob >= overlap_threshold) pos_chip2boxes_idx = defaultdict(list) for chip_id, box_id in zip(chip_ids, box_ids): if chip_id not in pos_chip_ids: continue raw_gt_box_idx = valid_boxes_idx[box_id] pos_chip2boxes_idx[chip_id].append(raw_gt_box_idx) return pos_chip2boxes_idx def _get_neg_boxes_and_chips(self, chips: 'Cx4', pos_chip_ids: 'D', proposals: 'Px4'): """ :param chips: :param pos_chip_ids: :param proposals: :return: neg_chip2box_num, None or dict: chipid->neg_box_num """ if not self.use_neg_chip: return None # train proposals maybe None if proposals is None or len(proposals) < 1: return None valid_ratio_range = self._cur_valid_ratio_range im_size = self._cur_im_size scale = self._cur_scale valid_props, _ = self._validate_boxes(valid_ratio_range, im_size, proposals, scale) neg_boxes = self._find_neg_boxes(chips, pos_chip_ids, valid_props) neg_chip2box_num = self._find_neg_chips(chips, pos_chip_ids, neg_boxes) return neg_chip2box_num def _find_neg_boxes(self, chips: 'Cx4', pos_chip_ids: 'D', valid_props: 'Px4'): """ :return: neg_boxes: Nx4 """ if len(pos_chip_ids) == 0: return valid_props pos_chips = chips[pos_chip_ids] iob = intersection_over_box(pos_chips, valid_props) overlap_per_prop = np.max(iob, axis=0) non_overlap_props_idx = overlap_per_prop < 0.5 neg_boxes = valid_props[non_overlap_props_idx] return neg_boxes def _find_neg_chips(self, chips: 'Cx4', pos_chip_ids: 'D', neg_boxes: 'Nx4'): """ :return: neg_chip2box_num, dict: chipid->neg_box_num """ neg_chip_ids = np.setdiff1d(np.arange(len(chips)), pos_chip_ids) neg_chips = chips[neg_chip_ids] iob = intersection_over_box(neg_chips, neg_boxes) iob_threshold_to_find_chips = 0.7 chosen_neg_chip_ids, chip_id2overlap_box_num = \ self._find_chips_to_cover_overlaped_boxes(iob, iob_threshold_to_find_chips) neg_chipid2box_num = {} for cid in chosen_neg_chip_ids: box_num = chip_id2overlap_box_num[cid] raw_chip_id = neg_chip_ids[cid] neg_chipid2box_num[raw_chip_id] = box_num return neg_chipid2box_num def crop_infer_anno_records(self, records: List[dict]): """ transform image record to chips record :param records: :return: new_records, list of dict like { 'im_file': 'fake_image1.jpg', 'im_id': np.array([1]), # new _global_chip_id as im_id 'h': h, # chip height 'w': w, # chip width 'chip': [x1, y1, x2, y2] # added 'ori_im_h': ori_im_h # added, origin image height 'ori_im_w': ori_im_w # added, origin image width 'scale_i': 0 # added, } """ self.chip_records = [] self._global_chip_id = 1 # im_id start from 1 self._global_chip_id2img_id = {} for r in records: for scale_i in range(self.scale_num): self._get_current_scale_parameters(scale_i, r) # Cx4 chips = self._create_chips(r['h'], r['w'], self._cur_scale) cur_img_chip_record = self._get_chips_records(r, chips, scale_i) self.chip_records.extend(cur_img_chip_record) return self.chip_records def _get_chips_records(self, rec, chips, scale_i): cur_img_chip_records = [] ori_im_h = rec["h"] ori_im_w = rec["w"] im_file = rec["im_file"] ori_im_id = rec["im_id"] for id, chip in enumerate(chips): chip_rec = {} x1, y1, x2, y2 = chip chip_h = y2 - y1 chip_w = x2 - x1 chip_rec["im_file"] = im_file chip_rec["im_id"] = self._global_chip_id chip_rec["h"] = chip_h chip_rec["w"] = chip_w chip_rec["chip"] = chip chip_rec["ori_im_h"] = ori_im_h chip_rec["ori_im_w"] = ori_im_w chip_rec["scale_i"] = scale_i self._global_chip_id2img_id[self._global_chip_id] = int(ori_im_id) self._global_chip_id += 1 cur_img_chip_records.append(chip_rec) return cur_img_chip_records def aggregate_chips_detections(self, results, records=None): """ # 1. transform chip dets to image dets # 2. nms boxes per image; # 3. format output results :param results: :param roidb: :return: """ results = deepcopy(results) records = records if records else self.chip_records img_id2bbox = self._transform_chip2image_bboxes(results, records) nms_img_id2bbox = self._nms_dets(img_id2bbox) aggregate_results = self._reformat_results(nms_img_id2bbox) return aggregate_results def _transform_chip2image_bboxes(self, results, records): # 1. Transform chip dets to image dets; # 2. Filter valid range; # 3. Reformat and Aggregate chip dets to Get scale_cls_dets img_id2bbox = defaultdict(list) for result in results: bbox_locs = result['bbox'] bbox_nums = result['bbox_num'] if len(bbox_locs) == 1 and bbox_locs[0][ 0] == -1: # current batch has no detections # bbox_locs = array([[-1.]], dtype=float32); bbox_nums = [[1]] # MultiClassNMS output: If there is no detected boxes for all images, lod will be set to {1} and Out only contains one value which is -1. continue im_ids = result['im_id'] # replace with range(len(bbox_nums)) last_bbox_num = 0 for idx, im_id in enumerate(im_ids): cur_bbox_len = bbox_nums[idx] bboxes = bbox_locs[last_bbox_num:last_bbox_num + cur_bbox_len] last_bbox_num += cur_bbox_len # box: [num_id, score, xmin, ymin, xmax, ymax] if len(bboxes) == 0: # current image has no detections continue chip_rec = records[int(im_id) - 1] # im_id starts from 1, type is np.int64 image_size = max(chip_rec["ori_im_h"], chip_rec["ori_im_w"]) bboxes = transform_chip_boxes2image_boxes( bboxes, chip_rec["chip"], chip_rec["ori_im_h"], chip_rec["ori_im_w"]) scale_i = chip_rec["scale_i"] cur_scale = self._get_current_scale(self.target_sizes[scale_i], image_size) _, valid_boxes_idx = self._validate_boxes( self.valid_box_ratio_ranges[scale_i], image_size, bboxes[:, 2:], cur_scale) ori_img_id = self._global_chip_id2img_id[int(im_id)] img_id2bbox[ori_img_id].append(bboxes[valid_boxes_idx]) return img_id2bbox def _nms_dets(self, img_id2bbox): # 1. NMS on each image-class # 2. Limit number of detections to MAX_PER_IMAGE if requested max_per_img = self.max_per_img nms_thresh = self.nms_thresh for img_id in img_id2bbox: box = img_id2bbox[ img_id] # list of np.array of shape [N, 6], 6 is [label, score, x1, y1, x2, y2] box = np.concatenate(box, axis=0) nms_dets = nms(box, nms_thresh) if max_per_img > 0: if len(nms_dets) > max_per_img: keep = np.argsort(-nms_dets[:, 1])[:max_per_img] nms_dets = nms_dets[keep] img_id2bbox[img_id] = nms_dets return img_id2bbox def _reformat_results(self, img_id2bbox): """reformat results""" im_ids = img_id2bbox.keys() results = [] for img_id in im_ids: # output by original im_id order if len(img_id2bbox[img_id]) == 0: bbox = np.array( [[-1., 0., 0., 0., 0., 0.]]) # edge case: no detections bbox_num = np.array([0]) else: # np.array of shape [N, 6], 6 is [label, score, x1, y1, x2, y2] bbox = img_id2bbox[img_id] bbox_num = np.array([len(bbox)]) res = dict(im_id=np.array([[img_id]]), bbox=bbox, bbox_num=bbox_num) results.append(res) return results