545 lines
21 KiB
Python
545 lines
21 KiB
Python
# Copyright (c) OpenMMLab. All rights reserved.
|
|
import itertools
|
|
import os
|
|
from collections import defaultdict
|
|
|
|
import mmcv
|
|
import numpy as np
|
|
from mmcv.utils import print_log
|
|
from terminaltables import AsciiTable
|
|
|
|
from .api_wrappers import COCO, pq_compute_multi_core
|
|
from .builder import DATASETS
|
|
from .coco import CocoDataset
|
|
|
|
try:
|
|
import panopticapi
|
|
from panopticapi.evaluation import VOID
|
|
from panopticapi.utils import id2rgb
|
|
except ImportError:
|
|
panopticapi = None
|
|
id2rgb = None
|
|
VOID = None
|
|
|
|
__all__ = ['CocoPanopticDataset']
|
|
|
|
# A custom value to distinguish instance ID and category ID; need to
|
|
# be greater than the number of categories.
|
|
# For a pixel in the panoptic result map:
|
|
# pan_id = ins_id * INSTANCE_OFFSET + cat_id
|
|
INSTANCE_OFFSET = 1000
|
|
|
|
|
|
class COCOPanoptic(COCO):
|
|
"""This wrapper is for loading the panoptic style annotation file.
|
|
|
|
The format is shown in the CocoPanopticDataset class.
|
|
|
|
Args:
|
|
annotation_file (str): Path of annotation file.
|
|
"""
|
|
|
|
def __init__(self, annotation_file=None):
|
|
if panopticapi is None:
|
|
raise RuntimeError(
|
|
'panopticapi is not installed, please install it by: '
|
|
'pip install git+https://github.com/cocodataset/'
|
|
'panopticapi.git.')
|
|
|
|
super(COCOPanoptic, self).__init__(annotation_file)
|
|
|
|
def createIndex(self):
|
|
# create index
|
|
print('creating index...')
|
|
# anns stores 'segment_id -> annotation'
|
|
anns, cats, imgs = {}, {}, {}
|
|
img_to_anns, cat_to_imgs = defaultdict(list), defaultdict(list)
|
|
if 'annotations' in self.dataset:
|
|
for ann, img_info in zip(self.dataset['annotations'],
|
|
self.dataset['images']):
|
|
img_info['segm_file'] = ann['file_name']
|
|
for seg_ann in ann['segments_info']:
|
|
# to match with instance.json
|
|
seg_ann['image_id'] = ann['image_id']
|
|
seg_ann['height'] = img_info['height']
|
|
seg_ann['width'] = img_info['width']
|
|
img_to_anns[ann['image_id']].append(seg_ann)
|
|
# segment_id is not unique in coco dataset orz...
|
|
if seg_ann['id'] in anns.keys():
|
|
anns[seg_ann['id']].append(seg_ann)
|
|
else:
|
|
anns[seg_ann['id']] = [seg_ann]
|
|
|
|
if 'images' in self.dataset:
|
|
for img in self.dataset['images']:
|
|
imgs[img['id']] = img
|
|
|
|
if 'categories' in self.dataset:
|
|
for cat in self.dataset['categories']:
|
|
cats[cat['id']] = cat
|
|
|
|
if 'annotations' in self.dataset and 'categories' in self.dataset:
|
|
for ann in self.dataset['annotations']:
|
|
for seg_ann in ann['segments_info']:
|
|
cat_to_imgs[seg_ann['category_id']].append(ann['image_id'])
|
|
|
|
print('index created!')
|
|
|
|
self.anns = anns
|
|
self.imgToAnns = img_to_anns
|
|
self.catToImgs = cat_to_imgs
|
|
self.imgs = imgs
|
|
self.cats = cats
|
|
|
|
def load_anns(self, ids=[]):
|
|
"""Load anns with the specified ids.
|
|
|
|
self.anns is a list of annotation lists instead of a
|
|
list of annotations.
|
|
|
|
Args:
|
|
ids (int array): integer ids specifying anns
|
|
|
|
Returns:
|
|
anns (object array): loaded ann objects
|
|
"""
|
|
anns = []
|
|
|
|
if hasattr(ids, '__iter__') and hasattr(ids, '__len__'):
|
|
# self.anns is a list of annotation lists instead of
|
|
# a list of annotations
|
|
for id in ids:
|
|
anns += self.anns[id]
|
|
return anns
|
|
elif type(ids) == int:
|
|
return self.anns[ids]
|
|
|
|
|
|
@DATASETS.register_module()
|
|
class CocoPanopticDataset(CocoDataset):
|
|
"""Coco dataset for Panoptic segmentation.
|
|
|
|
The annotation format is shown as follows. The `ann` field is optional
|
|
for testing.
|
|
|
|
.. code-block:: none
|
|
|
|
[
|
|
{
|
|
'filename': f'{image_id:012}.png',
|
|
'image_id':9
|
|
'segments_info': {
|
|
[
|
|
{
|
|
'id': 8345037, (segment_id in panoptic png,
|
|
convert from rgb)
|
|
'category_id': 51,
|
|
'iscrowd': 0,
|
|
'bbox': (x1, y1, w, h),
|
|
'area': 24315,
|
|
'segmentation': list,(coded mask)
|
|
},
|
|
...
|
|
}
|
|
}
|
|
},
|
|
...
|
|
]
|
|
"""
|
|
CLASSES = [
|
|
'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train',
|
|
' truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign',
|
|
'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep',
|
|
'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella',
|
|
'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard',
|
|
'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard',
|
|
'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork',
|
|
'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange',
|
|
'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair',
|
|
'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv',
|
|
'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave',
|
|
'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase',
|
|
'scissors', 'teddy bear', 'hair drier', 'toothbrush', 'banner',
|
|
'blanket', 'bridge', 'cardboard', 'counter', 'curtain', 'door-stuff',
|
|
'floor-wood', 'flower', 'fruit', 'gravel', 'house', 'light',
|
|
'mirror-stuff', 'net', 'pillow', 'platform', 'playingfield',
|
|
'railroad', 'river', 'road', 'roof', 'sand', 'sea', 'shelf', 'snow',
|
|
'stairs', 'tent', 'towel', 'wall-brick', 'wall-stone', 'wall-tile',
|
|
'wall-wood', 'water-other', 'window-blind', 'window-other',
|
|
'tree-merged', 'fence-merged', 'ceiling-merged', 'sky-other-merged',
|
|
'cabinet-merged', 'table-merged', 'floor-other-merged',
|
|
'pavement-merged', 'mountain-merged', 'grass-merged', 'dirt-merged',
|
|
'paper-merged', 'food-other-merged', 'building-other-merged',
|
|
'rock-merged', 'wall-other-merged', 'rug-merged'
|
|
]
|
|
THING_CLASSES = [
|
|
'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train',
|
|
'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign',
|
|
'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep',
|
|
'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella',
|
|
'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard',
|
|
'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard',
|
|
'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork',
|
|
'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange',
|
|
'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair',
|
|
'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv',
|
|
'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave',
|
|
'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase',
|
|
'scissors', 'teddy bear', 'hair drier', 'toothbrush'
|
|
]
|
|
STUFF_CLASSES = [
|
|
'banner', 'blanket', 'bridge', 'cardboard', 'counter', 'curtain',
|
|
'door-stuff', 'floor-wood', 'flower', 'fruit', 'gravel', 'house',
|
|
'light', 'mirror-stuff', 'net', 'pillow', 'platform', 'playingfield',
|
|
'railroad', 'river', 'road', 'roof', 'sand', 'sea', 'shelf', 'snow',
|
|
'stairs', 'tent', 'towel', 'wall-brick', 'wall-stone', 'wall-tile',
|
|
'wall-wood', 'water-other', 'window-blind', 'window-other',
|
|
'tree-merged', 'fence-merged', 'ceiling-merged', 'sky-other-merged',
|
|
'cabinet-merged', 'table-merged', 'floor-other-merged',
|
|
'pavement-merged', 'mountain-merged', 'grass-merged', 'dirt-merged',
|
|
'paper-merged', 'food-other-merged', 'building-other-merged',
|
|
'rock-merged', 'wall-other-merged', 'rug-merged'
|
|
]
|
|
|
|
def load_annotations(self, ann_file):
|
|
"""Load annotation from COCO Panoptic style annotation file.
|
|
|
|
Args:
|
|
ann_file (str): Path of annotation file.
|
|
|
|
Returns:
|
|
list[dict]: Annotation info from COCO api.
|
|
"""
|
|
self.coco = COCOPanoptic(ann_file)
|
|
self.cat_ids = self.coco.get_cat_ids()
|
|
self.cat2label = {cat_id: i for i, cat_id in enumerate(self.cat_ids)}
|
|
self.categories = self.coco.cats
|
|
self.img_ids = self.coco.get_img_ids()
|
|
data_infos = []
|
|
for i in self.img_ids:
|
|
info = self.coco.load_imgs([i])[0]
|
|
info['filename'] = info['file_name']
|
|
info['segm_file'] = info['filename'].replace('jpg', 'png')
|
|
data_infos.append(info)
|
|
return data_infos
|
|
|
|
def get_ann_info(self, idx):
|
|
"""Get COCO annotation by index.
|
|
|
|
Args:
|
|
idx (int): Index of data.
|
|
|
|
Returns:
|
|
dict: Annotation info of specified index.
|
|
"""
|
|
img_id = self.data_infos[idx]['id']
|
|
ann_ids = self.coco.get_ann_ids(img_ids=[img_id])
|
|
ann_info = self.coco.load_anns(ann_ids)
|
|
# filter out unmatched images
|
|
ann_info = [i for i in ann_info if i['image_id'] == img_id]
|
|
return self._parse_ann_info(self.data_infos[idx], ann_info)
|
|
|
|
def _parse_ann_info(self, img_info, ann_info):
|
|
"""Parse annotations and load panoptic ground truths.
|
|
|
|
Args:
|
|
img_info (int): Image info of an image.
|
|
ann_info (list[dict]): Annotation info of an image.
|
|
|
|
Returns:
|
|
dict: A dict containing the following keys: bboxes, bboxes_ignore,
|
|
labels, masks, seg_map.
|
|
"""
|
|
gt_bboxes = []
|
|
gt_labels = []
|
|
gt_bboxes_ignore = []
|
|
gt_mask_infos = []
|
|
|
|
for i, ann in enumerate(ann_info):
|
|
x1, y1, w, h = ann['bbox']
|
|
if ann['area'] <= 0 or w < 1 or h < 1:
|
|
continue
|
|
bbox = [x1, y1, x1 + w, y1 + h]
|
|
|
|
category_id = ann['category_id']
|
|
contiguous_cat_id = self.cat2label[category_id]
|
|
|
|
is_thing = self.coco.load_cats(ids=category_id)[0]['isthing']
|
|
if is_thing:
|
|
is_crowd = ann.get('iscrowd', False)
|
|
if not is_crowd:
|
|
gt_bboxes.append(bbox)
|
|
gt_labels.append(contiguous_cat_id)
|
|
else:
|
|
gt_bboxes_ignore.append(bbox)
|
|
is_thing = False
|
|
|
|
mask_info = {
|
|
'id': ann['id'],
|
|
'category': contiguous_cat_id,
|
|
'is_thing': is_thing
|
|
}
|
|
gt_mask_infos.append(mask_info)
|
|
|
|
if gt_bboxes:
|
|
gt_bboxes = np.array(gt_bboxes, dtype=np.float32)
|
|
gt_labels = np.array(gt_labels, dtype=np.int64)
|
|
else:
|
|
gt_bboxes = np.zeros((0, 4), dtype=np.float32)
|
|
gt_labels = np.array([], dtype=np.int64)
|
|
|
|
if gt_bboxes_ignore:
|
|
gt_bboxes_ignore = np.array(gt_bboxes_ignore, dtype=np.float32)
|
|
else:
|
|
gt_bboxes_ignore = np.zeros((0, 4), dtype=np.float32)
|
|
|
|
ann = dict(
|
|
bboxes=gt_bboxes,
|
|
labels=gt_labels,
|
|
bboxes_ignore=gt_bboxes_ignore,
|
|
masks=gt_mask_infos,
|
|
seg_map=img_info['segm_file'])
|
|
|
|
return ann
|
|
|
|
def _filter_imgs(self, min_size=32):
|
|
"""Filter images too small or without ground truths."""
|
|
ids_with_ann = []
|
|
# check whether images have legal thing annotations.
|
|
for lists in self.coco.anns.values():
|
|
for item in lists:
|
|
category_id = item['category_id']
|
|
is_thing = self.coco.load_cats(ids=category_id)[0]['isthing']
|
|
if not is_thing:
|
|
continue
|
|
ids_with_ann.append(item['image_id'])
|
|
ids_with_ann = set(ids_with_ann)
|
|
|
|
valid_inds = []
|
|
valid_img_ids = []
|
|
for i, img_info in enumerate(self.data_infos):
|
|
img_id = self.img_ids[i]
|
|
if self.filter_empty_gt and img_id not in ids_with_ann:
|
|
continue
|
|
if min(img_info['width'], img_info['height']) >= min_size:
|
|
valid_inds.append(i)
|
|
valid_img_ids.append(img_id)
|
|
self.img_ids = valid_img_ids
|
|
return valid_inds
|
|
|
|
def _pan2json(self, results, outfile_prefix):
|
|
"""Convert panoptic results to COCO panoptic json style."""
|
|
label2cat = dict((v, k) for (k, v) in self.cat2label.items())
|
|
pred_annotations = []
|
|
outdir = os.path.join(os.path.dirname(outfile_prefix), 'panoptic')
|
|
|
|
for idx in range(len(self)):
|
|
img_id = self.img_ids[idx]
|
|
segm_file = self.data_infos[idx]['segm_file']
|
|
pan = results[idx]
|
|
|
|
pan_labels = np.unique(pan)
|
|
segm_info = []
|
|
for pan_label in pan_labels:
|
|
sem_label = pan_label % INSTANCE_OFFSET
|
|
# We reserve the length of self.CLASSES for VOID label
|
|
if sem_label == len(self.CLASSES):
|
|
continue
|
|
# convert sem_label to json label
|
|
cat_id = label2cat[sem_label]
|
|
is_thing = self.categories[cat_id]['isthing']
|
|
mask = pan == pan_label
|
|
area = mask.sum()
|
|
segm_info.append({
|
|
'id': int(pan_label),
|
|
'category_id': cat_id,
|
|
'isthing': is_thing,
|
|
'area': int(area)
|
|
})
|
|
# evaluation script uses 0 for VOID label.
|
|
pan[pan % INSTANCE_OFFSET == len(self.CLASSES)] = VOID
|
|
pan = id2rgb(pan).astype(np.uint8)
|
|
mmcv.imwrite(pan[:, :, ::-1], os.path.join(outdir, segm_file))
|
|
record = {
|
|
'image_id': img_id,
|
|
'segments_info': segm_info,
|
|
'file_name': segm_file
|
|
}
|
|
pred_annotations.append(record)
|
|
pan_json_results = dict(annotations=pred_annotations)
|
|
return pan_json_results
|
|
|
|
def results2json(self, results, outfile_prefix):
|
|
"""Dump the panoptic results to a COCO panoptic style json file.
|
|
|
|
Args:
|
|
results (dict): Testing results of the dataset.
|
|
outfile_prefix (str): The filename prefix of the json files. If the
|
|
prefix is "somepath/xxx", the json files will be named
|
|
"somepath/xxx.panoptic.json"
|
|
|
|
Returns:
|
|
dict[str: str]: The key is 'panoptic' and the value is
|
|
corresponding filename.
|
|
"""
|
|
result_files = dict()
|
|
pan_results = [result['pan_results'] for result in results]
|
|
pan_json_results = self._pan2json(pan_results, outfile_prefix)
|
|
result_files['panoptic'] = f'{outfile_prefix}.panoptic.json'
|
|
mmcv.dump(pan_json_results, result_files['panoptic'])
|
|
|
|
return result_files
|
|
|
|
def evaluate_pan_json(self,
|
|
result_files,
|
|
outfile_prefix,
|
|
logger=None,
|
|
classwise=False):
|
|
"""Evaluate PQ according to the panoptic results json file."""
|
|
imgs = self.coco.imgs
|
|
gt_json = self.coco.img_ann_map # image to annotations
|
|
gt_json = [{
|
|
'image_id': k,
|
|
'segments_info': v,
|
|
'file_name': imgs[k]['segm_file']
|
|
} for k, v in gt_json.items()]
|
|
pred_json = mmcv.load(result_files['panoptic'])
|
|
pred_json = dict(
|
|
(el['image_id'], el) for el in pred_json['annotations'])
|
|
|
|
# match the gt_anns and pred_anns in the same image
|
|
matched_annotations_list = []
|
|
for gt_ann in gt_json:
|
|
img_id = gt_ann['image_id']
|
|
if img_id not in pred_json.keys():
|
|
raise Exception('no prediction for the image'
|
|
' with id: {}'.format(img_id))
|
|
matched_annotations_list.append((gt_ann, pred_json[img_id]))
|
|
|
|
gt_folder = self.seg_prefix
|
|
pred_folder = os.path.join(os.path.dirname(outfile_prefix), 'panoptic')
|
|
|
|
pq_stat = pq_compute_multi_core(matched_annotations_list, gt_folder,
|
|
pred_folder, self.categories,
|
|
self.file_client)
|
|
|
|
metrics = [('All', None), ('Things', True), ('Stuff', False)]
|
|
pq_results = {}
|
|
|
|
for name, isthing in metrics:
|
|
pq_results[name], classwise_results = pq_stat.pq_average(
|
|
self.categories, isthing=isthing)
|
|
if name == 'All':
|
|
pq_results['classwise'] = classwise_results
|
|
|
|
classwise_results = None
|
|
if classwise:
|
|
classwise_results = {
|
|
k: v
|
|
for k, v in zip(self.CLASSES, pq_results['classwise'].values())
|
|
}
|
|
print_panoptic_table(pq_results, classwise_results, logger=logger)
|
|
|
|
return parse_pq_results(pq_results)
|
|
|
|
def evaluate(self,
|
|
results,
|
|
metric='PQ',
|
|
logger=None,
|
|
jsonfile_prefix=None,
|
|
classwise=False,
|
|
**kwargs):
|
|
"""Evaluation in COCO Panoptic protocol.
|
|
|
|
Args:
|
|
results (list[dict]): Testing results of the dataset.
|
|
metric (str | list[str]): Metrics to be evaluated. Only
|
|
support 'PQ' at present. 'pq' will be regarded as 'PQ.
|
|
logger (logging.Logger | str | None): Logger used for printing
|
|
related information during evaluation. Default: None.
|
|
jsonfile_prefix (str | None): The prefix of json files. It includes
|
|
the file path and the prefix of filename, e.g., "a/b/prefix".
|
|
If not specified, a temp file will be created. Default: None.
|
|
classwise (bool): Whether to print classwise evaluation results.
|
|
Default: False.
|
|
|
|
Returns:
|
|
dict[str, float]: COCO Panoptic style evaluation metric.
|
|
"""
|
|
metrics = metric if isinstance(metric, list) else [metric]
|
|
# Compatible with lowercase 'pq'
|
|
metrics = ['PQ' if metric == 'pq' else metric for metric in metrics]
|
|
allowed_metrics = ['PQ'] # todo: support other metrics like 'bbox'
|
|
for metric in metrics:
|
|
if metric not in allowed_metrics:
|
|
raise KeyError(f'metric {metric} is not supported')
|
|
|
|
result_files, tmp_dir = self.format_results(results, jsonfile_prefix)
|
|
eval_results = {}
|
|
|
|
outfile_prefix = os.path.join(tmp_dir.name, 'results') \
|
|
if tmp_dir is not None else jsonfile_prefix
|
|
if 'PQ' in metrics:
|
|
eval_pan_results = self.evaluate_pan_json(result_files,
|
|
outfile_prefix, logger,
|
|
classwise)
|
|
eval_results.update(eval_pan_results)
|
|
|
|
if tmp_dir is not None:
|
|
tmp_dir.cleanup()
|
|
return eval_results
|
|
|
|
|
|
def parse_pq_results(pq_results):
|
|
"""Parse the Panoptic Quality results."""
|
|
result = dict()
|
|
result['PQ'] = 100 * pq_results['All']['pq']
|
|
result['SQ'] = 100 * pq_results['All']['sq']
|
|
result['RQ'] = 100 * pq_results['All']['rq']
|
|
result['PQ_th'] = 100 * pq_results['Things']['pq']
|
|
result['SQ_th'] = 100 * pq_results['Things']['sq']
|
|
result['RQ_th'] = 100 * pq_results['Things']['rq']
|
|
result['PQ_st'] = 100 * pq_results['Stuff']['pq']
|
|
result['SQ_st'] = 100 * pq_results['Stuff']['sq']
|
|
result['RQ_st'] = 100 * pq_results['Stuff']['rq']
|
|
return result
|
|
|
|
|
|
def print_panoptic_table(pq_results, classwise_results=None, logger=None):
|
|
"""Print the panoptic evaluation results table.
|
|
|
|
Args:
|
|
pq_results(dict): The Panoptic Quality results.
|
|
classwise_results(dict | None): The classwise Panoptic Quality results.
|
|
The keys are class names and the values are metrics.
|
|
logger (logging.Logger | str | None): Logger used for printing
|
|
related information during evaluation. Default: None.
|
|
"""
|
|
|
|
headers = ['', 'PQ', 'SQ', 'RQ', 'categories']
|
|
data = [headers]
|
|
for name in ['All', 'Things', 'Stuff']:
|
|
numbers = [
|
|
f'{(pq_results[name][k] * 100):0.3f}' for k in ['pq', 'sq', 'rq']
|
|
]
|
|
row = [name] + numbers + [pq_results[name]['n']]
|
|
data.append(row)
|
|
table = AsciiTable(data)
|
|
print_log('Panoptic Evaluation Results:\n' + table.table, logger=logger)
|
|
|
|
if classwise_results is not None:
|
|
class_metrics = [(name, ) + tuple(f'{(metrics[k] * 100):0.3f}'
|
|
for k in ['pq', 'sq', 'rq'])
|
|
for name, metrics in classwise_results.items()]
|
|
num_columns = min(8, len(class_metrics) * 4)
|
|
results_flatten = list(itertools.chain(*class_metrics))
|
|
headers = ['category', 'PQ', 'SQ', 'RQ'] * (num_columns // 4)
|
|
results_2d = itertools.zip_longest(
|
|
*[results_flatten[i::num_columns] for i in range(num_columns)])
|
|
data = [headers]
|
|
data += [result for result in results_2d]
|
|
table = AsciiTable(data)
|
|
print_log(
|
|
'Classwise Panoptic Evaluation Results:\n' + table.table,
|
|
logger=logger)
|