kwcoco.coco_evaluator module

Evaluates a predicted coco dataset against a truth coco dataset.

This currently computes detection-level metrics.

The components in this module work programatically or as a command line script.

Todo

• [ ] does evaluate return one result or multiple results

  based on different configurations?

• [ ] max_dets - TODO: in original pycocoutils but not here

• [ ] Flag that allows for polygon instead of bounding box overlap

• [ ] How do we note what iou_thresh and area-range were in

  the result plots?

CommandLine

xdoctest -m kwcoco.coco_evaluator __doc__:0 --vd --slow

Example

>>> from kwcoco.coco_evaluator import *  # NOQA
>>> from kwcoco.coco_evaluator import CocoEvaluator
>>> import kwcoco
>>> # note: increase the number of images for better looking metrics
>>> true_dset = kwcoco.CocoDataset.demo('shapes8')
>>> from kwcoco.demo.perterb import perterb_coco
>>> kwargs = {
>>>     'box_noise': 0.5,
>>>     'n_fp': (0, 10),
>>>     'n_fn': (0, 10),
>>>     'with_probs': True,
>>> }
>>> pred_dset = perterb_coco(true_dset, **kwargs)
>>> print('true_dset = {!r}'.format(true_dset))
>>> print('pred_dset = {!r}'.format(pred_dset))
>>> config = {
>>>     'true_dataset': true_dset,
>>>     'pred_dataset': pred_dset,
>>>     'area_range': ['all', 'small'],
>>>     'iou_thresh': [0.3, 0.95],
>>> }
>>> coco_eval = CocoEvaluator(config)
>>> results = coco_eval.evaluate()
>>> # Now we can draw / serialize the results as we please
>>> dpath = ub.Path.appdir('kwcoco/tests/test_out_dpath').ensuredir()
>>> results_fpath = dpath / 'metrics.json'
>>> print('results_fpath = {!r}'.format(results_fpath))
>>> results.dump(results_fpath, indent='    ')
>>> measures = results['area_range=all,iou_thresh=0.3'].nocls_measures
>>> import pandas as pd
>>> print(pd.DataFrame(ub.dict_isect(
>>>     measures, ['f1', 'g1', 'mcc', 'thresholds',
>>>                'ppv', 'tpr', 'tnr', 'npv', 'fpr',
>>>                'tp_count', 'fp_count',
>>>                'tn_count', 'fn_count'])).iloc[::100])
>>> # xdoctest: +REQUIRES(module:kwplot)
>>> # xdoctest: +REQUIRES(--slow)
>>> results.dump_figures(dpath)
>>> print('dpath = {!r}'.format(dpath))
>>> # xdoctest: +REQUIRES(--vd)
>>> if ub.argflag('--vd') or 1:
>>>     import xdev
>>>     xdev.view_directory(dpath)

class kwcoco.coco_evaluator.CocoEvalConfig(*args, **kwargs)

Bases: DataConfig

Evaluate and score predicted versus truth detections / classifications in a COCO dataset

Valid options: []

Parameters:

• *args – positional arguments for this data config

• **kwargs – keyword arguments for this data config

default = {'ap_method': <Value('pycocotools')>, 'area_range': <Value(['all'])>, 'assign_workers': <Value(8)>, 'classes_of_interest': <Value(None)>, 'compat': <Value('mutex')>, 'force_pycocoutils': <Value(False)>, 'fp_cutoff': <Value(inf)>, 'ignore_classes': <Value(None)>, 'implicit_ignore_classes': <Value(['ignore'])>, 'implicit_negative_classes': <Value(['background'])>, 'iou_bias': <Value(1)>, 'iou_thresh': <Value(0.5)>, 'load_workers': <Value(0)>, 'max_dets': <Value(inf)>, 'monotonic_ppv': <Value(True)>, 'pred_dataset': <Value(None)>, 'true_dataset': <Value(None)>, 'use_area_attr': <Value('try')>, 'use_image_names': <Value(False)>}

normalize()

class kwcoco.coco_evaluator.CocoEvaluator(config)

Bases: object

Abstracts the evaluation process to execute on two coco datasets.

This can be run as a standalone script where the user specifies the paths to the true and predited dataset explicitly, or this can be used by a higher level script that produces the predictions and then sends them to this evaluator.

Example

>>> from kwcoco.coco_evaluator import CocoEvaluator
>>> from kwcoco.demo.perterb import perterb_coco
>>> import kwcoco
>>> true_dset = kwcoco.CocoDataset.demo('shapes8')
>>> kwargs = {
>>>     'box_noise': 0.5,
>>>     'n_fp': (0, 10),
>>>     'n_fn': (0, 10),
>>>     'with_probs': True,
>>> }
>>> pred_dset = perterb_coco(true_dset, **kwargs)
>>> config = {
>>>     'true_dataset': true_dset,
>>>     'pred_dataset': pred_dset,
>>>     'classes_of_interest': [],
>>> }
>>> coco_eval = CocoEvaluator(config)
>>> results = coco_eval.evaluate()

log(msg, level='INFO')

_init()

Performs initial coercion from given inputs into dictionaries of kwimage.Detection objects and attempts to ensure comparable category and image ids.

_ensure_init()

classmethod _rectify_classes(true_classes, pred_classes)

classmethod _coerce_dets(dataset, verbose=0, workers=0)

Coerce the input to a mapping from image-id to kwimage.Detection

Also capture a CocoDataset if possible.

Returns:

gid_to_det: mapping from gid to dets extra: any extra information we gathered via coercion

Return type:

Tuple[Dict[int, Detections], Dict]

Example

>>> from kwcoco.coco_evaluator import *  # NOQA
>>> import kwcoco
>>> coco_dset = kwcoco.CocoDataset.demo('shapes8')
>>> gid_to_det, extras = CocoEvaluator._coerce_dets(coco_dset)

Example

>>> # xdoctest: +REQUIRES(module:sqlalchemy)
>>> from kwcoco.coco_evaluator import *  # NOQA
>>> import kwcoco
>>> coco_dset = kwcoco.CocoDataset.demo('shapes8').view_sql()
>>> gid_to_det, extras = CocoEvaluator._coerce_dets(coco_dset)

_build_dmet()

Builds the detection metrics object

Returns:

DetectionMetrics - object that can perform assignment and

build confusion vectors.

evaluate()

Executes the main evaluation logic. Performs assignments between detections to make DetectionMetrics object, then creates per-item and ovr confusion vectors, and performs various threshold-vs-confusion analyses.

Returns:

container storing (and capable of drawing /

serializing) results

Return type:

CocoResults

kwcoco.coco_evaluator.dmet_area_weights(dmet, orig_weights, cfsn_vecs, area_ranges, coco_eval, use_area_attr=False)

Hacky function to compute confusion vector ignore weights for different area thresholds. Needs to be slightly refactored.

class kwcoco.coco_evaluator.CocoResults(resdata=None)

Bases: NiceRepr, DictProxy

CommandLine

xdoctest -m /home/joncrall/code/kwcoco/kwcoco/coco_evaluator.py CocoResults --profile

Example

>>> from kwcoco.coco_evaluator import *  # NOQA
>>> from kwcoco.coco_evaluator import CocoEvaluator
>>> import kwcoco
>>> true_dset = kwcoco.CocoDataset.demo('shapes2')
>>> from kwcoco.demo.perterb import perterb_coco
>>> kwargs = {
>>>     'box_noise': 0.5,
>>>     'n_fp': (0, 10),
>>>     'n_fn': (0, 10),
>>> }
>>> pred_dset = perterb_coco(true_dset, **kwargs)
>>> print('true_dset = {!r}'.format(true_dset))
>>> print('pred_dset = {!r}'.format(pred_dset))
>>> config = {
>>>     'true_dataset': true_dset,
>>>     'pred_dataset': pred_dset,
>>>     'area_range': ['small'],
>>>     'iou_thresh': [0.3],
>>> }
>>> coco_eval = CocoEvaluator(config)
>>> results = coco_eval.evaluate()
>>> # Now we can draw / serialize the results as we please
>>> dpath = ub.Path.appdir('kwcoco/tests/test_out_dpath').ensuredir()
>>> #
>>> # test deserialization works
>>> state = results.__json__()
>>> self2 = CocoResults.from_json(state)
>>> #
>>> # xdoctest: +REQUIRES(module:kwplot)
>>> results.dump_figures(dpath, figsize=(3, 2), tight=False)  # make this go faster
>>> results.dump(dpath / 'metrics.json', indent='    ')

dump_figures(out_dpath, expt_title=None, figsize='auto', tight=True)

classmethod from_json(state)

dump(file, indent='    ')

Serialize to json file

class kwcoco.coco_evaluator.CocoSingleResult(nocls_measures, ovr_measures, cfsn_vecs, meta=None)

Bases: NiceRepr

Container class to store, draw, summarize, and serialize results from CocoEvaluator.

Example

>>> # xdoctest: +REQUIRES(--slow)
>>> from kwcoco.coco_evaluator import *  # NOQA
>>> from kwcoco.coco_evaluator import CocoEvaluator
>>> import kwcoco
>>> true_dset = kwcoco.CocoDataset.demo('shapes8')
>>> from kwcoco.demo.perterb import perterb_coco
>>> kwargs = {
>>>     'box_noise': 0.2,
>>>     'n_fp': (0, 3),
>>>     'n_fn': (0, 3),
>>>     'with_probs': False,
>>> }
>>> pred_dset = perterb_coco(true_dset, **kwargs)
>>> print('true_dset = {!r}'.format(true_dset))
>>> print('pred_dset = {!r}'.format(pred_dset))
>>> config = {
>>>     'true_dataset': true_dset,
>>>     'pred_dataset': pred_dset,
>>>     'area_range': [(0, 32 ** 2), (32 ** 2, 96 ** 2)],
>>>     'iou_thresh': [0.3, 0.5, 0.95],
>>> }
>>> coco_eval = CocoEvaluator(config)
>>> results = coco_eval.evaluate()
>>> result = ub.peek(results.values())
>>> state = result.__json__()
>>> print('state = {}'.format(ub.urepr(state, nl=-1)))
>>> recon = CocoSingleResult.from_json(state)
>>> state = recon.__json__()
>>> print('state = {}'.format(ub.urepr(state, nl=-1)))

classmethod from_json(state)

dump(file, indent='    ')

Serialize to json file

dump_figures(out_dpath, expt_title=None, figsize='auto', tight=True, verbose=1)

kwcoco.coco_evaluator._writefig(fig, metrics_dpath, fname, figsize, verbose, tight)

kwcoco.coco_evaluator._load_dets(pred_fpaths, workers=0)

Example

>>> from kwcoco.coco_evaluator import _load_dets, _load_dets_worker
>>> import ubelt as ub
>>> import kwcoco
>>> dpath = ub.Path.appdir('kwcoco/tests/load_dets').ensuredir()
>>> N = 4
>>> pred_fpaths = []
>>> for i in range(1, N + 1):
>>>     dset = kwcoco.CocoDataset.demo('shapes{}'.format(i))
>>>     dset.fpath = dpath / 'shapes_{}.mscoco.json'.format(i)
>>>     dset.dump(dset.fpath)
>>>     pred_fpaths.append(dset.fpath)
>>> dets, coco_dset = _load_dets(pred_fpaths)
>>> print('dets = {!r}'.format(dets))
>>> print('coco_dset = {!r}'.format(coco_dset))

kwcoco.coco_evaluator._load_dets_worker(single_pred_fpath, with_coco=True)