:py:mod:`kwcoco.coco_evaluator`
===============================

.. py:module:: kwcoco.coco_evaluator

.. autoapi-nested-parse::

   Evaluates a predicted coco dataset against a truth coco dataset.

   The components in this module work programatically or as a command line script.

   .. todo::

      - [ ] does evaluate return one result or multiple results
            based on different configurations?
      
      - [ ] max_dets - TODO: in original pycocoutils but not here
      
      - [ ] Flag that allows for polygon instead of bounding box overlap
      
      - [ ] How do we note what iou_thresh and area-range were in
            the result plots?

   .. rubric:: CommandLine

   .. code-block:: bash

       xdoctest -m kwcoco.coco_evaluator __doc__:0 --vd --slow

   .. rubric:: Example

   >>> from kwcoco.coco_evaluator import *  # NOQA
   >>> from kwcoco.coco_evaluator import CocoEvaluator
   >>> import kwcoco
   >>> # note: increase the number of images for better looking metrics
   >>> true_dset = kwcoco.CocoDataset.demo('shapes8')
   >>> from kwcoco.demo.perterb import perterb_coco
   >>> kwargs = {
   >>>     'box_noise': 0.5,
   >>>     'n_fp': (0, 10),
   >>>     'n_fn': (0, 10),
   >>>     'with_probs': True,
   >>> }
   >>> pred_dset = perterb_coco(true_dset, **kwargs)
   >>> print('true_dset = {!r}'.format(true_dset))
   >>> print('pred_dset = {!r}'.format(pred_dset))
   >>> config = {
   >>>     'true_dataset': true_dset,
   >>>     'pred_dataset': pred_dset,
   >>>     'area_range': ['all', 'small'],
   >>>     'iou_thresh': [0.3, 0.95],
   >>> }
   >>> coco_eval = CocoEvaluator(config)
   >>> results = coco_eval.evaluate()
   >>> # Now we can draw / serialize the results as we please
   >>> dpath = ub.ensure_app_cache_dir('kwcoco/tests/test_out_dpath')
   >>> results_fpath = join(dpath, 'metrics.json')
   >>> print('results_fpath = {!r}'.format(results_fpath))
   >>> results.dump(results_fpath, indent='    ')
   >>> measures = results['area_range=all,iou_thresh=0.3'].nocls_measures
   >>> import pandas as pd
   >>> print(pd.DataFrame(ub.dict_isect(
   >>>     measures, ['f1', 'g1', 'mcc', 'thresholds',
   >>>                'ppv', 'tpr', 'tnr', 'npv', 'fpr',
   >>>                'tp_count', 'fp_count',
   >>>                'tn_count', 'fn_count'])).iloc[::100])
   >>> # xdoctest: +REQUIRES(module:kwplot)
   >>> # xdoctest: +REQUIRES(--slow)
   >>> results.dump_figures(dpath)
   >>> print('dpath = {!r}'.format(dpath))
   >>> # xdoctest: +REQUIRES(--vd)
   >>> if ub.argflag('--vd') or 1:
   >>>     import xdev
   >>>     xdev.view_directory(dpath)



Module Contents
---------------

Classes
~~~~~~~

.. autoapisummary::

   kwcoco.coco_evaluator.CocoEvalConfig
   kwcoco.coco_evaluator.CocoEvaluator
   kwcoco.coco_evaluator.CocoResults
   kwcoco.coco_evaluator.CocoSingleResult



Functions
~~~~~~~~~

.. autoapisummary::

   kwcoco.coco_evaluator.dmet_area_weights
   kwcoco.coco_evaluator._writefig
   kwcoco.coco_evaluator._load_dets
   kwcoco.coco_evaluator._load_dets_worker



Attributes
~~~~~~~~~~

.. autoapisummary::

   kwcoco.coco_evaluator.profile
   kwcoco.coco_evaluator.COCO_SAMPLER_CLS


.. py:data:: profile
   

   

.. py:data:: COCO_SAMPLER_CLS
   

   

.. py:class:: CocoEvalConfig(data=None, default=None, cmdline=False)

   Bases: :py:obj:`scriptconfig.Config`

   Evaluate and score predicted versus truth detections / classifications in a COCO dataset

   .. py:attribute:: default
      

      

   .. py:method:: normalize(self)

      overloadable function called after each load



.. py:class:: CocoEvaluator(coco_eval, config)

   Bases: :py:obj:`object`

   Abstracts the evaluation process to execute on two coco datasets.

   This can be run as a standalone script where the user specifies the paths
   to the true and predited dataset explicitly, or this can be used by a
   higher level script that produces the predictions and then sends them to
   this evaluator.

   .. rubric:: Example

   >>> from kwcoco.coco_evaluator import CocoEvaluator
   >>> from kwcoco.demo.perterb import perterb_coco
   >>> import kwcoco
   >>> true_dset = kwcoco.CocoDataset.demo('shapes8')
   >>> kwargs = {
   >>>     'box_noise': 0.5,
   >>>     'n_fp': (0, 10),
   >>>     'n_fn': (0, 10),
   >>>     'with_probs': True,
   >>> }
   >>> pred_dset = perterb_coco(true_dset, **kwargs)
   >>> config = {
   >>>     'true_dataset': true_dset,
   >>>     'pred_dataset': pred_dset,
   >>>     'classes_of_interest': [],
   >>> }
   >>> coco_eval = CocoEvaluator(config)
   >>> results = coco_eval.evaluate()

   .. py:attribute:: Config
      

      

   .. py:method:: log(coco_eval, msg, level='INFO')


   .. py:method:: _init(coco_eval)

      Performs initial coercion from given inputs into dictionaries of
      kwimage.Detection objects and attempts to ensure comparable category
      and image ids.


   .. py:method:: _ensure_init(coco_eval)


   .. py:method:: _rectify_classes(coco_eval, true_classes, pred_classes)
      :classmethod:


   .. py:method:: _coerce_dets(CocoEvaluator, dataset, verbose=0, workers=0)
      :classmethod:

      Coerce the input to a mapping from image-id to kwimage.Detection

      Also capture a CocoDataset if possible.

      :returns:     gid_to_det: mapping from gid to dets
                    extra: any extra information we gathered via coercion
      :rtype: Tuple[Dict[int, Detections], Dict]

      .. rubric:: Example

      >>> from kwcoco.coco_evaluator import *  # NOQA
      >>> import kwcoco
      >>> coco_dset = kwcoco.CocoDataset.demo('shapes8')
      >>> gid_to_det, extras = CocoEvaluator._coerce_dets(coco_dset)

      .. rubric:: Example

      >>> # xdoctest: +REQUIRES(module:sqlalchemy)
      >>> from kwcoco.coco_evaluator import *  # NOQA
      >>> import kwcoco
      >>> coco_dset = kwcoco.CocoDataset.demo('shapes8').view_sql()
      >>> gid_to_det, extras = CocoEvaluator._coerce_dets(coco_dset)


   .. py:method:: _build_dmet(coco_eval)

      Builds the detection metrics object

      :returns:

                DetectionMetrics - object that can perform assignment and
                    build confusion vectors.


   .. py:method:: evaluate(coco_eval)

      Executes the main evaluation logic. Performs assignments between
      detections to make DetectionMetrics object, then creates per-item and
      ovr confusion vectors, and performs various threshold-vs-confusion
      analyses.

      :returns:

                container storing (and capable of drawing /
                    serializing) results
      :rtype: CocoResults



.. py:function:: dmet_area_weights(dmet, orig_weights, cfsn_vecs, area_ranges, coco_eval, use_area_attr=False)

   Hacky function to compute confusion vector ignore weights for different
   area thresholds. Needs to be slightly refactored.


.. py:class:: CocoResults(results, resdata=None)

   Bases: :py:obj:`ubelt.NiceRepr`, :py:obj:`kwcoco.metrics.util.DictProxy`

   .. rubric:: CommandLine

   .. code-block:: bash

       xdoctest -m /home/joncrall/code/kwcoco/kwcoco/coco_evaluator.py CocoResults --profile

   .. rubric:: Example

   >>> from kwcoco.coco_evaluator import *  # NOQA
   >>> from kwcoco.coco_evaluator import CocoEvaluator
   >>> import kwcoco
   >>> true_dset = kwcoco.CocoDataset.demo('shapes2')
   >>> from kwcoco.demo.perterb import perterb_coco
   >>> kwargs = {
   >>>     'box_noise': 0.5,
   >>>     'n_fp': (0, 10),
   >>>     'n_fn': (0, 10),
   >>> }
   >>> pred_dset = perterb_coco(true_dset, **kwargs)
   >>> print('true_dset = {!r}'.format(true_dset))
   >>> print('pred_dset = {!r}'.format(pred_dset))
   >>> config = {
   >>>     'true_dataset': true_dset,
   >>>     'pred_dataset': pred_dset,
   >>>     'area_range': ['small'],
   >>>     'iou_thresh': [0.3],
   >>> }
   >>> coco_eval = CocoEvaluator(config)
   >>> results = coco_eval.evaluate()
   >>> # Now we can draw / serialize the results as we please
   >>> dpath = ub.ensure_app_cache_dir('kwcoco/tests/test_out_dpath')
   >>> #
   >>> # test deserialization works
   >>> state = results.__json__()
   >>> self2 = CocoResults.from_json(state)
   >>> #
   >>> # xdoctest: +REQUIRES(module:kwplot)
   >>> results.dump_figures(dpath, figsize=(3, 2), tight=False)  # make this go faster
   >>> results.dump(join(dpath, 'metrics.json'), indent='    ')

   .. py:method:: dump_figures(results, out_dpath, expt_title=None, figsize='auto', tight=True)


   .. py:method:: __json__(results)


   .. py:method:: from_json(cls, state)
      :classmethod:


   .. py:method:: dump(result, file, indent='    ')

      Serialize to json file



.. py:class:: CocoSingleResult(result, nocls_measures, ovr_measures, cfsn_vecs, meta=None)

   Bases: :py:obj:`ubelt.NiceRepr`

   Container class to store, draw, summarize, and serialize results from
   CocoEvaluator.

   .. rubric:: Example

   >>> # xdoctest: +REQUIRES(--slow)
   >>> from kwcoco.coco_evaluator import *  # NOQA
   >>> from kwcoco.coco_evaluator import CocoEvaluator
   >>> import kwcoco
   >>> true_dset = kwcoco.CocoDataset.demo('shapes8')
   >>> from kwcoco.demo.perterb import perterb_coco
   >>> kwargs = {
   >>>     'box_noise': 0.2,
   >>>     'n_fp': (0, 3),
   >>>     'n_fn': (0, 3),
   >>>     'with_probs': False,
   >>> }
   >>> pred_dset = perterb_coco(true_dset, **kwargs)
   >>> print('true_dset = {!r}'.format(true_dset))
   >>> print('pred_dset = {!r}'.format(pred_dset))
   >>> config = {
   >>>     'true_dataset': true_dset,
   >>>     'pred_dataset': pred_dset,
   >>>     'area_range': [(0, 32 ** 2), (32 ** 2, 96 ** 2)],
   >>>     'iou_thresh': [0.3, 0.5, 0.95],
   >>> }
   >>> coco_eval = CocoEvaluator(config)
   >>> results = coco_eval.evaluate()
   >>> result = ub.peek(results.values())
   >>> state = result.__json__()
   >>> print('state = {}'.format(ub.repr2(state, nl=-1)))
   >>> recon = CocoSingleResult.from_json(state)
   >>> state = recon.__json__()
   >>> print('state = {}'.format(ub.repr2(state, nl=-1)))

   .. py:method:: __nice__(result)


   .. py:method:: from_json(cls, state)
      :classmethod:


   .. py:method:: __json__(result)


   .. py:method:: dump(result, file, indent='    ')

      Serialize to json file


   .. py:method:: dump_figures(result, out_dpath, expt_title=None, figsize='auto', tight=True, verbose=1)



.. py:function:: _writefig(fig, metrics_dpath, fname, figsize, verbose, tight)


.. py:function:: _load_dets(pred_fpaths, workers=0)

   .. rubric:: Example

   >>> from kwcoco.coco_evaluator import _load_dets, _load_dets_worker
   >>> import ubelt as ub
   >>> import kwcoco
   >>> from os.path import join
   >>> dpath = ub.ensure_app_cache_dir('kwcoco/tests/load_dets')
   >>> N = 4
   >>> pred_fpaths = []
   >>> for i in range(1, N + 1):
   >>>     dset = kwcoco.CocoDataset.demo('shapes{}'.format(i))
   >>>     dset.fpath = join(dpath, 'shapes_{}.mscoco.json'.format(i))
   >>>     dset.dump(dset.fpath)
   >>>     pred_fpaths.append(dset.fpath)
   >>> dets, coco_dset = _load_dets(pred_fpaths)
   >>> print('dets = {!r}'.format(dets))
   >>> print('coco_dset = {!r}'.format(coco_dset))


.. py:function:: _load_dets_worker(single_pred_fpath, with_coco=True)