Tangledup-ai/sam3_local
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source 
fbshipit-source-id: da6be2f26e3a1202f4bffde8cb980e2dcb851294
This commit is contained in:
facebook-github-bot
2025-11-18 23:07:42 -08:00
commit a13e358df4
504 changed files with 122758 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
sam3/eval/hota_eval_toolkit/trackeval/__init__.py Normal file
View File

@@ -0,0 +1,4 @@
# flake8: noqa
from . import datasets, metrics, utils
from .eval import Evaluator

68
sam3/eval/hota_eval_toolkit/trackeval/_timing.py Normal file
View File

@@ -0,0 +1,68 @@
# flake8: noqa
import inspect
from functools import wraps
from time import perf_counter
DO_TIMING = False
DISPLAY_LESS_PROGRESS = False
timer_dict = {}
counter = 0
def time(f):
@wraps(f)
def wrap(*args, **kw):
if DO_TIMING:
# Run function with timing
ts = perf_counter()
result = f(*args, **kw)
te = perf_counter()
tt = te - ts
# Get function name
arg_names = inspect.getfullargspec(f)[0]
if arg_names[0] == "self" and DISPLAY_LESS_PROGRESS:
return result
elif arg_names[0] == "self":
method_name = type(args[0]).__name__ + "." + f.__name__
else:
method_name = f.__name__
# Record accumulative time in each function for analysis
if method_name in timer_dict.keys():
timer_dict[method_name] += tt
else:
timer_dict[method_name] = tt
# If code is finished, display timing summary
if method_name == "Evaluator.evaluate":
print("")
print("Timing analysis:")
for key, value in timer_dict.items():
print("%-70s %2.4f sec" % (key, value))
else:
# Get function argument values for printing special arguments of interest
arg_titles = ["tracker", "seq", "cls"]
arg_vals = []
for i, a in enumerate(arg_names):
if a in arg_titles:
arg_vals.append(args[i])
arg_text = "(" + ", ".join(arg_vals) + ")"
# Display methods and functions with different indentation.
if arg_names[0] == "self":
print("%-74s %2.4f sec" % (" " * 4 + method_name + arg_text, tt))
elif arg_names[0] == "test":
pass
else:
global counter
counter += 1
print("%i %-70s %2.4f sec" % (counter, method_name + arg_text, tt))
return result
else:
# If config["TIME_PROGRESS"] is false, or config["USE_PARALLEL"] is true, run functions normally without timing.
return f(*args, **kw)
return wrap

4
sam3/eval/hota_eval_toolkit/trackeval/datasets/__init__.py Normal file
View File

@@ -0,0 +1,4 @@
# flake8: noqa
from .tao_ow import TAO_OW
from .youtube_vis import YouTubeVIS

379
sam3/eval/hota_eval_toolkit/trackeval/datasets/_base_dataset.py Normal file
View File

@@ -0,0 +1,379 @@
# flake8: noqa
import csv
import io
import os
import traceback
import zipfile
from abc import ABC, abstractmethod
from copy import deepcopy
import numpy as np
from .. import _timing
from ..utils import TrackEvalException
class _BaseDataset(ABC):
@abstractmethod
def __init__(self):
self.tracker_list = None
self.seq_list = None
self.class_list = None
self.output_fol = None
self.output_sub_fol = None
self.should_classes_combine = True
self.use_super_categories = False
# Functions to implement:
@staticmethod
@abstractmethod
def get_default_dataset_config(): ...
@abstractmethod
def _load_raw_file(self, tracker, seq, is_gt): ...
@_timing.time
@abstractmethod
def get_preprocessed_seq_data(self, raw_data, cls): ...
@abstractmethod
def _calculate_similarities(self, gt_dets_t, tracker_dets_t): ...
# Helper functions for all datasets:
@classmethod
def get_class_name(cls):
return cls.__name__
def get_name(self):
return self.get_class_name()
def get_output_fol(self, tracker):
return os.path.join(self.output_fol, tracker, self.output_sub_fol)
def get_display_name(self, tracker):
"""Can be overwritten if the trackers name (in files) is different to how it should be displayed.
By default this method just returns the trackers name as is.
"""
return tracker
def get_eval_info(self):
"""Return info about the dataset needed for the Evaluator"""
return self.tracker_list, self.seq_list, self.class_list
@_timing.time
def get_raw_seq_data(self, tracker, seq):
"""Loads raw data (tracker and ground-truth) for a single tracker on a single sequence.
Raw data includes all of the information needed for both preprocessing and evaluation, for all classes.
A later function (get_processed_seq_data) will perform such preprocessing and extract relevant information for
the evaluation of each class.
This returns a dict which contains the fields:
[num_timesteps]: integer
[gt_ids, tracker_ids, gt_classes, tracker_classes, tracker_confidences]:
list (for each timestep) of 1D NDArrays (for each det).
[gt_dets, tracker_dets, gt_crowd_ignore_regions]: list (for each timestep) of lists of detections.
[similarity_scores]: list (for each timestep) of 2D NDArrays.
[gt_extras]: dict (for each extra) of lists (for each timestep) of 1D NDArrays (for each det).
gt_extras contains dataset specific information used for preprocessing such as occlusion and truncation levels.
Note that similarities are extracted as part of the dataset and not the metric, because almost all metrics are
independent of the exact method of calculating the similarity. However datasets are not (e.g. segmentation
masks vs 2D boxes vs 3D boxes).
We calculate the similarity before preprocessing because often both preprocessing and evaluation require it and
we don't wish to calculate this twice.
We calculate similarity between all gt and tracker classes (not just each class individually) to allow for
calculation of metrics such as class confusion matrices. Typically the impact of this on performance is low.
"""
# Load raw data.
raw_gt_data = self._load_raw_file(tracker, seq, is_gt=True)
raw_tracker_data = self._load_raw_file(tracker, seq, is_gt=False)
raw_data = {**raw_tracker_data, **raw_gt_data} # Merges dictionaries
# Calculate similarities for each timestep.
similarity_scores = []
for t, (gt_dets_t, tracker_dets_t) in enumerate(
zip(raw_data["gt_dets"], raw_data["tracker_dets"])
):
ious = self._calculate_similarities(gt_dets_t, tracker_dets_t)
similarity_scores.append(ious)
raw_data["similarity_scores"] = similarity_scores
return raw_data
@staticmethod
def _load_simple_text_file(
file,
time_col=0,
id_col=None,
remove_negative_ids=False,
valid_filter=None,
crowd_ignore_filter=None,
convert_filter=None,
is_zipped=False,
zip_file=None,
force_delimiters=None,
):
"""Function that loads data which is in a commonly used text file format.
Assumes each det is given by one row of a text file.
There is no limit to the number or meaning of each column,
however one column needs to give the timestep of each det (time_col) which is default col 0.
The file dialect (deliminator, num cols, etc) is determined automatically.
This function automatically separates dets by timestep,
and is much faster than alternatives such as np.loadtext or pandas.
If remove_negative_ids is True and id_col is not None, dets with negative values in id_col are excluded.
These are not excluded from ignore data.
valid_filter can be used to only include certain classes.
It is a dict with ints as keys, and lists as values,
such that a row is included if "row[key].lower() is in value" for all key/value pairs in the dict.
If None, all classes are included.
crowd_ignore_filter can be used to read crowd_ignore regions separately. It has the same format as valid filter.
convert_filter can be used to convert value read to another format.
This is used most commonly to convert classes given as string to a class id.
This is a dict such that the key is the column to convert, and the value is another dict giving the mapping.
Optionally, input files could be a zip of multiple text files for storage efficiency.
Returns read_data and ignore_data.
Each is a dict (with keys as timesteps as strings) of lists (over dets) of lists (over column values).
Note that all data is returned as strings, and must be converted to float/int later if needed.
Note that timesteps will not be present in the returned dict keys if there are no dets for them
"""
if remove_negative_ids and id_col is None:
raise TrackEvalException(
"remove_negative_ids is True, but id_col is not given."
)
if crowd_ignore_filter is None:
crowd_ignore_filter = {}
if convert_filter is None:
convert_filter = {}
try:
if is_zipped: # Either open file directly or within a zip.
if zip_file is None:
raise TrackEvalException(
"is_zipped set to True, but no zip_file is given."
)
archive = zipfile.ZipFile(os.path.join(zip_file), "r")
fp = io.TextIOWrapper(archive.open(file, "r"))
else:
fp = open(file)
read_data = {}
crowd_ignore_data = {}
fp.seek(0, os.SEEK_END)
# check if file is empty
if fp.tell():
fp.seek(0)
dialect = csv.Sniffer().sniff(
fp.readline(), delimiters=force_delimiters
) # Auto determine structure.
dialect.skipinitialspace = (
True # Deal with extra spaces between columns
)
fp.seek(0)
reader = csv.reader(fp, dialect)
for row in reader:
try:
# Deal with extra trailing spaces at the end of rows
if row[-1] in "":
row = row[:-1]
timestep = str(int(float(row[time_col])))
# Read ignore regions separately.
is_ignored = False
for ignore_key, ignore_value in crowd_ignore_filter.items():
if row[ignore_key].lower() in ignore_value:
# Convert values in one column (e.g. string to id)
for (
convert_key,
convert_value,
) in convert_filter.items():
row[convert_key] = convert_value[
row[convert_key].lower()
]
# Save data separated by timestep.
if timestep in crowd_ignore_data.keys():
crowd_ignore_data[timestep].append(row)
else:
crowd_ignore_data[timestep] = [row]
is_ignored = True
if (
is_ignored
): # if det is an ignore region, it cannot be a normal det.
continue
# Exclude some dets if not valid.
if valid_filter is not None:
for key, value in valid_filter.items():
if row[key].lower() not in value:
continue
if remove_negative_ids:
if int(float(row[id_col])) < 0:
continue
# Convert values in one column (e.g. string to id)
for convert_key, convert_value in convert_filter.items():
row[convert_key] = convert_value[row[convert_key].lower()]
# Save data separated by timestep.
if timestep in read_data.keys():
read_data[timestep].append(row)
else:
read_data[timestep] = [row]
except Exception:
exc_str_init = (
"In file %s the following line cannot be read correctly: \n"
% os.path.basename(file)
)
exc_str = " ".join([exc_str_init] + row)
raise TrackEvalException(exc_str)
fp.close()
except Exception:
print("Error loading file: %s, printing traceback." % file)
traceback.print_exc()
raise TrackEvalException(
"File %s cannot be read because it is either not present or invalidly formatted"
% os.path.basename(file)
)
return read_data, crowd_ignore_data
@staticmethod
def _calculate_mask_ious(masks1, masks2, is_encoded=False, do_ioa=False):
"""Calculates the IOU (intersection over union) between two arrays of segmentation masks.
If is_encoded a run length encoding with pycocotools is assumed as input format, otherwise an input of numpy
arrays of the shape (num_masks, height, width) is assumed and the encoding is performed.
If do_ioa (intersection over area) , then calculates the intersection over the area of masks1 - this is commonly
used to determine if detections are within crowd ignore region.
:param masks1: first set of masks (numpy array of shape (num_masks, height, width) if not encoded,
else pycocotools rle encoded format)
:param masks2: second set of masks (numpy array of shape (num_masks, height, width) if not encoded,
else pycocotools rle encoded format)
:param is_encoded: whether the input is in pycocotools rle encoded format
:param do_ioa: whether to perform IoA computation
:return: the IoU/IoA scores
"""
# Only loaded when run to reduce minimum requirements
from pycocotools import mask as mask_utils
# use pycocotools for run length encoding of masks
if not is_encoded:
masks1 = mask_utils.encode(
np.array(np.transpose(masks1, (1, 2, 0)), order="F")
)
masks2 = mask_utils.encode(
np.array(np.transpose(masks2, (1, 2, 0)), order="F")
)
# use pycocotools for iou computation of rle encoded masks
ious = mask_utils.iou(masks1, masks2, [do_ioa] * len(masks2))
if len(masks1) == 0 or len(masks2) == 0:
ious = np.asarray(ious).reshape(len(masks1), len(masks2))
assert (ious >= 0 - np.finfo("float").eps).all()
assert (ious <= 1 + np.finfo("float").eps).all()
return ious
@staticmethod
def _calculate_box_ious(bboxes1, bboxes2, box_format="xywh", do_ioa=False):
"""Calculates the IOU (intersection over union) between two arrays of boxes.
Allows variable box formats ('xywh' and 'x0y0x1y1').
If do_ioa (intersection over area) , then calculates the intersection over the area of boxes1 - this is commonly
used to determine if detections are within crowd ignore region.
"""
if box_format in "xywh":
# layout: (x0, y0, w, h)
bboxes1 = deepcopy(bboxes1)
bboxes2 = deepcopy(bboxes2)
bboxes1[:, 2] = bboxes1[:, 0] + bboxes1[:, 2]
bboxes1[:, 3] = bboxes1[:, 1] + bboxes1[:, 3]
bboxes2[:, 2] = bboxes2[:, 0] + bboxes2[:, 2]
bboxes2[:, 3] = bboxes2[:, 1] + bboxes2[:, 3]
elif box_format not in "x0y0x1y1":
raise (TrackEvalException("box_format %s is not implemented" % box_format))
# layout: (x0, y0, x1, y1)
min_ = np.minimum(bboxes1[:, np.newaxis, :], bboxes2[np.newaxis, :, :])
max_ = np.maximum(bboxes1[:, np.newaxis, :], bboxes2[np.newaxis, :, :])
intersection = np.maximum(min_[..., 2] - max_[..., 0], 0) * np.maximum(
min_[..., 3] - max_[..., 1], 0
)
area1 = (bboxes1[..., 2] - bboxes1[..., 0]) * (
bboxes1[..., 3] - bboxes1[..., 1]
)
if do_ioa:
ioas = np.zeros_like(intersection)
valid_mask = area1 > 0 + np.finfo("float").eps
ioas[valid_mask, :] = (
intersection[valid_mask, :] / area1[valid_mask][:, np.newaxis]
)
return ioas
else:
area2 = (bboxes2[..., 2] - bboxes2[..., 0]) * (
bboxes2[..., 3] - bboxes2[..., 1]
)
union = area1[:, np.newaxis] + area2[np.newaxis, :] - intersection
intersection[area1 <= 0 + np.finfo("float").eps, :] = 0
intersection[:, area2 <= 0 + np.finfo("float").eps] = 0
intersection[union <= 0 + np.finfo("float").eps] = 0
union[union <= 0 + np.finfo("float").eps] = 1
ious = intersection / union
return ious
@staticmethod
def _calculate_euclidean_similarity(dets1, dets2, zero_distance=2.0):
"""Calculates the euclidean distance between two sets of detections, and then converts this into a similarity
measure with values between 0 and 1 using the following formula: sim = max(0, 1 - dist/zero_distance).
The default zero_distance of 2.0, corresponds to the default used in MOT15_3D, such that a 0.5 similarity
threshold corresponds to a 1m distance threshold for TPs.
"""
dist = np.linalg.norm(dets1[:, np.newaxis] - dets2[np.newaxis, :], axis=2)
sim = np.maximum(0, 1 - dist / zero_distance)
return sim
@staticmethod
def _check_unique_ids(data, after_preproc=False):
"""Check the requirement that the tracker_ids and gt_ids are unique per timestep"""
gt_ids = data["gt_ids"]
tracker_ids = data["tracker_ids"]
for t, (gt_ids_t, tracker_ids_t) in enumerate(zip(gt_ids, tracker_ids)):
if len(tracker_ids_t) > 0:
unique_ids, counts = np.unique(tracker_ids_t, return_counts=True)
if np.max(counts) != 1:
duplicate_ids = unique_ids[counts > 1]
exc_str_init = (
"Tracker predicts the same ID more than once in a single timestep "
"(seq: %s, frame: %i, ids:" % (data["seq"], t + 1)
)
exc_str = (
" ".join([exc_str_init] + [str(d) for d in duplicate_ids]) + ")"
)
if after_preproc:
exc_str_init += (
"\n Note that this error occurred after preprocessing (but not before), "
"so ids may not be as in file, and something seems wrong with preproc."
)
raise TrackEvalException(exc_str)
if len(gt_ids_t) > 0:
unique_ids, counts = np.unique(gt_ids_t, return_counts=True)
if np.max(counts) != 1:
duplicate_ids = unique_ids[counts > 1]
exc_str_init = (
"Ground-truth has the same ID more than once in a single timestep "
"(seq: %s, frame: %i, ids:" % (data["seq"], t + 1)
)
exc_str = (
" ".join([exc_str_init] + [str(d) for d in duplicate_ids]) + ")"
)
if after_preproc:
exc_str_init += (
"\n Note that this error occurred after preprocessing (but not before), "
"so ids may not be as in file, and something seems wrong with preproc."
)
raise TrackEvalException(exc_str)

891
sam3/eval/hota_eval_toolkit/trackeval/datasets/tao_ow.py Normal file
View File

@@ -0,0 +1,891 @@
# flake8: noqa
import itertools
import json
import os
from collections import defaultdict
import numpy as np
from scipy.optimize import linear_sum_assignment
from .. import _timing, utils
from ..utils import TrackEvalException
from ._base_dataset import _BaseDataset
class TAO_OW(_BaseDataset):
"""Dataset class for TAO tracking"""
@staticmethod
def get_default_dataset_config():
"""Default class config values"""
code_path = utils.get_code_path()
default_config = {
"GT_FOLDER": os.path.join(
code_path, "data/gt/tao/tao_training"
), # Location of GT data
"TRACKERS_FOLDER": os.path.join(
code_path, "data/trackers/tao/tao_training"
), # Trackers location
"OUTPUT_FOLDER": None, # Where to save eval results (if None, same as TRACKERS_FOLDER)
"TRACKERS_TO_EVAL": None, # Filenames of trackers to eval (if None, all in folder)
"CLASSES_TO_EVAL": None, # Classes to eval (if None, all classes)
"SPLIT_TO_EVAL": "training", # Valid: 'training', 'val'
"PRINT_CONFIG": True, # Whether to print current config
"TRACKER_SUB_FOLDER": "data", # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER
"OUTPUT_SUB_FOLDER": "", # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER
"TRACKER_DISPLAY_NAMES": None, # Names of trackers to display, if None: TRACKERS_TO_EVAL
"MAX_DETECTIONS": 300, # Number of maximal allowed detections per image (0 for unlimited)
"SUBSET": "all",
}
return default_config
def __init__(self, config=None):
"""Initialise dataset, checking that all required files are present"""
super().__init__()
# Fill non-given config values with defaults
self.config = utils.init_config(
config, self.get_default_dataset_config(), self.get_name()
)
self.gt_fol = self.config["GT_FOLDER"]
self.tracker_fol = self.config["TRACKERS_FOLDER"]
self.should_classes_combine = True
self.use_super_categories = False
self.tracker_sub_fol = self.config["TRACKER_SUB_FOLDER"]
self.output_fol = self.config["OUTPUT_FOLDER"]
if self.output_fol is None:
self.output_fol = self.tracker_fol
self.output_sub_fol = self.config["OUTPUT_SUB_FOLDER"]
gt_dir_files = [
file for file in os.listdir(self.gt_fol) if file.endswith(".json")
]
if len(gt_dir_files) != 1:
raise TrackEvalException(
self.gt_fol + " does not contain exactly one json file."
)
with open(os.path.join(self.gt_fol, gt_dir_files[0])) as f:
self.gt_data = json.load(f)
self.subset = self.config["SUBSET"]
if self.subset != "all":
# Split GT data into `known`, `unknown` or `distractor`
self._split_known_unknown_distractor()
self.gt_data = self._filter_gt_data(self.gt_data)
# merge categories marked with a merged tag in TAO dataset
self._merge_categories(self.gt_data["annotations"] + self.gt_data["tracks"])
# Get sequences to eval and sequence information
self.seq_list = [
vid["name"].replace("/", "-") for vid in self.gt_data["videos"]
]
self.seq_name_to_seq_id = {
vid["name"].replace("/", "-"): vid["id"] for vid in self.gt_data["videos"]
}
# compute mappings from videos to annotation data
self.videos_to_gt_tracks, self.videos_to_gt_images = self._compute_vid_mappings(
self.gt_data["annotations"]
)
# compute sequence lengths
self.seq_lengths = {vid["id"]: 0 for vid in self.gt_data["videos"]}
for img in self.gt_data["images"]:
self.seq_lengths[img["video_id"]] += 1
self.seq_to_images_to_timestep = self._compute_image_to_timestep_mappings()
self.seq_to_classes = {
vid["id"]: {
"pos_cat_ids": list(
{
track["category_id"]
for track in self.videos_to_gt_tracks[vid["id"]]
}
),
"neg_cat_ids": vid["neg_category_ids"],
"not_exhaustively_labeled_cat_ids": vid["not_exhaustive_category_ids"],
}
for vid in self.gt_data["videos"]
}
# Get classes to eval
considered_vid_ids = [self.seq_name_to_seq_id[vid] for vid in self.seq_list]
seen_cats = set(
[
cat_id
for vid_id in considered_vid_ids
for cat_id in self.seq_to_classes[vid_id]["pos_cat_ids"]
]
)
# only classes with ground truth are evaluated in TAO
self.valid_classes = [
cls["name"] for cls in self.gt_data["categories"] if cls["id"] in seen_cats
]
# cls_name_to_cls_id_map = {cls['name']: cls['id'] for cls in self.gt_data['categories']}
if self.config["CLASSES_TO_EVAL"]:
# self.class_list = [cls.lower() if cls.lower() in self.valid_classes else None
# for cls in self.config['CLASSES_TO_EVAL']]
self.class_list = ["object"] # class-agnostic
if not all(self.class_list):
raise TrackEvalException(
"Attempted to evaluate an invalid class. Only classes "
+ ", ".join(self.valid_classes)
+ " are valid (classes present in ground truth data)."
)
else:
# self.class_list = [cls for cls in self.valid_classes]
self.class_list = ["object"] # class-agnostic
# self.class_name_to_class_id = {k: v for k, v in cls_name_to_cls_id_map.items() if k in self.class_list}
self.class_name_to_class_id = {"object": 1} # class-agnostic
# Get trackers to eval
if self.config["TRACKERS_TO_EVAL"] is None:
self.tracker_list = os.listdir(self.tracker_fol)
else:
self.tracker_list = self.config["TRACKERS_TO_EVAL"]
if self.config["TRACKER_DISPLAY_NAMES"] is None:
self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list))
elif (self.config["TRACKERS_TO_EVAL"] is not None) and (
len(self.config["TRACKER_DISPLAY_NAMES"]) == len(self.tracker_list)
):
self.tracker_to_disp = dict(
zip(self.tracker_list, self.config["TRACKER_DISPLAY_NAMES"])
)
else:
raise TrackEvalException(
"List of tracker files and tracker display names do not match."
)
self.tracker_data = {tracker: dict() for tracker in self.tracker_list}
for tracker in self.tracker_list:
tr_dir_files = [
file
for file in os.listdir(
os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol)
)
if file.endswith(".json")
]
if len(tr_dir_files) != 1:
raise TrackEvalException(
os.path.join(self.tracker_fol, tracker, self.tracker_sub_fol)
+ " does not contain exactly one json file."
)
with open(
os.path.join(
self.tracker_fol, tracker, self.tracker_sub_fol, tr_dir_files[0]
)
) as f:
curr_data = json.load(f)
# limit detections if MAX_DETECTIONS > 0
if self.config["MAX_DETECTIONS"]:
curr_data = self._limit_dets_per_image(curr_data)
# fill missing video ids
self._fill_video_ids_inplace(curr_data)
# make track ids unique over whole evaluation set
self._make_track_ids_unique(curr_data)
# merge categories marked with a merged tag in TAO dataset
self._merge_categories(curr_data)
# get tracker sequence information
curr_videos_to_tracker_tracks, curr_videos_to_tracker_images = (
self._compute_vid_mappings(curr_data)
)
self.tracker_data[tracker]["vids_to_tracks"] = curr_videos_to_tracker_tracks
self.tracker_data[tracker]["vids_to_images"] = curr_videos_to_tracker_images
def get_display_name(self, tracker):
return self.tracker_to_disp[tracker]
def _load_raw_file(self, tracker, seq, is_gt):
"""Load a file (gt or tracker) in the TAO format
If is_gt, this returns a dict which contains the fields:
[gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det).
[gt_dets]: list (for each timestep) of lists of detections.
[classes_to_gt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as
keys and corresponding segmentations as values) for each track
[classes_to_gt_track_ids, classes_to_gt_track_areas, classes_to_gt_track_lengths]: dictionary with class values
as keys and lists (for each track) as values
if not is_gt, this returns a dict which contains the fields:
[tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det).
[tracker_dets]: list (for each timestep) of lists of detections.
[classes_to_dt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as
keys and corresponding segmentations as values) for each track
[classes_to_dt_track_ids, classes_to_dt_track_areas, classes_to_dt_track_lengths]: dictionary with class values
as keys and lists as values
[classes_to_dt_track_scores]: dictionary with class values as keys and 1D numpy arrays as values
"""
seq_id = self.seq_name_to_seq_id[seq]
# File location
if is_gt:
imgs = self.videos_to_gt_images[seq_id]
else:
imgs = self.tracker_data[tracker]["vids_to_images"][seq_id]
# Convert data to required format
num_timesteps = self.seq_lengths[seq_id]
img_to_timestep = self.seq_to_images_to_timestep[seq_id]
data_keys = ["ids", "classes", "dets"]
if not is_gt:
data_keys += ["tracker_confidences"]
raw_data = {key: [None] * num_timesteps for key in data_keys}
for img in imgs:
# some tracker data contains images without any ground truth information, these are ignored
try:
t = img_to_timestep[img["id"]]
except KeyError:
continue
annotations = img["annotations"]
raw_data["dets"][t] = np.atleast_2d(
[ann["bbox"] for ann in annotations]
).astype(float)
raw_data["ids"][t] = np.atleast_1d(
[ann["track_id"] for ann in annotations]
).astype(int)
raw_data["classes"][t] = np.atleast_1d([1 for _ in annotations]).astype(
int
) # class-agnostic
if not is_gt:
raw_data["tracker_confidences"][t] = np.atleast_1d(
[ann["score"] for ann in annotations]
).astype(float)
for t, d in enumerate(raw_data["dets"]):
if d is None:
raw_data["dets"][t] = np.empty((0, 4)).astype(float)
raw_data["ids"][t] = np.empty(0).astype(int)
raw_data["classes"][t] = np.empty(0).astype(int)
if not is_gt:
raw_data["tracker_confidences"][t] = np.empty(0)
if is_gt:
key_map = {"ids": "gt_ids", "classes": "gt_classes", "dets": "gt_dets"}
else:
key_map = {
"ids": "tracker_ids",
"classes": "tracker_classes",
"dets": "tracker_dets",
}
for k, v in key_map.items():
raw_data[v] = raw_data.pop(k)
# all_classes = [self.class_name_to_class_id[cls] for cls in self.class_list]
all_classes = [1] # class-agnostic
if is_gt:
classes_to_consider = all_classes
all_tracks = self.videos_to_gt_tracks[seq_id]
else:
# classes_to_consider = self.seq_to_classes[seq_id]['pos_cat_ids'] \
# + self.seq_to_classes[seq_id]['neg_cat_ids']
classes_to_consider = all_classes # class-agnostic
all_tracks = self.tracker_data[tracker]["vids_to_tracks"][seq_id]
# classes_to_tracks = {cls: [track for track in all_tracks if track['category_id'] == cls]
# if cls in classes_to_consider else [] for cls in all_classes}
classes_to_tracks = {
cls: [track for track in all_tracks] if cls in classes_to_consider else []
for cls in all_classes
} # class-agnostic
# mapping from classes to track information
raw_data["classes_to_tracks"] = {
cls: [
{
det["image_id"]: np.atleast_1d(det["bbox"])
for det in track["annotations"]
}
for track in tracks
]
for cls, tracks in classes_to_tracks.items()
}
raw_data["classes_to_track_ids"] = {
cls: [track["id"] for track in tracks]
for cls, tracks in classes_to_tracks.items()
}
raw_data["classes_to_track_areas"] = {
cls: [track["area"] for track in tracks]
for cls, tracks in classes_to_tracks.items()
}
raw_data["classes_to_track_lengths"] = {
cls: [len(track["annotations"]) for track in tracks]
for cls, tracks in classes_to_tracks.items()
}
if not is_gt:
raw_data["classes_to_dt_track_scores"] = {
cls: np.array(
[
np.mean([float(x["score"]) for x in track["annotations"]])
for track in tracks
]
)
for cls, tracks in classes_to_tracks.items()
}
if is_gt:
key_map = {
"classes_to_tracks": "classes_to_gt_tracks",
"classes_to_track_ids": "classes_to_gt_track_ids",
"classes_to_track_lengths": "classes_to_gt_track_lengths",
"classes_to_track_areas": "classes_to_gt_track_areas",
}
else:
key_map = {
"classes_to_tracks": "classes_to_dt_tracks",
"classes_to_track_ids": "classes_to_dt_track_ids",
"classes_to_track_lengths": "classes_to_dt_track_lengths",
"classes_to_track_areas": "classes_to_dt_track_areas",
}
for k, v in key_map.items():
raw_data[v] = raw_data.pop(k)
raw_data["num_timesteps"] = num_timesteps
raw_data["neg_cat_ids"] = self.seq_to_classes[seq_id]["neg_cat_ids"]
raw_data["not_exhaustively_labeled_cls"] = self.seq_to_classes[seq_id][
"not_exhaustively_labeled_cat_ids"
]
raw_data["seq"] = seq
return raw_data
@_timing.time
def get_preprocessed_seq_data(self, raw_data, cls):
"""Preprocess data for a single sequence for a single class ready for evaluation.
Inputs:
- raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data().
- cls is the class to be evaluated.
Outputs:
- data is a dict containing all of the information that metrics need to perform evaluation.
It contains the following fields:
[num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers.
[gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det).
[gt_dets, tracker_dets]: list (for each timestep) of lists of detections.
[similarity_scores]: list (for each timestep) of 2D NDArrays.
Notes:
General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps.
1) Extract only detections relevant for the class to be evaluated (including distractor detections).
2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a
distractor class, or otherwise marked as to be removed.
3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain
other criteria (e.g. are too small).
4) Remove gt dets that were only useful for preprocessing and not for actual evaluation.
After the above preprocessing steps, this function also calculates the number of gt and tracker detections
and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are
unique within each timestep.
TAO:
In TAO, the 4 preproc steps are as follow:
1) All classes present in the ground truth data are evaluated separately.
2) No matched tracker detections are removed.
3) Unmatched tracker detections are removed if there is not ground truth data and the class does not
belong to the categories marked as negative for this sequence. Additionally, unmatched tracker
detections for classes which are marked as not exhaustively labeled are removed.
4) No gt detections are removed.
Further, for TrackMAP computation track representations for the given class are accessed from a dictionary
and the tracks from the tracker data are sorted according to the tracker confidence.
"""
cls_id = self.class_name_to_class_id[cls]
is_not_exhaustively_labeled = cls_id in raw_data["not_exhaustively_labeled_cls"]
is_neg_category = cls_id in raw_data["neg_cat_ids"]
data_keys = [
"gt_ids",
"tracker_ids",
"gt_dets",
"tracker_dets",
"tracker_confidences",
"similarity_scores",
]
data = {key: [None] * raw_data["num_timesteps"] for key in data_keys}
unique_gt_ids = []
unique_tracker_ids = []
num_gt_dets = 0
num_tracker_dets = 0
for t in range(raw_data["num_timesteps"]):
# Only extract relevant dets for this class for preproc and eval (cls)
gt_class_mask = np.atleast_1d(raw_data["gt_classes"][t] == cls_id)
gt_class_mask = gt_class_mask.astype(bool)
gt_ids = raw_data["gt_ids"][t][gt_class_mask]
gt_dets = raw_data["gt_dets"][t][gt_class_mask]
tracker_class_mask = np.atleast_1d(raw_data["tracker_classes"][t] == cls_id)
tracker_class_mask = tracker_class_mask.astype(bool)
tracker_ids = raw_data["tracker_ids"][t][tracker_class_mask]
tracker_dets = raw_data["tracker_dets"][t][tracker_class_mask]
tracker_confidences = raw_data["tracker_confidences"][t][tracker_class_mask]
similarity_scores = raw_data["similarity_scores"][t][gt_class_mask, :][
:, tracker_class_mask
]
# Match tracker and gt dets (with hungarian algorithm).
unmatched_indices = np.arange(tracker_ids.shape[0])
if gt_ids.shape[0] > 0 and tracker_ids.shape[0] > 0:
matching_scores = similarity_scores.copy()
matching_scores[matching_scores < 0.5 - np.finfo("float").eps] = 0
match_rows, match_cols = linear_sum_assignment(-matching_scores)
actually_matched_mask = (
matching_scores[match_rows, match_cols] > 0 + np.finfo("float").eps
)
match_cols = match_cols[actually_matched_mask]
unmatched_indices = np.delete(unmatched_indices, match_cols, axis=0)
if gt_ids.shape[0] == 0 and not is_neg_category:
to_remove_tracker = unmatched_indices
elif is_not_exhaustively_labeled:
to_remove_tracker = unmatched_indices
else:
to_remove_tracker = np.array([], dtype=int)
# remove all unwanted unmatched tracker detections
data["tracker_ids"][t] = np.delete(tracker_ids, to_remove_tracker, axis=0)
data["tracker_dets"][t] = np.delete(tracker_dets, to_remove_tracker, axis=0)
data["tracker_confidences"][t] = np.delete(
tracker_confidences, to_remove_tracker, axis=0
)
similarity_scores = np.delete(similarity_scores, to_remove_tracker, axis=1)
data["gt_ids"][t] = gt_ids
data["gt_dets"][t] = gt_dets
data["similarity_scores"][t] = similarity_scores
unique_gt_ids += list(np.unique(data["gt_ids"][t]))
unique_tracker_ids += list(np.unique(data["tracker_ids"][t]))
num_tracker_dets += len(data["tracker_ids"][t])
num_gt_dets += len(data["gt_ids"][t])
# Re-label IDs such that there are no empty IDs
if len(unique_gt_ids) > 0:
unique_gt_ids = np.unique(unique_gt_ids)
gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1))
gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids))
for t in range(raw_data["num_timesteps"]):
if len(data["gt_ids"][t]) > 0:
data["gt_ids"][t] = gt_id_map[data["gt_ids"][t]].astype(int)
if len(unique_tracker_ids) > 0:
unique_tracker_ids = np.unique(unique_tracker_ids)
tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1))
tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids))
for t in range(raw_data["num_timesteps"]):
if len(data["tracker_ids"][t]) > 0:
data["tracker_ids"][t] = tracker_id_map[
data["tracker_ids"][t]
].astype(int)
# Record overview statistics.
data["num_tracker_dets"] = num_tracker_dets
data["num_gt_dets"] = num_gt_dets
data["num_tracker_ids"] = len(unique_tracker_ids)
data["num_gt_ids"] = len(unique_gt_ids)
data["num_timesteps"] = raw_data["num_timesteps"]
data["seq"] = raw_data["seq"]
# get track representations
data["gt_tracks"] = raw_data["classes_to_gt_tracks"][cls_id]
data["gt_track_ids"] = raw_data["classes_to_gt_track_ids"][cls_id]
data["gt_track_lengths"] = raw_data["classes_to_gt_track_lengths"][cls_id]
data["gt_track_areas"] = raw_data["classes_to_gt_track_areas"][cls_id]
data["dt_tracks"] = raw_data["classes_to_dt_tracks"][cls_id]
data["dt_track_ids"] = raw_data["classes_to_dt_track_ids"][cls_id]
data["dt_track_lengths"] = raw_data["classes_to_dt_track_lengths"][cls_id]
data["dt_track_areas"] = raw_data["classes_to_dt_track_areas"][cls_id]
data["dt_track_scores"] = raw_data["classes_to_dt_track_scores"][cls_id]
data["not_exhaustively_labeled"] = is_not_exhaustively_labeled
data["iou_type"] = "bbox"
# sort tracker data tracks by tracker confidence scores
if data["dt_tracks"]:
idx = np.argsort(
[-score for score in data["dt_track_scores"]], kind="mergesort"
)
data["dt_track_scores"] = [data["dt_track_scores"][i] for i in idx]
data["dt_tracks"] = [data["dt_tracks"][i] for i in idx]
data["dt_track_ids"] = [data["dt_track_ids"][i] for i in idx]
data["dt_track_lengths"] = [data["dt_track_lengths"][i] for i in idx]
data["dt_track_areas"] = [data["dt_track_areas"][i] for i in idx]
# Ensure that ids are unique per timestep.
self._check_unique_ids(data)
return data
def _calculate_similarities(self, gt_dets_t, tracker_dets_t):
similarity_scores = self._calculate_box_ious(gt_dets_t, tracker_dets_t)
return similarity_scores
def _merge_categories(self, annotations):
"""
Merges categories with a merged tag. Adapted from https://github.com/TAO-Dataset
:param annotations: the annotations in which the classes should be merged
:return: None
"""
merge_map = {}
for category in self.gt_data["categories"]:
if "merged" in category:
for to_merge in category["merged"]:
merge_map[to_merge["id"]] = category["id"]
for ann in annotations:
ann["category_id"] = merge_map.get(ann["category_id"], ann["category_id"])
def _compute_vid_mappings(self, annotations):
"""
Computes mappings from Videos to corresponding tracks and images.
:param annotations: the annotations for which the mapping should be generated
:return: the video-to-track-mapping, the video-to-image-mapping
"""
vids_to_tracks = {}
vids_to_imgs = {}
vid_ids = [vid["id"] for vid in self.gt_data["videos"]]
# compute an mapping from image IDs to images
images = {}
for image in self.gt_data["images"]:
images[image["id"]] = image
for ann in annotations:
ann["area"] = ann["bbox"][2] * ann["bbox"][3]
vid = ann["video_id"]
if ann["video_id"] not in vids_to_tracks.keys():
vids_to_tracks[ann["video_id"]] = list()
if ann["video_id"] not in vids_to_imgs.keys():
vids_to_imgs[ann["video_id"]] = list()
# Fill in vids_to_tracks
tid = ann["track_id"]
exist_tids = [track["id"] for track in vids_to_tracks[vid]]
try:
index1 = exist_tids.index(tid)
except ValueError:
index1 = -1
if tid not in exist_tids:
curr_track = {
"id": tid,
"category_id": ann["category_id"],
"video_id": vid,
"annotations": [ann],
}
vids_to_tracks[vid].append(curr_track)
else:
vids_to_tracks[vid][index1]["annotations"].append(ann)
# Fill in vids_to_imgs
img_id = ann["image_id"]
exist_img_ids = [img["id"] for img in vids_to_imgs[vid]]
try:
index2 = exist_img_ids.index(img_id)
except ValueError:
index2 = -1
if index2 == -1:
curr_img = {"id": img_id, "annotations": [ann]}
vids_to_imgs[vid].append(curr_img)
else:
vids_to_imgs[vid][index2]["annotations"].append(ann)
# sort annotations by frame index and compute track area
for vid, tracks in vids_to_tracks.items():
for track in tracks:
track["annotations"] = sorted(
track["annotations"],
key=lambda x: images[x["image_id"]]["frame_index"],
)
# Computer average area
track["area"] = sum(x["area"] for x in track["annotations"]) / len(
track["annotations"]
)
# Ensure all videos are present
for vid_id in vid_ids:
if vid_id not in vids_to_tracks.keys():
vids_to_tracks[vid_id] = []
if vid_id not in vids_to_imgs.keys():
vids_to_imgs[vid_id] = []
return vids_to_tracks, vids_to_imgs
def _compute_image_to_timestep_mappings(self):
"""
Computes a mapping from images to the corresponding timestep in the sequence.
:return: the image-to-timestep-mapping
"""
images = {}
for image in self.gt_data["images"]:
images[image["id"]] = image
seq_to_imgs_to_timestep = {vid["id"]: dict() for vid in self.gt_data["videos"]}
for vid in seq_to_imgs_to_timestep:
curr_imgs = [img["id"] for img in self.videos_to_gt_images[vid]]
curr_imgs = sorted(curr_imgs, key=lambda x: images[x]["frame_index"])
seq_to_imgs_to_timestep[vid] = {
curr_imgs[i]: i for i in range(len(curr_imgs))
}
return seq_to_imgs_to_timestep
def _limit_dets_per_image(self, annotations):
"""
Limits the number of detections for each image to config['MAX_DETECTIONS']. Adapted from
https://github.com/TAO-Dataset/
:param annotations: the annotations in which the detections should be limited
:return: the annotations with limited detections
"""
max_dets = self.config["MAX_DETECTIONS"]
img_ann = defaultdict(list)
for ann in annotations:
img_ann[ann["image_id"]].append(ann)
for img_id, _anns in img_ann.items():
if len(_anns) <= max_dets:
continue
_anns = sorted(_anns, key=lambda x: x["score"], reverse=True)
img_ann[img_id] = _anns[:max_dets]
return [ann for anns in img_ann.values() for ann in anns]
def _fill_video_ids_inplace(self, annotations):
"""
Fills in missing video IDs inplace. Adapted from https://github.com/TAO-Dataset/
:param annotations: the annotations for which the videos IDs should be filled inplace
:return: None
"""
missing_video_id = [x for x in annotations if "video_id" not in x]
if missing_video_id:
image_id_to_video_id = {
x["id"]: x["video_id"] for x in self.gt_data["images"]
}
for x in missing_video_id:
x["video_id"] = image_id_to_video_id[x["image_id"]]
@staticmethod
def _make_track_ids_unique(annotations):
"""
Makes the track IDs unqiue over the whole annotation set. Adapted from https://github.com/TAO-Dataset/
:param annotations: the annotation set
:return: the number of updated IDs
"""
track_id_videos = {}
track_ids_to_update = set()
max_track_id = 0
for ann in annotations:
t = ann["track_id"]
if t not in track_id_videos:
track_id_videos[t] = ann["video_id"]
if ann["video_id"] != track_id_videos[t]:
# Track id is assigned to multiple videos
track_ids_to_update.add(t)
max_track_id = max(max_track_id, t)
if track_ids_to_update:
print("true")
next_id = itertools.count(max_track_id + 1)
new_track_ids = defaultdict(lambda: next(next_id))
for ann in annotations:
t = ann["track_id"]
v = ann["video_id"]
if t in track_ids_to_update:
ann["track_id"] = new_track_ids[t, v]
return len(track_ids_to_update)
def _split_known_unknown_distractor(self):
all_ids = set(
[i for i in range(1, 2000)]
) # 2000 is larger than the max category id in TAO-OW.
# `knowns` includes 78 TAO_category_ids that corresponds to 78 COCO classes.
# (The other 2 COCO classes do not have corresponding classes in TAO).
self.knowns = {
4,
13,
1038,
544,
1057,
34,
35,
36,
41,
45,
58,
60,
579,
1091,
1097,
1099,
78,
79,
81,
91,
1115,
1117,
95,
1122,
99,
1132,
621,
1135,
625,
118,
1144,
126,
642,
1155,
133,
1162,
139,
154,
174,
185,
699,
1215,
714,
717,
1229,
211,
729,
221,
229,
747,
235,
237,
779,
276,
805,
299,
829,
852,
347,
371,
382,
896,
392,
926,
937,
428,
429,
961,
452,
979,
980,
982,
475,
480,
993,
1001,
502,
1018,
}
# `distractors` is defined as in the paper "Opening up Open-World Tracking"
self.distractors = {
20,
63,
108,
180,
188,
204,
212,
247,
303,
403,
407,
415,
490,
504,
507,
513,
529,
567,
569,
588,
672,
691,
702,
708,
711,
720,
736,
737,
798,
813,
815,
827,
831,
851,
877,
883,
912,
971,
976,
1130,
1133,
1134,
1169,
1184,
1220,
}
self.unknowns = all_ids.difference(self.knowns.union(self.distractors))
def _filter_gt_data(self, raw_gt_data):
"""
Filter out irrelevant data in the raw_gt_data
Args:
raw_gt_data: directly loaded from json.
Returns:
filtered gt_data
"""
valid_cat_ids = list()
if self.subset == "known":
valid_cat_ids = self.knowns
elif self.subset == "distractor":
valid_cat_ids = self.distractors
elif self.subset == "unknown":
valid_cat_ids = self.unknowns
# elif self.subset == "test_only_unknowns":
# valid_cat_ids = test_only_unknowns
else:
raise Exception("The parameter `SUBSET` is incorrect")
filtered = dict()
filtered["videos"] = raw_gt_data["videos"]
# filtered["videos"] = list()
unwanted_vid = set()
# for video in raw_gt_data["videos"]:
# datasrc = video["name"].split('/')[1]
# if datasrc in data_srcs:
# filtered["videos"].append(video)
# else:
# unwanted_vid.add(video["id"])
filtered["annotations"] = list()
for ann in raw_gt_data["annotations"]:
if (ann["video_id"] not in unwanted_vid) and (
ann["category_id"] in valid_cat_ids
):
filtered["annotations"].append(ann)
filtered["tracks"] = list()
for track in raw_gt_data["tracks"]:
if (track["video_id"] not in unwanted_vid) and (
track["category_id"] in valid_cat_ids
):
filtered["tracks"].append(track)
filtered["images"] = list()
for image in raw_gt_data["images"]:
if image["video_id"] not in unwanted_vid:
filtered["images"].append(image)
filtered["categories"] = list()
for cat in raw_gt_data["categories"]:
if cat["id"] in valid_cat_ids:
filtered["categories"].append(cat)
filtered["info"] = raw_gt_data["info"]
filtered["licenses"] = raw_gt_data["licenses"]
return filtered

524
sam3/eval/hota_eval_toolkit/trackeval/datasets/youtube_vis.py Normal file
View File

@@ -0,0 +1,524 @@
# flake8: noqa
# note: this file has been modified from its original version in TrackEval in
# https://github.com/JonathonLuiten/TrackEval/blob/master/trackeval/datasets/youtube_vis.py
# to support the following:
# 1) bbox evaluation (via `IOU_TYPE`)
# 2) passing GT and prediction data as Python objects (via `GT_JSON_OBJECT` and `TRACKER_JSON_OBJECT`)
# 3) specifying a custom dataset name (via `DATASET_NAME`)
import json
import os
import numpy as np
from .. import _timing, utils
from ..utils import TrackEvalException
from ._base_dataset import _BaseDataset
class YouTubeVIS(_BaseDataset):
"""Dataset class for YouTubeVIS tracking"""
@staticmethod
def get_default_dataset_config():
"""Default class config values"""
code_path = utils.get_code_path()
default_config = {
"GT_FOLDER": os.path.join(
code_path, "data/gt/youtube_vis/"
), # Location of GT data
"TRACKERS_FOLDER": os.path.join(code_path, "data/trackers/youtube_vis/"),
# Trackers location
"OUTPUT_FOLDER": None, # Where to save eval results (if None, same as TRACKERS_FOLDER)
"TRACKERS_TO_EVAL": None, # Filenames of trackers to eval (if None, all in folder)
"CLASSES_TO_EVAL": None, # Classes to eval (if None, all classes)
"SPLIT_TO_EVAL": "train_sub_split", # Valid: 'train', 'val', 'train_sub_split'
"PRINT_CONFIG": True, # Whether to print current config
"OUTPUT_SUB_FOLDER": "", # Output files are saved in OUTPUT_FOLDER/tracker_name/OUTPUT_SUB_FOLDER
"TRACKER_SUB_FOLDER": "data", # Tracker files are in TRACKER_FOLDER/tracker_name/TRACKER_SUB_FOLDER
"TRACKER_DISPLAY_NAMES": None, # Names of trackers to display, if None: TRACKERS_TO_EVAL
# Added for video phrase AP evaluation -- allow directly specifying the GT JSON data and Tracker (result)
# JSON data as Python objects, without reading from files.
"GT_JSON_OBJECT": None,
"TRACKER_JSON_OBJECT": None,
"IOU_TYPE": "segm",
"DATASET_NAME": "video",
}
return default_config
def __init__(self, config=None):
"""Initialise dataset, checking that all required files are present"""
super().__init__()
# Fill non-given config values with defaults
self.config = utils.init_config(config, self.get_default_dataset_config())
self.gt_fol = (
self.config["GT_FOLDER"] + "youtube_vis_" + self.config["SPLIT_TO_EVAL"]
)
self.tracker_fol = (
self.config["TRACKERS_FOLDER"]
+ "youtube_vis_"
+ self.config["SPLIT_TO_EVAL"]
)
self.use_super_categories = False
self.should_classes_combine = True
assert self.config["IOU_TYPE"] in ["segm", "bbox"]
self.iou_type = self.config["IOU_TYPE"]
print("=" * 100)
print(f"Evaluate annotation type *{self.iou_type}*")
self.dataset_name = self.config["DATASET_NAME"]
self.output_fol = self.config["OUTPUT_FOLDER"]
if self.output_fol is None:
self.output_fol = self.tracker_fol
self.output_sub_fol = self.config["OUTPUT_SUB_FOLDER"]
self.tracker_sub_fol = self.config["TRACKER_SUB_FOLDER"]
if self.config["GT_JSON_OBJECT"] is not None:
# allow directly specifying the GT JSON data without reading from files
gt_json = self.config["GT_JSON_OBJECT"]
assert isinstance(gt_json, dict)
assert "videos" in gt_json
assert "categories" in gt_json
assert "annotations" in gt_json
self.gt_data = gt_json
else:
if not os.path.exists(self.gt_fol):
print("GT folder not found: " + self.gt_fol)
raise TrackEvalException(
"GT folder not found: " + os.path.basename(self.gt_fol)
)
gt_dir_files = [
file for file in os.listdir(self.gt_fol) if file.endswith(".json")
]
if len(gt_dir_files) != 1:
raise TrackEvalException(
self.gt_fol + " does not contain exactly one json file."
)
with open(os.path.join(self.gt_fol, gt_dir_files[0])) as f:
self.gt_data = json.load(f)
# Get classes to eval
self.valid_classes = [cls["name"] for cls in self.gt_data["categories"]]
cls_name_to_cls_id_map = {
cls["name"]: cls["id"] for cls in self.gt_data["categories"]
}
if self.config["CLASSES_TO_EVAL"]:
self.class_list = [
cls.lower() if cls.lower() in self.valid_classes else None
for cls in self.config["CLASSES_TO_EVAL"]
]
if not all(self.class_list):
raise TrackEvalException(
"Attempted to evaluate an invalid class. Only classes "
+ ", ".join(self.valid_classes)
+ " are valid."
)
else:
self.class_list = [cls["name"] for cls in self.gt_data["categories"]]
self.class_name_to_class_id = {
k: v for k, v in cls_name_to_cls_id_map.items() if k in self.class_list
}
# Get sequences to eval and check gt files exist
self.seq_list = [
vid["file_names"][0].split("/")[0] for vid in self.gt_data["videos"]
]
self.seq_name_to_seq_id = {
vid["file_names"][0].split("/")[0]: vid["id"]
for vid in self.gt_data["videos"]
}
self.seq_lengths = {
vid["id"]: len(vid["file_names"]) for vid in self.gt_data["videos"]
}
# encode masks and compute track areas
self._prepare_gt_annotations()
# Get trackers to eval
if self.config["TRACKER_JSON_OBJECT"] is not None:
# allow directly specifying the tracker JSON data without reading from files
tracker_json = self.config["TRACKER_JSON_OBJECT"]
assert isinstance(tracker_json, list)
self.tracker_list = ["tracker"]
elif self.config["TRACKERS_TO_EVAL"] is None:
self.tracker_list = os.listdir(self.tracker_fol)
else:
self.tracker_list = self.config["TRACKERS_TO_EVAL"]
if self.config["TRACKER_DISPLAY_NAMES"] is None:
self.tracker_to_disp = dict(zip(self.tracker_list, self.tracker_list))
elif (self.config["TRACKERS_TO_EVAL"] is not None) and (
len(self.config["TRACKER_DISPLAY_NAMES"]) == len(self.tracker_list)
):
self.tracker_to_disp = dict(
zip(self.tracker_list, self.config["TRACKER_DISPLAY_NAMES"])
)
else:
raise TrackEvalException(
"List of tracker files and tracker display names do not match."
)
# counter for globally unique track IDs
self.global_tid_counter = 0
self.tracker_data = dict()
if self.config["TRACKER_JSON_OBJECT"] is not None:
# allow directly specifying the tracker JSON data without reading from files
tracker = self.tracker_list[0]
self.tracker_data[tracker] = tracker_json
else:
for tracker in self.tracker_list:
tracker_dir_path = os.path.join(
self.tracker_fol, tracker, self.tracker_sub_fol
)
tr_dir_files = [
file
for file in os.listdir(tracker_dir_path)
if file.endswith(".json")
]
if len(tr_dir_files) != 1:
raise TrackEvalException(
tracker_dir_path + " does not contain exactly one json file."
)
with open(os.path.join(tracker_dir_path, tr_dir_files[0])) as f:
curr_data = json.load(f)
self.tracker_data[tracker] = curr_data
def get_display_name(self, tracker):
return self.tracker_to_disp[tracker]
def _load_raw_file(self, tracker, seq, is_gt):
"""Load a file (gt or tracker) in the YouTubeVIS format
If is_gt, this returns a dict which contains the fields:
[gt_ids, gt_classes] : list (for each timestep) of 1D NDArrays (for each det).
[gt_dets]: list (for each timestep) of lists of detections.
[classes_to_gt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as
keys and corresponding segmentations as values) for each track
[classes_to_gt_track_ids, classes_to_gt_track_areas, classes_to_gt_track_iscrowd]: dictionary with class values
as keys and lists (for each track) as values
if not is_gt, this returns a dict which contains the fields:
[tracker_ids, tracker_classes, tracker_confidences] : list (for each timestep) of 1D NDArrays (for each det).
[tracker_dets]: list (for each timestep) of lists of detections.
[classes_to_dt_tracks]: dictionary with class values as keys and list of dictionaries (with frame indices as
keys and corresponding segmentations as values) for each track
[classes_to_dt_track_ids, classes_to_dt_track_areas]: dictionary with class values as keys and lists as values
[classes_to_dt_track_scores]: dictionary with class values as keys and 1D numpy arrays as values
"""
# select sequence tracks
seq_id = self.seq_name_to_seq_id[seq]
if is_gt:
tracks = [
ann for ann in self.gt_data["annotations"] if ann["video_id"] == seq_id
]
else:
tracks = self._get_tracker_seq_tracks(tracker, seq_id)
# Convert data to required format
num_timesteps = self.seq_lengths[seq_id]
data_keys = ["ids", "classes", "dets"]
if not is_gt:
data_keys += ["tracker_confidences"]
raw_data = {key: [None] * num_timesteps for key in data_keys}
result_key = "segmentations" if self.iou_type == "segm" else "bboxes"
for t in range(num_timesteps):
raw_data["dets"][t] = [
track[result_key][t] for track in tracks if track[result_key][t]
]
raw_data["ids"][t] = np.atleast_1d(
[track["id"] for track in tracks if track[result_key][t]]
).astype(int)
raw_data["classes"][t] = np.atleast_1d(
[track["category_id"] for track in tracks if track[result_key][t]]
).astype(int)
if not is_gt:
raw_data["tracker_confidences"][t] = np.atleast_1d(
[track["score"] for track in tracks if track[result_key][t]]
).astype(float)
if is_gt:
key_map = {"ids": "gt_ids", "classes": "gt_classes", "dets": "gt_dets"}
else:
key_map = {
"ids": "tracker_ids",
"classes": "tracker_classes",
"dets": "tracker_dets",
}
for k, v in key_map.items():
raw_data[v] = raw_data.pop(k)
all_cls_ids = {self.class_name_to_class_id[cls] for cls in self.class_list}
classes_to_tracks = {
cls: [track for track in tracks if track["category_id"] == cls]
for cls in all_cls_ids
}
# mapping from classes to track representations and track information
raw_data["classes_to_tracks"] = {
cls: [
{i: track[result_key][i] for i in range(len(track[result_key]))}
for track in tracks
]
for cls, tracks in classes_to_tracks.items()
}
raw_data["classes_to_track_ids"] = {
cls: [track["id"] for track in tracks]
for cls, tracks in classes_to_tracks.items()
}
raw_data["classes_to_track_areas"] = {
cls: [track["area"] for track in tracks]
for cls, tracks in classes_to_tracks.items()
}
if is_gt:
raw_data["classes_to_gt_track_iscrowd"] = {
cls: [track["iscrowd"] for track in tracks]
for cls, tracks in classes_to_tracks.items()
}
else:
raw_data["classes_to_dt_track_scores"] = {
cls: np.array([track["score"] for track in tracks])
for cls, tracks in classes_to_tracks.items()
}
if is_gt:
key_map = {
"classes_to_tracks": "classes_to_gt_tracks",
"classes_to_track_ids": "classes_to_gt_track_ids",
"classes_to_track_areas": "classes_to_gt_track_areas",
}
else:
key_map = {
"classes_to_tracks": "classes_to_dt_tracks",
"classes_to_track_ids": "classes_to_dt_track_ids",
"classes_to_track_areas": "classes_to_dt_track_areas",
}
for k, v in key_map.items():
raw_data[v] = raw_data.pop(k)
raw_data["num_timesteps"] = num_timesteps
raw_data["seq"] = seq
return raw_data
@_timing.time
def get_preprocessed_seq_data(self, raw_data, cls):
"""Preprocess data for a single sequence for a single class ready for evaluation.
Inputs:
- raw_data is a dict containing the data for the sequence already read in by get_raw_seq_data().
- cls is the class to be evaluated.
Outputs:
- data is a dict containing all of the information that metrics need to perform evaluation.
It contains the following fields:
[num_timesteps, num_gt_ids, num_tracker_ids, num_gt_dets, num_tracker_dets] : integers.
[gt_ids, tracker_ids, tracker_confidences]: list (for each timestep) of 1D NDArrays (for each det).
[gt_dets, tracker_dets]: list (for each timestep) of lists of detections.
[similarity_scores]: list (for each timestep) of 2D NDArrays.
Notes:
General preprocessing (preproc) occurs in 4 steps. Some datasets may not use all of these steps.
1) Extract only detections relevant for the class to be evaluated (including distractor detections).
2) Match gt dets and tracker dets. Remove tracker dets that are matched to a gt det that is of a
distractor class, or otherwise marked as to be removed.
3) Remove unmatched tracker dets if they fall within a crowd ignore region or don't meet a certain
other criteria (e.g. are too small).
4) Remove gt dets that were only useful for preprocessing and not for actual evaluation.
After the above preprocessing steps, this function also calculates the number of gt and tracker detections
and unique track ids. It also relabels gt and tracker ids to be contiguous and checks that ids are
unique within each timestep.
YouTubeVIS:
In YouTubeVIS, the 4 preproc steps are as follow:
1) There are 40 classes which are evaluated separately.
2) No matched tracker dets are removed.
3) No unmatched tracker dets are removed.
4) No gt dets are removed.
Further, for TrackMAP computation track representations for the given class are accessed from a dictionary
and the tracks from the tracker data are sorted according to the tracker confidence.
"""
cls_id = self.class_name_to_class_id[cls]
data_keys = [
"gt_ids",
"tracker_ids",
"gt_dets",
"tracker_dets",
"similarity_scores",
]
data = {key: [None] * raw_data["num_timesteps"] for key in data_keys}
unique_gt_ids = []
unique_tracker_ids = []
num_gt_dets = 0
num_tracker_dets = 0
for t in range(raw_data["num_timesteps"]):
# Only extract relevant dets for this class for eval (cls)
gt_class_mask = np.atleast_1d(raw_data["gt_classes"][t] == cls_id)
gt_class_mask = gt_class_mask.astype(bool)
gt_ids = raw_data["gt_ids"][t][gt_class_mask]
gt_dets = [
raw_data["gt_dets"][t][ind]
for ind in range(len(gt_class_mask))
if gt_class_mask[ind]
]
tracker_class_mask = np.atleast_1d(raw_data["tracker_classes"][t] == cls_id)
tracker_class_mask = tracker_class_mask.astype(bool)
tracker_ids = raw_data["tracker_ids"][t][tracker_class_mask]
tracker_dets = [
raw_data["tracker_dets"][t][ind]
for ind in range(len(tracker_class_mask))
if tracker_class_mask[ind]
]
similarity_scores = raw_data["similarity_scores"][t][gt_class_mask, :][
:, tracker_class_mask
]
data["tracker_ids"][t] = tracker_ids
data["tracker_dets"][t] = tracker_dets
data["gt_ids"][t] = gt_ids
data["gt_dets"][t] = gt_dets
data["similarity_scores"][t] = similarity_scores
unique_gt_ids += list(np.unique(data["gt_ids"][t]))
unique_tracker_ids += list(np.unique(data["tracker_ids"][t]))
num_tracker_dets += len(data["tracker_ids"][t])
num_gt_dets += len(data["gt_ids"][t])
# Re-label IDs such that there are no empty IDs
if len(unique_gt_ids) > 0:
unique_gt_ids = np.unique(unique_gt_ids)
gt_id_map = np.nan * np.ones((np.max(unique_gt_ids) + 1))
gt_id_map[unique_gt_ids] = np.arange(len(unique_gt_ids))
for t in range(raw_data["num_timesteps"]):
if len(data["gt_ids"][t]) > 0:
data["gt_ids"][t] = gt_id_map[data["gt_ids"][t]].astype(int)
if len(unique_tracker_ids) > 0:
unique_tracker_ids = np.unique(unique_tracker_ids)
tracker_id_map = np.nan * np.ones((np.max(unique_tracker_ids) + 1))
tracker_id_map[unique_tracker_ids] = np.arange(len(unique_tracker_ids))
for t in range(raw_data["num_timesteps"]):
if len(data["tracker_ids"][t]) > 0:
data["tracker_ids"][t] = tracker_id_map[
data["tracker_ids"][t]
].astype(int)
# Ensure that ids are unique per timestep.
self._check_unique_ids(data)
# Record overview statistics.
data["num_tracker_dets"] = num_tracker_dets
data["num_gt_dets"] = num_gt_dets
data["num_tracker_ids"] = len(unique_tracker_ids)
data["num_gt_ids"] = len(unique_gt_ids)
data["num_timesteps"] = raw_data["num_timesteps"]
data["seq"] = raw_data["seq"]
# get track representations
data["gt_tracks"] = raw_data["classes_to_gt_tracks"][cls_id]
data["gt_track_ids"] = raw_data["classes_to_gt_track_ids"][cls_id]
data["gt_track_areas"] = raw_data["classes_to_gt_track_areas"][cls_id]
data["gt_track_iscrowd"] = raw_data["classes_to_gt_track_iscrowd"][cls_id]
data["dt_tracks"] = raw_data["classes_to_dt_tracks"][cls_id]
data["dt_track_ids"] = raw_data["classes_to_dt_track_ids"][cls_id]
data["dt_track_areas"] = raw_data["classes_to_dt_track_areas"][cls_id]
data["dt_track_scores"] = raw_data["classes_to_dt_track_scores"][cls_id]
data["iou_type"] = "mask"
# sort tracker data tracks by tracker confidence scores
if data["dt_tracks"]:
idx = np.argsort(
[-score for score in data["dt_track_scores"]], kind="mergesort"
)
data["dt_track_scores"] = [data["dt_track_scores"][i] for i in idx]
data["dt_tracks"] = [data["dt_tracks"][i] for i in idx]
data["dt_track_ids"] = [data["dt_track_ids"][i] for i in idx]
data["dt_track_areas"] = [data["dt_track_areas"][i] for i in idx]
return data
def _calculate_similarities(self, gt_dets_t, tracker_dets_t):
if self.iou_type == "segm":
similarity_scores = self._calculate_mask_ious(
gt_dets_t, tracker_dets_t, is_encoded=True, do_ioa=False
)
else:
gt_dets_t = np.array(gt_dets_t, dtype=np.float32).reshape(-1, 4)
tracker_dets_t = np.array(tracker_dets_t, dtype=np.float32).reshape(-1, 4)
similarity_scores = self._calculate_box_ious(
gt_dets_t, tracker_dets_t, box_format="xywh", do_ioa=False
)
return similarity_scores
def _prepare_gt_annotations(self):
"""
Prepares GT data by rle encoding segmentations and computing the average track area.
:return: None
"""
if self.iou_type == "segm":
# only loaded when needed to reduce minimum requirements
from pycocotools import mask as mask_utils
for track in self.gt_data["annotations"]:
h = track["height"]
w = track["width"]
for i, seg in enumerate(track["segmentations"]):
if seg is not None and isinstance(seg["counts"], list):
track["segmentations"][i] = mask_utils.frPyObjects(seg, h, w)
areas = [a for a in track["areas"] if a]
if len(areas) == 0:
track["area"] = 0
else:
track["area"] = np.array(areas).mean()
else:
for track in self.gt_data["annotations"]:
# For bbox eval, compute areas from bboxes if not already available
areas = [a for a in track.get("areas", []) if a]
if not areas:
areas = []
for bbox in track.get("bboxes", []):
if bbox is not None:
areas.append(bbox[2] * bbox[3])
track["area"] = np.array(areas).mean() if areas else 0
def _get_tracker_seq_tracks(self, tracker, seq_id):
"""
Prepares tracker data for a given sequence. Extracts all annotations for given sequence ID, computes
average track area and assigns a track ID.
:param tracker: the given tracker
:param seq_id: the sequence ID
:return: the extracted tracks
"""
# only loaded when needed to reduce minimum requirements
from pycocotools import mask as mask_utils
tracks = [
ann for ann in self.tracker_data[tracker] if ann["video_id"] == seq_id
]
for track in tracks:
if "areas" not in track:
if self.iou_type == "segm":
for seg in track["segmentations"]:
if seg:
track["areas"].append(mask_utils.area(seg))
else:
track["areas"].append(None)
else:
for bbox in track["bboxes"]:
if bbox:
track["areas"].append(bbox[2] * bbox[3])
else:
track["areas"].append(None)
areas = [a for a in track["areas"] if a]
if len(areas) == 0:
track["area"] = 0
else:
track["area"] = np.array(areas).mean()
track["id"] = self.global_tid_counter
self.global_tid_counter += 1
return tracks
def get_name(self):
return self.dataset_name

395
sam3/eval/hota_eval_toolkit/trackeval/eval.py Normal file
View File

@@ -0,0 +1,395 @@
# flake8: noqa
import os
import time
import traceback
from functools import partial
from multiprocessing.pool import Pool
import numpy as np
from . import _timing, utils
from .metrics import Count
from .utils import TrackEvalException
try:
import tqdm
TQDM_IMPORTED = True
except ImportError as _:
TQDM_IMPORTED = False
class Evaluator:
"""Evaluator class for evaluating different metrics for different datasets"""
@staticmethod
def get_default_eval_config():
"""Returns the default config values for evaluation"""
code_path = utils.get_code_path()
default_config = {
"USE_PARALLEL": False,
"NUM_PARALLEL_CORES": 8,
"BREAK_ON_ERROR": True, # Raises exception and exits with error
"RETURN_ON_ERROR": False, # if not BREAK_ON_ERROR, then returns from function on error
"LOG_ON_ERROR": os.path.join(
code_path, "error_log.txt"
), # if not None, save any errors into a log file.
"PRINT_RESULTS": True,
"PRINT_ONLY_COMBINED": False,
"PRINT_CONFIG": True,
"TIME_PROGRESS": True,
"DISPLAY_LESS_PROGRESS": True,
"OUTPUT_SUMMARY": True,
"OUTPUT_EMPTY_CLASSES": True, # If False, summary files are not output for classes with no detections
"OUTPUT_DETAILED": True,
"PLOT_CURVES": True,
}
return default_config
def __init__(self, config=None):
"""Initialise the evaluator with a config file"""
self.config = utils.init_config(config, self.get_default_eval_config(), "Eval")
# Only run timing analysis if not run in parallel.
if self.config["TIME_PROGRESS"] and not self.config["USE_PARALLEL"]:
_timing.DO_TIMING = True
if self.config["DISPLAY_LESS_PROGRESS"]:
_timing.DISPLAY_LESS_PROGRESS = True
def _combine_results(
self,
res,
metrics_list,
metric_names,
dataset,
res_field="COMBINED_SEQ",
target_tag=None,
):
assert res_field.startswith("COMBINED_SEQ")
# collecting combined cls keys (cls averaged, det averaged, super classes)
tracker_list, seq_list, class_list = dataset.get_eval_info()
combined_cls_keys = []
res[res_field] = {}
# narrow the target for evaluation
if target_tag is not None:
target_video_ids = [
annot["video_id"]
for annot in dataset.gt_data["annotations"]
if target_tag in annot["tags"]
]
vid2name = {
video["id"]: video["file_names"][0].split("/")[0]
for video in dataset.gt_data["videos"]
}
target_video_ids = set(target_video_ids)
target_video = [vid2name[video_id] for video_id in target_video_ids]
if len(target_video) == 0:
raise TrackEvalException(
"No sequences found with the tag %s" % target_tag
)
target_annotations = [
annot
for annot in dataset.gt_data["annotations"]
if annot["video_id"] in target_video_ids
]
assert all(target_tag in annot["tags"] for annot in target_annotations), (
f"Not all annotations in the target sequences have the target tag {target_tag}. "
"We currently only support a target tag at the sequence level, not at the annotation level."
)
else:
target_video = seq_list
# combine sequences for each class
for c_cls in class_list:
res[res_field][c_cls] = {}
for metric, metric_name in zip(metrics_list, metric_names):
curr_res = {
seq_key: seq_value[c_cls][metric_name]
for seq_key, seq_value in res.items()
if not seq_key.startswith("COMBINED_SEQ")
and seq_key in target_video
}
res[res_field][c_cls][metric_name] = metric.combine_sequences(curr_res)
# combine classes
if dataset.should_classes_combine:
combined_cls_keys += [
"cls_comb_cls_av",
"cls_comb_det_av",
"all",
]
res[res_field]["cls_comb_cls_av"] = {}
res[res_field]["cls_comb_det_av"] = {}
for metric, metric_name in zip(metrics_list, metric_names):
cls_res = {
cls_key: cls_value[metric_name]
for cls_key, cls_value in res[res_field].items()
if cls_key not in combined_cls_keys
}
res[res_field]["cls_comb_cls_av"][metric_name] = (
metric.combine_classes_class_averaged(cls_res)
)
res[res_field]["cls_comb_det_av"][metric_name] = (
metric.combine_classes_det_averaged(cls_res)
)
# combine classes to super classes
if dataset.use_super_categories:
for cat, sub_cats in dataset.super_categories.items():
combined_cls_keys.append(cat)
res[res_field][cat] = {}
for metric, metric_name in zip(metrics_list, metric_names):
cat_res = {
cls_key: cls_value[metric_name]
for cls_key, cls_value in res[res_field].items()
if cls_key in sub_cats
}
res[res_field][cat][metric_name] = (
metric.combine_classes_det_averaged(cat_res)
)
return res, combined_cls_keys
def _summarize_results(
self,
res,
tracker,
metrics_list,
metric_names,
dataset,
res_field,
combined_cls_keys,
):
config = self.config
output_fol = dataset.get_output_fol(tracker)
tracker_display_name = dataset.get_display_name(tracker)
for c_cls in res[
res_field
].keys(): # class_list + combined classes if calculated
summaries = []
details = []
num_dets = res[res_field][c_cls]["Count"]["Dets"]
if config["OUTPUT_EMPTY_CLASSES"] or num_dets > 0:
for metric, metric_name in zip(metrics_list, metric_names):
# for combined classes there is no per sequence evaluation
if c_cls in combined_cls_keys:
table_res = {res_field: res[res_field][c_cls][metric_name]}
else:
table_res = {
seq_key: seq_value[c_cls][metric_name]
for seq_key, seq_value in res.items()
}
if config["PRINT_RESULTS"] and config["PRINT_ONLY_COMBINED"]:
dont_print = (
dataset.should_classes_combine
and c_cls not in combined_cls_keys
)
if not dont_print:
metric.print_table(
{res_field: table_res[res_field]},
tracker_display_name,
c_cls,
res_field,
res_field,
)
elif config["PRINT_RESULTS"]:
metric.print_table(
table_res, tracker_display_name, c_cls, res_field, res_field
)
if config["OUTPUT_SUMMARY"]:
summaries.append(metric.summary_results(table_res))
if config["OUTPUT_DETAILED"]:
details.append(metric.detailed_results(table_res))
if config["PLOT_CURVES"]:
metric.plot_single_tracker_results(
table_res,
tracker_display_name,
c_cls,
output_fol,
)
if config["OUTPUT_SUMMARY"]:
utils.write_summary_results(summaries, c_cls, output_fol)
if config["OUTPUT_DETAILED"]:
utils.write_detailed_results(details, c_cls, output_fol)
@_timing.time
def evaluate(self, dataset_list, metrics_list, show_progressbar=False):
"""Evaluate a set of metrics on a set of datasets"""
config = self.config
metrics_list = metrics_list + [Count()] # Count metrics are always run
metric_names = utils.validate_metrics_list(metrics_list)
dataset_names = [dataset.get_name() for dataset in dataset_list]
output_res = {}
output_msg = {}
for dataset, dataset_name in zip(dataset_list, dataset_names):
# Get dataset info about what to evaluate
output_res[dataset_name] = {}
output_msg[dataset_name] = {}
tracker_list, seq_list, class_list = dataset.get_eval_info()
print(
"\nEvaluating %i tracker(s) on %i sequence(s) for %i class(es) on %s dataset using the following "
"metrics: %s\n"
% (
len(tracker_list),
len(seq_list),
len(class_list),
dataset_name,
", ".join(metric_names),
)
)
# Evaluate each tracker
for tracker in tracker_list:
# if not config['BREAK_ON_ERROR'] then go to next tracker without breaking
try:
# Evaluate each sequence in parallel or in series.
# returns a nested dict (res), indexed like: res[seq][class][metric_name][sub_metric field]
# e.g. res[seq_0001][pedestrian][hota][DetA]
print("\nEvaluating %s\n" % tracker)
time_start = time.time()
if config["USE_PARALLEL"]:
if show_progressbar and TQDM_IMPORTED:
seq_list_sorted = sorted(seq_list)
with Pool(config["NUM_PARALLEL_CORES"]) as pool, tqdm.tqdm(
total=len(seq_list)
) as pbar:
_eval_sequence = partial(
eval_sequence,
dataset=dataset,
tracker=tracker,
class_list=class_list,
metrics_list=metrics_list,
metric_names=metric_names,
)
results = []
for r in pool.imap(
_eval_sequence, seq_list_sorted, chunksize=20
):
results.append(r)
pbar.update()
res = dict(zip(seq_list_sorted, results))
else:
with Pool(config["NUM_PARALLEL_CORES"]) as pool:
_eval_sequence = partial(
eval_sequence,
dataset=dataset,
tracker=tracker,
class_list=class_list,
metrics_list=metrics_list,
metric_names=metric_names,
)
results = pool.map(_eval_sequence, seq_list)
res = dict(zip(seq_list, results))
else:
res = {}
if show_progressbar and TQDM_IMPORTED:
seq_list_sorted = sorted(seq_list)
for curr_seq in tqdm.tqdm(seq_list_sorted):
res[curr_seq] = eval_sequence(
curr_seq,
dataset,
tracker,
class_list,
metrics_list,
metric_names,
)
else:
for curr_seq in sorted(seq_list):
res[curr_seq] = eval_sequence(
curr_seq,
dataset,
tracker,
class_list,
metrics_list,
metric_names,
)
# Combine results over all sequences and then over all classes
res, combined_cls_keys = self._combine_results(
res, metrics_list, metric_names, dataset, "COMBINED_SEQ"
)
if np.all(
["tags" in annot for annot in dataset.gt_data["annotations"]]
):
# Combine results over the challenging sequences and then over all classes
# currently only support "tracking_challenging_pair"
res, _ = self._combine_results(
res,
metrics_list,
metric_names,
dataset,
"COMBINED_SEQ_CHALLENGING",
"tracking_challenging_pair",
)
# Print and output results in various formats
if config["TIME_PROGRESS"]:
print(
"\nAll sequences for %s finished in %.2f seconds"
% (tracker, time.time() - time_start)
)
self._summarize_results(
res,
tracker,
metrics_list,
metric_names,
dataset,
"COMBINED_SEQ",
combined_cls_keys,
)
if "COMBINED_SEQ_CHALLENGING" in res:
self._summarize_results(
res,
tracker,
metrics_list,
metric_names,
dataset,
"COMBINED_SEQ_CHALLENGING",
combined_cls_keys,
)
# Output for returning from function
output_res[dataset_name][tracker] = res
output_msg[dataset_name][tracker] = "Success"
except Exception as err:
output_res[dataset_name][tracker] = None
if type(err) == TrackEvalException:
output_msg[dataset_name][tracker] = str(err)
else:
output_msg[dataset_name][tracker] = "Unknown error occurred."
print("Tracker %s was unable to be evaluated." % tracker)
print(err)
traceback.print_exc()
if config["LOG_ON_ERROR"] is not None:
with open(config["LOG_ON_ERROR"], "a") as f:
print(dataset_name, file=f)
print(tracker, file=f)
print(traceback.format_exc(), file=f)
print("\n\n\n", file=f)
if config["BREAK_ON_ERROR"]:
raise err
elif config["RETURN_ON_ERROR"]:
return output_res, output_msg
return output_res, output_msg
@_timing.time
def eval_sequence(seq, dataset, tracker, class_list, metrics_list, metric_names):
"""Function for evaluating a single sequence"""
raw_data = dataset.get_raw_seq_data(tracker, seq)
seq_res = {}
for cls in class_list:
seq_res[cls] = {}
data = dataset.get_preprocessed_seq_data(raw_data, cls)
for metric, met_name in zip(metrics_list, metric_names):
seq_res[cls][met_name] = metric.eval_sequence(data)
return seq_res

4
sam3/eval/hota_eval_toolkit/trackeval/metrics/__init__.py Normal file
View File

@@ -0,0 +1,4 @@
# flake8: noqa
from .count import Count
from .hota import HOTA

145
sam3/eval/hota_eval_toolkit/trackeval/metrics/_base_metric.py Normal file
View File

@@ -0,0 +1,145 @@
# flake8: noqa
from abc import ABC, abstractmethod
import numpy as np
from .. import _timing
from ..utils import TrackEvalException
class _BaseMetric(ABC):
@abstractmethod
def __init__(self):
self.plottable = False
self.integer_fields = []
self.float_fields = []
self.array_labels = []
self.integer_array_fields = []
self.float_array_fields = []
self.fields = []
self.summary_fields = []
self.registered = False
#####################################################################
# Abstract functions for subclasses to implement
@_timing.time
@abstractmethod
def eval_sequence(self, data): ...
@abstractmethod
def combine_sequences(self, all_res): ...
@abstractmethod
def combine_classes_class_averaged(self, all_res, ignore_empty_classes=False): ...
@abstractmethod
def combine_classes_det_averaged(self, all_res): ...
def plot_single_tracker_results(self, all_res, tracker, output_folder, cls):
"""Plot results of metrics, only valid for metrics with self.plottable"""
if self.plottable:
raise NotImplementedError(
"plot_results is not implemented for metric %s" % self.get_name()
)
else:
pass
#####################################################################
# Helper functions which are useful for all metrics:
@classmethod
def get_name(cls):
return cls.__name__
@staticmethod
def _combine_sum(all_res, field):
"""Combine sequence results via sum"""
return sum([all_res[k][field] for k in all_res.keys()])
@staticmethod
def _combine_weighted_av(all_res, field, comb_res, weight_field):
"""Combine sequence results via weighted average"""
return sum(
[all_res[k][field] * all_res[k][weight_field] for k in all_res.keys()]
) / np.maximum(1.0, comb_res[weight_field])
def print_table(
self, table_res, tracker, cls, res_field="COMBINED_SEQ", output_lable="COMBINED"
):
"""Prints table of results for all sequences"""
print("")
metric_name = self.get_name()
self._row_print(
[metric_name + ": " + tracker + "-" + cls] + self.summary_fields
)
for seq, results in sorted(table_res.items()):
if seq.startswith("COMBINED_SEQ"):
continue
summary_res = self._summary_row(results)
self._row_print([seq] + summary_res)
summary_res = self._summary_row(table_res[res_field])
self._row_print([output_lable] + summary_res)
def _summary_row(self, results_):
vals = []
for h in self.summary_fields:
if h in self.float_array_fields:
vals.append("{0:1.5g}".format(100 * np.mean(results_[h])))
elif h in self.float_fields:
vals.append("{0:1.5g}".format(100 * float(results_[h])))
elif h in self.integer_fields:
vals.append("{0:d}".format(int(results_[h])))
else:
raise NotImplementedError(
"Summary function not implemented for this field type."
)
return vals
@staticmethod
def _row_print(*argv):
"""Prints results in an evenly spaced rows, with more space in first row"""
if len(argv) == 1:
argv = argv[0]
to_print = "%-35s" % argv[0]
for v in argv[1:]:
to_print += "%-10s" % str(v)
print(to_print)
def summary_results(self, table_res):
"""Returns a simple summary of final results for a tracker"""
return dict(
zip(self.summary_fields, self._summary_row(table_res["COMBINED_SEQ"]))
)
def detailed_results(self, table_res):
"""Returns detailed final results for a tracker"""
# Get detailed field information
detailed_fields = self.float_fields + self.integer_fields
for h in self.float_array_fields + self.integer_array_fields:
for alpha in [int(100 * x) for x in self.array_labels]:
detailed_fields.append(h + "___" + str(alpha))
detailed_fields.append(h + "___AUC")
# Get detailed results
detailed_results = {}
for seq, res in table_res.items():
detailed_row = self._detailed_row(res)
if len(detailed_row) != len(detailed_fields):
raise TrackEvalException(
"Field names and data have different sizes (%i and %i)"
% (len(detailed_row), len(detailed_fields))
)
detailed_results[seq] = dict(zip(detailed_fields, detailed_row))
return detailed_results
def _detailed_row(self, res):
detailed_row = []
for h in self.float_fields + self.integer_fields:
detailed_row.append(res[h])
for h in self.float_array_fields + self.integer_array_fields:
for i, alpha in enumerate([int(100 * x) for x in self.array_labels]):
detailed_row.append(res[h][i])
detailed_row.append(np.mean(res[h]))
return detailed_row

48
sam3/eval/hota_eval_toolkit/trackeval/metrics/count.py Normal file
View File

@@ -0,0 +1,48 @@
# flake8: noqa
from .. import _timing
from ._base_metric import _BaseMetric
class Count(_BaseMetric):
"""Class which simply counts the number of tracker and gt detections and ids."""
def __init__(self, config=None):
super().__init__()
self.integer_fields = ["Dets", "GT_Dets", "IDs", "GT_IDs"]
self.fields = self.integer_fields
self.summary_fields = self.fields
@_timing.time
def eval_sequence(self, data):
"""Returns counts for one sequence"""
# Get results
res = {
"Dets": data["num_tracker_dets"],
"GT_Dets": data["num_gt_dets"],
"IDs": data["num_tracker_ids"],
"GT_IDs": data["num_gt_ids"],
"Frames": data["num_timesteps"],
}
return res
def combine_sequences(self, all_res):
"""Combines metrics across all sequences"""
res = {}
for field in self.integer_fields:
res[field] = self._combine_sum(all_res, field)
return res
def combine_classes_class_averaged(self, all_res, ignore_empty_classes=None):
"""Combines metrics across all classes by averaging over the class values"""
res = {}
for field in self.integer_fields:
res[field] = self._combine_sum(all_res, field)
return res
def combine_classes_det_averaged(self, all_res):
"""Combines metrics across all classes by averaging over the detection values"""
res = {}
for field in self.integer_fields:
res[field] = self._combine_sum(all_res, field)
return res

291
sam3/eval/hota_eval_toolkit/trackeval/metrics/hota.py Normal file
View File

@@ -0,0 +1,291 @@
# flake8: noqa
import os
import numpy as np
from scipy.optimize import linear_sum_assignment
from .. import _timing
from ._base_metric import _BaseMetric
class HOTA(_BaseMetric):
"""Class which implements the HOTA metrics.
See: https://link.springer.com/article/10.1007/s11263-020-01375-2
"""
def __init__(self, config=None):
super().__init__()
self.plottable = True
self.array_labels = np.arange(0.05, 0.99, 0.05)
self.integer_array_fields = ["HOTA_TP", "HOTA_FN", "HOTA_FP"]
self.float_array_fields = [
"HOTA",
"DetA",
"AssA",
"DetRe",
"DetPr",
"AssRe",
"AssPr",
"LocA",
"OWTA",
]
self.float_fields = ["HOTA(0)", "LocA(0)", "HOTALocA(0)"]
self.fields = (
self.float_array_fields + self.integer_array_fields + self.float_fields
)
self.summary_fields = self.float_array_fields + self.float_fields
@_timing.time
def eval_sequence(self, data):
"""Calculates the HOTA metrics for one sequence"""
# Initialise results
res = {}
for field in self.float_array_fields + self.integer_array_fields:
res[field] = np.zeros((len(self.array_labels)), dtype=float)
for field in self.float_fields:
res[field] = 0
# Return result quickly if tracker or gt sequence is empty
if data["num_tracker_dets"] == 0:
res["HOTA_FN"] = data["num_gt_dets"] * np.ones(
(len(self.array_labels)), dtype=float
)
res["LocA"] = np.ones((len(self.array_labels)), dtype=float)
res["LocA(0)"] = 1.0
return res
if data["num_gt_dets"] == 0:
res["HOTA_FP"] = data["num_tracker_dets"] * np.ones(
(len(self.array_labels)), dtype=float
)
res["LocA"] = np.ones((len(self.array_labels)), dtype=float)
res["LocA(0)"] = 1.0
return res
# Variables counting global association
potential_matches_count = np.zeros(
(data["num_gt_ids"], data["num_tracker_ids"])
)
gt_id_count = np.zeros((data["num_gt_ids"], 1))
tracker_id_count = np.zeros((1, data["num_tracker_ids"]))
# First loop through each timestep and accumulate global track information.
for t, (gt_ids_t, tracker_ids_t) in enumerate(
zip(data["gt_ids"], data["tracker_ids"])
):
# Count the potential matches between ids in each timestep
# These are normalised, weighted by the match similarity.
similarity = data["similarity_scores"][t]
sim_iou_denom = (
similarity.sum(0)[np.newaxis, :]
+ similarity.sum(1)[:, np.newaxis]
- similarity
)
sim_iou = np.zeros_like(similarity)
sim_iou_mask = sim_iou_denom > 0 + np.finfo("float").eps
sim_iou[sim_iou_mask] = (
similarity[sim_iou_mask] / sim_iou_denom[sim_iou_mask]
)
potential_matches_count[
gt_ids_t[:, np.newaxis], tracker_ids_t[np.newaxis, :]
] += sim_iou
# Calculate the total number of dets for each gt_id and tracker_id.
gt_id_count[gt_ids_t] += 1
tracker_id_count[0, tracker_ids_t] += 1
# Calculate overall jaccard alignment score (before unique matching) between IDs
global_alignment_score = potential_matches_count / (
gt_id_count + tracker_id_count - potential_matches_count
)
matches_counts = [
np.zeros_like(potential_matches_count) for _ in self.array_labels
]
# Calculate scores for each timestep
for t, (gt_ids_t, tracker_ids_t) in enumerate(
zip(data["gt_ids"], data["tracker_ids"])
):
# Deal with the case that there are no gt_det/tracker_det in a timestep.
if len(gt_ids_t) == 0:
for a, alpha in enumerate(self.array_labels):
res["HOTA_FP"][a] += len(tracker_ids_t)
continue
if len(tracker_ids_t) == 0:
for a, alpha in enumerate(self.array_labels):
res["HOTA_FN"][a] += len(gt_ids_t)
continue
# Get matching scores between pairs of dets for optimizing HOTA
similarity = data["similarity_scores"][t]
score_mat = (
global_alignment_score[
gt_ids_t[:, np.newaxis], tracker_ids_t[np.newaxis, :]
]
* similarity
)
# Hungarian algorithm to find best matches
match_rows, match_cols = linear_sum_assignment(-score_mat)
# Calculate and accumulate basic statistics
for a, alpha in enumerate(self.array_labels):
actually_matched_mask = (
similarity[match_rows, match_cols] >= alpha - np.finfo("float").eps
)
alpha_match_rows = match_rows[actually_matched_mask]
alpha_match_cols = match_cols[actually_matched_mask]
num_matches = len(alpha_match_rows)
res["HOTA_TP"][a] += num_matches
res["HOTA_FN"][a] += len(gt_ids_t) - num_matches
res["HOTA_FP"][a] += len(tracker_ids_t) - num_matches
if num_matches > 0:
res["LocA"][a] += sum(
similarity[alpha_match_rows, alpha_match_cols]
)
matches_counts[a][
gt_ids_t[alpha_match_rows], tracker_ids_t[alpha_match_cols]
] += 1
# Calculate association scores (AssA, AssRe, AssPr) for the alpha value.
# First calculate scores per gt_id/tracker_id combo and then average over the number of detections.
for a, alpha in enumerate(self.array_labels):
matches_count = matches_counts[a]
ass_a = matches_count / np.maximum(
1, gt_id_count + tracker_id_count - matches_count
)
res["AssA"][a] = np.sum(matches_count * ass_a) / np.maximum(
1, res["HOTA_TP"][a]
)
ass_re = matches_count / np.maximum(1, gt_id_count)
res["AssRe"][a] = np.sum(matches_count * ass_re) / np.maximum(
1, res["HOTA_TP"][a]
)
ass_pr = matches_count / np.maximum(1, tracker_id_count)
res["AssPr"][a] = np.sum(matches_count * ass_pr) / np.maximum(
1, res["HOTA_TP"][a]
)
# Calculate final scores
res["LocA"] = np.maximum(1e-10, res["LocA"]) / np.maximum(1e-10, res["HOTA_TP"])
res = self._compute_final_fields(res)
return res
def combine_sequences(self, all_res):
"""Combines metrics across all sequences"""
res = {}
for field in self.integer_array_fields:
res[field] = self._combine_sum(all_res, field)
for field in ["AssRe", "AssPr", "AssA"]:
res[field] = self._combine_weighted_av(
all_res, field, res, weight_field="HOTA_TP"
)
loca_weighted_sum = sum(
[all_res[k]["LocA"] * all_res[k]["HOTA_TP"] for k in all_res.keys()]
)
res["LocA"] = np.maximum(1e-10, loca_weighted_sum) / np.maximum(
1e-10, res["HOTA_TP"]
)
res = self._compute_final_fields(res)
return res
def combine_classes_class_averaged(self, all_res, ignore_empty_classes=False):
"""Combines metrics across all classes by averaging over the class values.
If 'ignore_empty_classes' is True, then it only sums over classes with at least one gt or predicted detection.
"""
res = {}
for field in self.integer_array_fields:
if ignore_empty_classes:
res[field] = self._combine_sum(
{
k: v
for k, v in all_res.items()
if (
v["HOTA_TP"] + v["HOTA_FN"] + v["HOTA_FP"]
> 0 + np.finfo("float").eps
).any()
},
field,
)
else:
res[field] = self._combine_sum(
{k: v for k, v in all_res.items()}, field
)
for field in self.float_fields + self.float_array_fields:
if ignore_empty_classes:
res[field] = np.mean(
[
v[field]
for v in all_res.values()
if (
v["HOTA_TP"] + v["HOTA_FN"] + v["HOTA_FP"]
> 0 + np.finfo("float").eps
).any()
],
axis=0,
)
else:
res[field] = np.mean([v[field] for v in all_res.values()], axis=0)
return res
def combine_classes_det_averaged(self, all_res):
"""Combines metrics across all classes by averaging over the detection values"""
res = {}
for field in self.integer_array_fields:
res[field] = self._combine_sum(all_res, field)
for field in ["AssRe", "AssPr", "AssA"]:
res[field] = self._combine_weighted_av(
all_res, field, res, weight_field="HOTA_TP"
)
loca_weighted_sum = sum(
[all_res[k]["LocA"] * all_res[k]["HOTA_TP"] for k in all_res.keys()]
)
res["LocA"] = np.maximum(1e-10, loca_weighted_sum) / np.maximum(
1e-10, res["HOTA_TP"]
)
res = self._compute_final_fields(res)
return res
@staticmethod
def _compute_final_fields(res):
"""Calculate sub-metric ('field') values which only depend on other sub-metric values.
This function is used both for both per-sequence calculation, and in combining values across sequences.
"""
res["DetRe"] = res["HOTA_TP"] / np.maximum(1, res["HOTA_TP"] + res["HOTA_FN"])
res["DetPr"] = res["HOTA_TP"] / np.maximum(1, res["HOTA_TP"] + res["HOTA_FP"])
res["DetA"] = res["HOTA_TP"] / np.maximum(
1, res["HOTA_TP"] + res["HOTA_FN"] + res["HOTA_FP"]
)
res["HOTA"] = np.sqrt(res["DetA"] * res["AssA"])
res["OWTA"] = np.sqrt(res["DetRe"] * res["AssA"])
res["HOTA(0)"] = res["HOTA"][0]
res["LocA(0)"] = res["LocA"][0]
res["HOTALocA(0)"] = res["HOTA(0)"] * res["LocA(0)"]
return res
def plot_single_tracker_results(self, table_res, tracker, cls, output_folder):
"""Create plot of results"""
# Only loaded when run to reduce minimum requirements
from matplotlib import pyplot as plt
res = table_res["COMBINED_SEQ"]
styles_to_plot = ["r", "b", "g", "b--", "b:", "g--", "g:", "m"]
for name, style in zip(self.float_array_fields, styles_to_plot):
plt.plot(self.array_labels, res[name], style)
plt.xlabel("alpha")
plt.ylabel("score")
plt.title(tracker + " - " + cls)
plt.axis([0, 1, 0, 1])
legend = []
for name in self.float_array_fields:
legend += [name + " (" + str(np.round(np.mean(res[name]), 2)) + ")"]
plt.legend(legend, loc="lower left")
out_file = os.path.join(output_folder, cls + "_plot.pdf")
os.makedirs(os.path.dirname(out_file), exist_ok=True)
plt.savefig(out_file)
plt.savefig(out_file.replace(".pdf", ".png"))
plt.clf()

195
sam3/eval/hota_eval_toolkit/trackeval/utils.py Normal file
View File

@@ -0,0 +1,195 @@
# flake8: noqa
import argparse
import csv
import os
from collections import OrderedDict
def init_config(config, default_config, name=None):
"""Initialise non-given config values with defaults"""
if config is None:
config = default_config
else:
for k in default_config.keys():
if k not in config.keys():
config[k] = default_config[k]
if name and config["PRINT_CONFIG"]:
print("\n%s Config:" % name)
for c in config.keys():
print("%-20s : %-30s" % (c, config[c]))
return config
def update_config(config):
"""
Parse the arguments of a script and updates the config values for a given value if specified in the arguments.
:param config: the config to update
:return: the updated config
"""
parser = argparse.ArgumentParser()
for setting in config.keys():
if type(config[setting]) == list or type(config[setting]) == type(None):
parser.add_argument("--" + setting, nargs="+")
else:
parser.add_argument("--" + setting)
args = parser.parse_args().__dict__
for setting in args.keys():
if args[setting] is not None:
if type(config[setting]) == type(True):
if args[setting] == "True":
x = True
elif args[setting] == "False":
x = False
else:
raise Exception(
"Command line parameter " + setting + "must be True or False"
)
elif type(config[setting]) == type(1):
x = int(args[setting])
elif type(args[setting]) == type(None):
x = None
else:
x = args[setting]
config[setting] = x
return config
def get_code_path():
"""Get base path where code is"""
return os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
def validate_metrics_list(metrics_list):
"""Get names of metric class and ensures they are unique, further checks that the fields within each metric class
do not have overlapping names.
"""
metric_names = [metric.get_name() for metric in metrics_list]
# check metric names are unique
if len(metric_names) != len(set(metric_names)):
raise TrackEvalException(
"Code being run with multiple metrics of the same name"
)
fields = []
for m in metrics_list:
fields += m.fields
# check metric fields are unique
if len(fields) != len(set(fields)):
raise TrackEvalException(
"Code being run with multiple metrics with fields of the same name"
)
return metric_names
def write_summary_results(summaries, cls, output_folder):
"""Write summary results to file"""
fields = sum([list(s.keys()) for s in summaries], [])
values = sum([list(s.values()) for s in summaries], [])
# In order to remain consistent upon new fields being adding, for each of the following fields if they are present
# they will be output in the summary first in the order below. Any further fields will be output in the order each
# metric family is called, and within each family either in the order they were added to the dict (python >= 3.6) or
# randomly (python < 3.6).
default_order = [
"HOTA",
"DetA",
"AssA",
"DetRe",
"DetPr",
"AssRe",
"AssPr",
"LocA",
"OWTA",
"HOTA(0)",
"LocA(0)",
"HOTALocA(0)",
"MOTA",
"MOTP",
"MODA",
"CLR_Re",
"CLR_Pr",
"MTR",
"PTR",
"MLR",
"CLR_TP",
"CLR_FN",
"CLR_FP",
"IDSW",
"MT",
"PT",
"ML",
"Frag",
"sMOTA",
"IDF1",
"IDR",
"IDP",
"IDTP",
"IDFN",
"IDFP",
"Dets",
"GT_Dets",
"IDs",
"GT_IDs",
]
default_ordered_dict = OrderedDict(
zip(default_order, [None for _ in default_order])
)
for f, v in zip(fields, values):
default_ordered_dict[f] = v
for df in default_order:
if default_ordered_dict[df] is None:
del default_ordered_dict[df]
fields = list(default_ordered_dict.keys())
values = list(default_ordered_dict.values())
out_file = os.path.join(output_folder, cls + "_summary.txt")
os.makedirs(os.path.dirname(out_file), exist_ok=True)
with open(out_file, "w", newline="") as f:
writer = csv.writer(f, delimiter=" ")
writer.writerow(fields)
writer.writerow(values)
def write_detailed_results(details, cls, output_folder):
"""Write detailed results to file"""
sequences = details[0].keys()
fields = ["seq"] + sum([list(s["COMBINED_SEQ"].keys()) for s in details], [])
out_file = os.path.join(output_folder, cls + "_detailed.csv")
os.makedirs(os.path.dirname(out_file), exist_ok=True)
with open(out_file, "w", newline="") as f:
writer = csv.writer(f)
writer.writerow(fields)
for seq in sorted(sequences):
if seq == "COMBINED_SEQ":
continue
writer.writerow([seq] + sum([list(s[seq].values()) for s in details], []))
writer.writerow(
["COMBINED"] + sum([list(s["COMBINED_SEQ"].values()) for s in details], [])
)
def load_detail(file):
"""Loads detailed data for a tracker."""
data = {}
with open(file) as f:
for i, row_text in enumerate(f):
row = row_text.replace("\r", "").replace("\n", "").split(",")
if i == 0:
keys = row[1:]
continue
current_values = row[1:]
seq = row[0]
if seq == "COMBINED":
seq = "COMBINED_SEQ"
if (len(current_values) == len(keys)) and seq != "":
data[seq] = {}
for key, value in zip(keys, current_values):
data[seq][key] = float(value)
return data
class TrackEvalException(Exception):
"""Custom exception for catching expected errors."""
...