-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #22 from geoaigroup/Moughnieh
mAP for segmentation apps
- Loading branch information
Showing
1 changed file
with
318 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,318 @@ | ||
''' | ||
URL : https://github.com/cosmiq/solaris | ||
Code based on: https://github.com/CosmiQ/solaris/blob/main/solaris/eval/vector.py with some modifications | ||
''' | ||
|
||
import os | ||
import glob | ||
from tqdm import tqdm | ||
import numpy as np | ||
import geopandas as gpd | ||
|
||
|
||
def average_score_by_class(ious, threshold=0.5): | ||
""" for a list of object ious by class, test if they are a counted as a | ||
positive or a negative. | ||
Arguments | ||
--------- | ||
ious : list of lists | ||
A list containing individual lists of ious for eachobject class. | ||
threshold : float , optional | ||
A value between 0.0 and 1.0 that determines the threshold for a true positve. | ||
The 0.5 is its default value , which it can be changed optionally. | ||
Returns | ||
--------- | ||
average_by_class : list | ||
A list containing the ratio of true positives for each class | ||
""" | ||
binary_scoring_lists = [] | ||
for x in ious: | ||
items = [] | ||
for i in x: | ||
if i >= threshold: | ||
items.append(1) | ||
else: | ||
items.append(0) | ||
binary_scoring_lists.append(items) | ||
average_by_class = [] | ||
for l in binary_scoring_lists: | ||
average_by_class.append(np.nanmean(l)) | ||
return average_by_class | ||
|
||
|
||
|
||
|
||
def calculate_iou(pred_poly, test_data_GDF): | ||
"""Get the best intersection over union for a predicted polygon. | ||
Arguments | ||
--------- | ||
pred_poly : :py:class:`shapely.Polygon` | ||
Prediction polygon to test. | ||
test_data_GDF : :py:class:`geopandas.GeoDataFrame` | ||
GeoDataFrame of ground truth polygons to test ``pred_poly`` against. | ||
Returns | ||
------- | ||
iou_GDF : :py:class:`geopandas.GeoDataFrame` | ||
A subset of ``test_data_GDF`` that overlaps ``pred_poly`` with an added | ||
column ``iou_score`` which indicates the intersection over union value. | ||
""" | ||
|
||
# Fix bowties and self-intersections | ||
if not pred_poly.is_valid: | ||
pred_poly = pred_poly.buffer(0.0) | ||
|
||
precise_matches = test_data_GDF[test_data_GDF.intersects(pred_poly)] | ||
|
||
iou_row_list = [] | ||
for _, row in precise_matches.iterrows(): | ||
# Load ground truth polygon and check exact iou | ||
test_poly = row.geometry | ||
# Ignore invalid polygons for now | ||
if pred_poly.is_valid and test_poly.is_valid: | ||
intersection = pred_poly.intersection(test_poly).area | ||
union = pred_poly.union(test_poly).area | ||
# Calculate iou | ||
iou_score = intersection / float(union) | ||
else: | ||
iou_score = 0 | ||
row['iou_score'] = iou_score | ||
iou_row_list.append(row) | ||
|
||
iou_GDF = gpd.GeoDataFrame(iou_row_list) | ||
return iou_GDF | ||
|
||
def get_all_objects(proposal_polygons_dir, gt_polygons_dir, | ||
prediction_cat_attrib="class", gt_cat_attrib='make', | ||
file_format="geojson"): | ||
""" Using the proposal and ground truth polygons, calculate the total. | ||
Filenames of predictions and ground-truth must be identical. | ||
unique classes present in each | ||
Arguments | ||
--------- | ||
proposal_polygons_dir : str | ||
The path that contains any model proposal polygons | ||
gt_polygons_dir : str | ||
The path that contains the ground truth polygons | ||
prediction_cat_attrib : str | ||
The column or attribute within the predictions that specifies | ||
unique classes | ||
replace "class" with the column/attribute name | ||
gt_cat_attrib : str | ||
The column or attribute within the ground truth that | ||
specifies unique classes | ||
replace 'make' with the column/attribute name | ||
file_format : str , optional | ||
The extension or file format for predictions | ||
Returns | ||
--------- | ||
prop_objs : list | ||
All unique objects that exist in the proposals | ||
gt_obj : list | ||
All unique objects that exist in the ground truth | ||
all_objs : list | ||
A union of the prop_objs and gt_objs lists | ||
""" | ||
objs = [] | ||
os.chdir(proposal_polygons_dir) | ||
search = "*" + file_format | ||
proposal_geojsons = glob.glob(search) | ||
for geojson in tqdm(proposal_geojsons): | ||
ground_truth_poly = os.path.join(gt_polygons_dir, geojson) | ||
if os.path.exists(ground_truth_poly): | ||
ground_truth_gdf = gpd.read_file(ground_truth_poly) | ||
proposal_gdf = gpd.read_file(geojson) | ||
for index, row in (proposal_gdf.iterrows()): | ||
objs.append(row[prediction_cat_attrib]) | ||
prop_objs = list(set(objs)) | ||
os.chdir(gt_polygons_dir) | ||
search = "*" + file_format | ||
objs = [] | ||
gt_geojsons = glob.glob(search) | ||
for geojson in tqdm(gt_geojsons): | ||
proposal_poly = os.path.join(proposal_polygons_dir, geojson) | ||
if os.path.exists(proposal_poly): | ||
proposal_gdf = gpd.read_file(proposal_poly) | ||
ground_truth_gdf = gpd.read_file(geojson) | ||
for index, row in (ground_truth_gdf.iterrows()): | ||
objs.append(row[gt_cat_attrib]) | ||
gt_objs = list(set(objs)) | ||
all_objs = gt_objs + prop_objs | ||
all_objs = list(set(all_objs)) | ||
return prop_objs, gt_objs, all_objs | ||
|
||
def calculate_matching_iou(iou_GDF, proposal_row, ground_truth_gdf, prediction_cat_attrib, gt_cat_attrib): | ||
""" | ||
Calculate the matching IoU for a single proposal against ground truth. | ||
Arguments: | ||
----------- | ||
iou_GDF : GeoDataFrame | ||
GeoDataFrame containing IoU scores. | ||
proposal_row : GeoSeries | ||
The row from the proposal GeoDataFrame. | ||
ground_truth_gdf : GeoDataFrame | ||
The GeoDataFrame containing ground truth polygons. | ||
prediction_cat_attrib : str | ||
The attribute in the proposal for class/category. | ||
gt_cat_attrib : str | ||
The attribute in the ground truth for class/category. | ||
Returns: | ||
-------- | ||
float | ||
The IoU score if a matching ground truth is found, 0 otherwise. | ||
""" | ||
if 'iou_score' in iou_GDF.columns: | ||
# Find the maximum IoU score and corresponding ground truth | ||
iou = iou_GDF.iou_score.max() | ||
max_iou_row = iou_GDF.loc[iou_GDF['iou_score'].idxmax()] | ||
id_1 = proposal_row[prediction_cat_attrib] | ||
id_2 = ground_truth_gdf.loc[max_iou_row.name][gt_cat_attrib] | ||
|
||
# Check if the class/category matches | ||
if id_1 == id_2: | ||
ground_truth_gdf.drop(max_iou_row.name, axis=0, inplace=True) | ||
return iou | ||
return 0 | ||
|
||
def process_object_category(proposal_gdf, object_category, gt_cat_attrib, confidence_attrib=None): | ||
""" | ||
Process each object category for IoU and confidence score accumulation. | ||
Arguments: | ||
----------- | ||
proposal_gdf : GeoDataFrame | ||
The GeoDataFrame containing proposal polygons. | ||
object_category : str | ||
The current object category to process. | ||
gt_cat_attrib : str | ||
The attribute in the ground truth for class/category. | ||
confidence_attrib : str, optional | ||
The attribute in the proposal for confidence scores. | ||
Returns: | ||
-------- | ||
ious : list | ||
List of IoU scores for the object category (all zeros in this case). | ||
confidences : list | ||
List of confidence scores for the object category. | ||
""" | ||
ious = [] | ||
confidences = [] | ||
proposal_gdf_filtered = proposal_gdf[proposal_gdf[gt_cat_attrib] == object_category] | ||
|
||
for _ in range(len(proposal_gdf_filtered)): | ||
ious.append(0) # Append 0 for IoU as there's no matching ground truth | ||
|
||
if confidence_attrib: | ||
confidences = [row[confidence_attrib] for _, row in proposal_gdf_filtered.iterrows()] | ||
|
||
return ious, confidences | ||
|
||
|
||
|
||
|
||
def precision_calc(proposal_polygons_dir, gt_polygons_dir, | ||
prediction_cat_attrib="class", gt_cat_attrib='make', confidence_attrib=None, | ||
object_subset=[], threshold=0.5, file_format="geojson"): | ||
""" Using the proposal and ground truth polygons, calculate precision metrics. | ||
Filenames of predictions and ground-truth must be identical. Will only | ||
calculate metric for classes that exist in the ground truth. | ||
Arguments | ||
--------- | ||
proposal_polygons_dir : str | ||
The path that contains any model proposal polygons | ||
gt_polygons_dir : str | ||
The path that contains the ground truth polygons | ||
prediction_cat_attrib : str | ||
The column or attribute within the predictions that specifies | ||
unique classes | ||
replace "class" with the name of the column/attribute | ||
gt_cat_attrib : str | ||
The column or attribute within the ground truth that | ||
specifies unique classes | ||
replace "make" with the name of the column/attribute | ||
confidence_attrib : str | ||
The column or attribute within the proposal polygons that | ||
specifies model confidence for each prediction | ||
if it exists, replace "None" with the name of this column | ||
object_subset : list | ||
A list or subset of the unique objects that are contained within the | ||
ground truth polygons. If empty, this will be | ||
auto-created using all classes that appear ground truth polygons. | ||
threshold : float | ||
A value between 0.0 and 1.0 that determines the IOU threshold for a | ||
true positve. | ||
It's set by default to 0.5.Can be changed optionally. | ||
file_format : str | ||
The extension or file format for predictions | ||
by default set to geojson format | ||
Returns | ||
--------- | ||
iou_holder : list of lists | ||
An iou score for each object per class (precision specific) | ||
precision_by_class : list | ||
A list containing the precision score for each class | ||
mPrecision : float | ||
The mean precision score of precision_by_class | ||
confidences : list of lists | ||
All confidences for each object for each class | ||
""" | ||
ious = [] | ||
dir = os.chdir(proposal_polygons_dir) | ||
search = "*" + file_format | ||
proposal_geojsons = glob.glob(search) | ||
iou_holder = [] | ||
confidences = [] | ||
if len(object_subset) == 0: | ||
prop_objs, object_subset, all_objs = get_all_objects( | ||
proposal_polygons_dir, gt_polygons_dir, | ||
prediction_cat_attrib=prediction_cat_attrib, | ||
gt_cat_attrib=gt_cat_attrib, file_format=file_format) | ||
for i in range(len(object_subset)): | ||
iou_holder.append([]) | ||
confidences.append([]) | ||
|
||
for geojson in tqdm(proposal_geojsons): | ||
ground_truth_poly = os.path.join(gt_polygons_dir, geojson) | ||
if os.path.exists(ground_truth_poly): | ||
ground_truth_gdf = gpd.read_file(ground_truth_poly) | ||
proposal_gdf = gpd.read_file(geojson) | ||
i = 0 | ||
for obj in object_subset: | ||
conf_holder = [] | ||
proposal_gdf2 = proposal_gdf[proposal_gdf[prediction_cat_attrib] == obj] | ||
for index, row in (proposal_gdf2.iterrows()): | ||
if confidence_attrib is not None: | ||
conf_holder.append(row[confidence_attrib]) | ||
iou_GDF = calculate_iou(row.geometry, ground_truth_gdf) | ||
iou = calculate_matching_iou(iou_GDF, row, ground_truth_gdf, prediction_cat_attrib, gt_cat_attrib) | ||
ious.append(iou) | ||
for item in ious: | ||
iou_holder[i].append(item) | ||
if confidence_attrib is not None: | ||
for conf in conf_holder: | ||
confidences[i].append(conf) | ||
ious = [] | ||
i += 1 | ||
else: | ||
print("Warning- No ground truth for:", geojson) | ||
proposal_gdf = gpd.read_file(geojson) | ||
i = 0 | ||
|
||
for j, obj in enumerate(object_subset): | ||
ious, conf_holder = process_object_category(proposal_gdf, obj, gt_cat_attrib, confidence_attrib) | ||
iou_holder[j].extend(ious) | ||
confidences[j].extend(conf_holder) | ||
i += 1 | ||
ious = [] | ||
precision_by_class = average_score_by_class(iou_holder, threshold=threshold) | ||
precision_by_class = list(np.nan_to_num(precision_by_class)) | ||
mPrecision = np.nanmean(precision_by_class) | ||
print("mPrecision:", mPrecision) | ||
|
||
return iou_holder, precision_by_class, mPrecision, confidences | ||
|