mirror of
https://github.com/PaddlePaddle/PaddleOCR.git
synced 2025-06-26 21:24:27 +00:00
475 lines
18 KiB
Python
475 lines
18 KiB
Python
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
import math
|
|
from collections import namedtuple
|
|
import numpy as np
|
|
from shapely.geometry import Polygon
|
|
|
|
|
|
class DetectionDetEvalEvaluator(object):
|
|
def __init__(
|
|
self,
|
|
area_recall_constraint=0.8,
|
|
area_precision_constraint=0.4,
|
|
ev_param_ind_center_diff_thr=1,
|
|
mtype_oo_o=1.0,
|
|
mtype_om_o=0.8,
|
|
mtype_om_m=1.0,
|
|
):
|
|
self.area_recall_constraint = area_recall_constraint
|
|
self.area_precision_constraint = area_precision_constraint
|
|
self.ev_param_ind_center_diff_thr = ev_param_ind_center_diff_thr
|
|
self.mtype_oo_o = mtype_oo_o
|
|
self.mtype_om_o = mtype_om_o
|
|
self.mtype_om_m = mtype_om_m
|
|
|
|
def evaluate_image(self, gt, pred):
|
|
def get_union(pD, pG):
|
|
return Polygon(pD).union(Polygon(pG)).area
|
|
|
|
def get_intersection_over_union(pD, pG):
|
|
return get_intersection(pD, pG) / get_union(pD, pG)
|
|
|
|
def get_intersection(pD, pG):
|
|
return Polygon(pD).intersection(Polygon(pG)).area
|
|
|
|
def one_to_one_match(row, col):
|
|
cont = 0
|
|
for j in range(len(recallMat[0])):
|
|
if (
|
|
recallMat[row, j] >= self.area_recall_constraint
|
|
and precisionMat[row, j] >= self.area_precision_constraint
|
|
):
|
|
cont = cont + 1
|
|
if cont != 1:
|
|
return False
|
|
cont = 0
|
|
for i in range(len(recallMat)):
|
|
if (
|
|
recallMat[i, col] >= self.area_recall_constraint
|
|
and precisionMat[i, col] >= self.area_precision_constraint
|
|
):
|
|
cont = cont + 1
|
|
if cont != 1:
|
|
return False
|
|
|
|
if (
|
|
recallMat[row, col] >= self.area_recall_constraint
|
|
and precisionMat[row, col] >= self.area_precision_constraint
|
|
):
|
|
return True
|
|
return False
|
|
|
|
def num_overlaps_gt(gtNum):
|
|
cont = 0
|
|
for detNum in range(len(detRects)):
|
|
if detNum not in detDontCareRectsNum:
|
|
if recallMat[gtNum, detNum] > 0:
|
|
cont = cont + 1
|
|
return cont
|
|
|
|
def num_overlaps_det(detNum):
|
|
cont = 0
|
|
for gtNum in range(len(recallMat)):
|
|
if gtNum not in gtDontCareRectsNum:
|
|
if recallMat[gtNum, detNum] > 0:
|
|
cont = cont + 1
|
|
return cont
|
|
|
|
def is_single_overlap(row, col):
|
|
if num_overlaps_gt(row) == 1 and num_overlaps_det(col) == 1:
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
def one_to_many_match(gtNum):
|
|
many_sum = 0
|
|
detRects = []
|
|
for detNum in range(len(recallMat[0])):
|
|
if (
|
|
gtRectMat[gtNum] == 0
|
|
and detRectMat[detNum] == 0
|
|
and detNum not in detDontCareRectsNum
|
|
):
|
|
if precisionMat[gtNum, detNum] >= self.area_precision_constraint:
|
|
many_sum += recallMat[gtNum, detNum]
|
|
detRects.append(detNum)
|
|
if round(many_sum, 4) >= self.area_recall_constraint:
|
|
return True, detRects
|
|
else:
|
|
return False, []
|
|
|
|
def many_to_one_match(detNum):
|
|
many_sum = 0
|
|
gtRects = []
|
|
for gtNum in range(len(recallMat)):
|
|
if (
|
|
gtRectMat[gtNum] == 0
|
|
and detRectMat[detNum] == 0
|
|
and gtNum not in gtDontCareRectsNum
|
|
):
|
|
if recallMat[gtNum, detNum] >= self.area_recall_constraint:
|
|
many_sum += precisionMat[gtNum, detNum]
|
|
gtRects.append(gtNum)
|
|
if round(many_sum, 4) >= self.area_precision_constraint:
|
|
return True, gtRects
|
|
else:
|
|
return False, []
|
|
|
|
def center_distance(r1, r2):
|
|
return ((np.mean(r1, axis=0) - np.mean(r2, axis=0)) ** 2).sum() ** 0.5
|
|
|
|
def diag(r):
|
|
r = np.array(r)
|
|
return (
|
|
(r[:, 0].max() - r[:, 0].min()) ** 2
|
|
+ (r[:, 1].max() - r[:, 1].min()) ** 2
|
|
) ** 0.5
|
|
|
|
perSampleMetrics = {}
|
|
|
|
recall = 0
|
|
precision = 0
|
|
hmean = 0
|
|
recallAccum = 0.0
|
|
precisionAccum = 0.0
|
|
gtRects = []
|
|
detRects = []
|
|
gtPolPoints = []
|
|
detPolPoints = []
|
|
gtDontCareRectsNum = (
|
|
[]
|
|
) # Array of Ground Truth Rectangles' keys marked as don't Care
|
|
detDontCareRectsNum = (
|
|
[]
|
|
) # Array of Detected Rectangles' matched with a don't Care GT
|
|
pairs = []
|
|
evaluationLog = ""
|
|
|
|
recallMat = np.empty([1, 1])
|
|
precisionMat = np.empty([1, 1])
|
|
|
|
for n in range(len(gt)):
|
|
points = gt[n]["points"]
|
|
# transcription = gt[n]['text']
|
|
dontCare = gt[n]["ignore"]
|
|
|
|
if not Polygon(points).is_valid or not Polygon(points).is_simple:
|
|
continue
|
|
|
|
gtRects.append(points)
|
|
gtPolPoints.append(points)
|
|
if dontCare:
|
|
gtDontCareRectsNum.append(len(gtRects) - 1)
|
|
|
|
evaluationLog += (
|
|
"GT rectangles: "
|
|
+ str(len(gtRects))
|
|
+ (
|
|
" (" + str(len(gtDontCareRectsNum)) + " don't care)\n"
|
|
if len(gtDontCareRectsNum) > 0
|
|
else "\n"
|
|
)
|
|
)
|
|
|
|
for n in range(len(pred)):
|
|
points = pred[n]["points"]
|
|
|
|
if not Polygon(points).is_valid or not Polygon(points).is_simple:
|
|
continue
|
|
|
|
detRect = points
|
|
detRects.append(detRect)
|
|
detPolPoints.append(points)
|
|
if len(gtDontCareRectsNum) > 0:
|
|
for dontCareRectNum in gtDontCareRectsNum:
|
|
dontCareRect = gtRects[dontCareRectNum]
|
|
intersected_area = get_intersection(dontCareRect, detRect)
|
|
rdDimensions = Polygon(detRect).area
|
|
if rdDimensions == 0:
|
|
precision = 0
|
|
else:
|
|
precision = intersected_area / rdDimensions
|
|
if precision > self.area_precision_constraint:
|
|
detDontCareRectsNum.append(len(detRects) - 1)
|
|
break
|
|
|
|
evaluationLog += (
|
|
"DET rectangles: "
|
|
+ str(len(detRects))
|
|
+ (
|
|
" (" + str(len(detDontCareRectsNum)) + " don't care)\n"
|
|
if len(detDontCareRectsNum) > 0
|
|
else "\n"
|
|
)
|
|
)
|
|
|
|
if len(gtRects) == 0:
|
|
recall = 1
|
|
precision = 0 if len(detRects) > 0 else 1
|
|
|
|
if len(detRects) > 0:
|
|
# Calculate recall and precision matrixes
|
|
outputShape = [len(gtRects), len(detRects)]
|
|
recallMat = np.empty(outputShape)
|
|
precisionMat = np.empty(outputShape)
|
|
gtRectMat = np.zeros(len(gtRects), np.int8)
|
|
detRectMat = np.zeros(len(detRects), np.int8)
|
|
for gtNum in range(len(gtRects)):
|
|
for detNum in range(len(detRects)):
|
|
rG = gtRects[gtNum]
|
|
rD = detRects[detNum]
|
|
intersected_area = get_intersection(rG, rD)
|
|
rgDimensions = Polygon(rG).area
|
|
rdDimensions = Polygon(rD).area
|
|
recallMat[gtNum, detNum] = (
|
|
0 if rgDimensions == 0 else intersected_area / rgDimensions
|
|
)
|
|
precisionMat[gtNum, detNum] = (
|
|
0 if rdDimensions == 0 else intersected_area / rdDimensions
|
|
)
|
|
|
|
# Find one-to-one matches
|
|
evaluationLog += "Find one-to-one matches\n"
|
|
for gtNum in range(len(gtRects)):
|
|
for detNum in range(len(detRects)):
|
|
if (
|
|
gtRectMat[gtNum] == 0
|
|
and detRectMat[detNum] == 0
|
|
and gtNum not in gtDontCareRectsNum
|
|
and detNum not in detDontCareRectsNum
|
|
):
|
|
match = one_to_one_match(gtNum, detNum)
|
|
if match is True:
|
|
# in deteval we have to make other validation before mark as one-to-one
|
|
if is_single_overlap(gtNum, detNum) is True:
|
|
rG = gtRects[gtNum]
|
|
rD = detRects[detNum]
|
|
normDist = center_distance(rG, rD)
|
|
normDist /= diag(rG) + diag(rD)
|
|
normDist *= 2.0
|
|
if normDist < self.ev_param_ind_center_diff_thr:
|
|
gtRectMat[gtNum] = 1
|
|
detRectMat[detNum] = 1
|
|
recallAccum += self.mtype_oo_o
|
|
precisionAccum += self.mtype_oo_o
|
|
pairs.append(
|
|
{"gt": gtNum, "det": detNum, "type": "OO"}
|
|
)
|
|
evaluationLog += (
|
|
"Match GT #"
|
|
+ str(gtNum)
|
|
+ " with Det #"
|
|
+ str(detNum)
|
|
+ "\n"
|
|
)
|
|
else:
|
|
evaluationLog += (
|
|
"Match Discarded GT #"
|
|
+ str(gtNum)
|
|
+ " with Det #"
|
|
+ str(detNum)
|
|
+ " normDist: "
|
|
+ str(normDist)
|
|
+ " \n"
|
|
)
|
|
else:
|
|
evaluationLog += (
|
|
"Match Discarded GT #"
|
|
+ str(gtNum)
|
|
+ " with Det #"
|
|
+ str(detNum)
|
|
+ " not single overlap\n"
|
|
)
|
|
# Find one-to-many matches
|
|
evaluationLog += "Find one-to-many matches\n"
|
|
for gtNum in range(len(gtRects)):
|
|
if gtNum not in gtDontCareRectsNum:
|
|
match, matchesDet = one_to_many_match(gtNum)
|
|
if match is True:
|
|
evaluationLog += "num_overlaps_gt=" + str(
|
|
num_overlaps_gt(gtNum)
|
|
)
|
|
# in deteval we have to make other validation before mark as one-to-one
|
|
if num_overlaps_gt(gtNum) >= 2:
|
|
gtRectMat[gtNum] = 1
|
|
recallAccum += (
|
|
self.mtype_oo_o
|
|
if len(matchesDet) == 1
|
|
else self.mtype_om_o
|
|
)
|
|
precisionAccum += (
|
|
self.mtype_oo_o
|
|
if len(matchesDet) == 1
|
|
else self.mtype_om_o * len(matchesDet)
|
|
)
|
|
pairs.append(
|
|
{
|
|
"gt": gtNum,
|
|
"det": matchesDet,
|
|
"type": "OO" if len(matchesDet) == 1 else "OM",
|
|
}
|
|
)
|
|
for detNum in matchesDet:
|
|
detRectMat[detNum] = 1
|
|
evaluationLog += (
|
|
"Match GT #"
|
|
+ str(gtNum)
|
|
+ " with Det #"
|
|
+ str(matchesDet)
|
|
+ "\n"
|
|
)
|
|
else:
|
|
evaluationLog += (
|
|
"Match Discarded GT #"
|
|
+ str(gtNum)
|
|
+ " with Det #"
|
|
+ str(matchesDet)
|
|
+ " not single overlap\n"
|
|
)
|
|
|
|
# Find many-to-one matches
|
|
evaluationLog += "Find many-to-one matches\n"
|
|
for detNum in range(len(detRects)):
|
|
if detNum not in detDontCareRectsNum:
|
|
match, matchesGt = many_to_one_match(detNum)
|
|
if match is True:
|
|
# in deteval we have to make other validation before mark as one-to-one
|
|
if num_overlaps_det(detNum) >= 2:
|
|
detRectMat[detNum] = 1
|
|
recallAccum += (
|
|
self.mtype_oo_o
|
|
if len(matchesGt) == 1
|
|
else self.mtype_om_m * len(matchesGt)
|
|
)
|
|
precisionAccum += (
|
|
self.mtype_oo_o
|
|
if len(matchesGt) == 1
|
|
else self.mtype_om_m
|
|
)
|
|
pairs.append(
|
|
{
|
|
"gt": matchesGt,
|
|
"det": detNum,
|
|
"type": "OO" if len(matchesGt) == 1 else "MO",
|
|
}
|
|
)
|
|
for gtNum in matchesGt:
|
|
gtRectMat[gtNum] = 1
|
|
evaluationLog += (
|
|
"Match GT #"
|
|
+ str(matchesGt)
|
|
+ " with Det #"
|
|
+ str(detNum)
|
|
+ "\n"
|
|
)
|
|
else:
|
|
evaluationLog += (
|
|
"Match Discarded GT #"
|
|
+ str(matchesGt)
|
|
+ " with Det #"
|
|
+ str(detNum)
|
|
+ " not single overlap\n"
|
|
)
|
|
|
|
numGtCare = len(gtRects) - len(gtDontCareRectsNum)
|
|
if numGtCare == 0:
|
|
recall = float(1)
|
|
precision = float(0) if len(detRects) > 0 else float(1)
|
|
else:
|
|
recall = float(recallAccum) / numGtCare
|
|
precision = (
|
|
float(0)
|
|
if (len(detRects) - len(detDontCareRectsNum)) == 0
|
|
else float(precisionAccum)
|
|
/ (len(detRects) - len(detDontCareRectsNum))
|
|
)
|
|
hmean = (
|
|
0
|
|
if (precision + recall) == 0
|
|
else 2.0 * precision * recall / (precision + recall)
|
|
)
|
|
|
|
numGtCare = len(gtRects) - len(gtDontCareRectsNum)
|
|
numDetCare = len(detRects) - len(detDontCareRectsNum)
|
|
|
|
perSampleMetrics = {
|
|
"precision": precision,
|
|
"recall": recall,
|
|
"hmean": hmean,
|
|
"pairs": pairs,
|
|
"recallMat": [] if len(detRects) > 100 else recallMat.tolist(),
|
|
"precisionMat": [] if len(detRects) > 100 else precisionMat.tolist(),
|
|
"gtPolPoints": gtPolPoints,
|
|
"detPolPoints": detPolPoints,
|
|
"gtCare": numGtCare,
|
|
"detCare": numDetCare,
|
|
"gtDontCare": gtDontCareRectsNum,
|
|
"detDontCare": detDontCareRectsNum,
|
|
"recallAccum": recallAccum,
|
|
"precisionAccum": precisionAccum,
|
|
"evaluationLog": evaluationLog,
|
|
}
|
|
|
|
return perSampleMetrics
|
|
|
|
def combine_results(self, results):
|
|
numGt = 0
|
|
numDet = 0
|
|
methodRecallSum = 0
|
|
methodPrecisionSum = 0
|
|
|
|
for result in results:
|
|
numGt += result["gtCare"]
|
|
numDet += result["detCare"]
|
|
methodRecallSum += result["recallAccum"]
|
|
methodPrecisionSum += result["precisionAccum"]
|
|
|
|
methodRecall = 0 if numGt == 0 else methodRecallSum / numGt
|
|
methodPrecision = 0 if numDet == 0 else methodPrecisionSum / numDet
|
|
methodHmean = (
|
|
0
|
|
if methodRecall + methodPrecision == 0
|
|
else 2 * methodRecall * methodPrecision / (methodRecall + methodPrecision)
|
|
)
|
|
|
|
methodMetrics = {
|
|
"precision": methodPrecision,
|
|
"recall": methodRecall,
|
|
"hmean": methodHmean,
|
|
}
|
|
|
|
return methodMetrics
|
|
|
|
|
|
if __name__ == "__main__":
|
|
evaluator = DetectionDetEvalEvaluator()
|
|
gts = [
|
|
[
|
|
{
|
|
"points": [(0, 0), (1, 0), (1, 1), (0, 1)],
|
|
"text": 1234,
|
|
"ignore": False,
|
|
},
|
|
{
|
|
"points": [(2, 2), (3, 2), (3, 3), (2, 3)],
|
|
"text": 5678,
|
|
"ignore": True,
|
|
},
|
|
]
|
|
]
|
|
preds = [
|
|
[
|
|
{
|
|
"points": [(0.1, 0.1), (1, 0), (1, 1), (0, 1)],
|
|
"text": 123,
|
|
"ignore": False,
|
|
}
|
|
]
|
|
]
|
|
results = []
|
|
for gt, pred in zip(gts, preds):
|
|
results.append(evaluator.evaluate_image(gt, pred))
|
|
metrics = evaluator.combine_results(results)
|
|
print(metrics)
|