【工具代碼】目標檢測繪製評估曲線

draw.py

# -*- coding: UTF-8 -*-
from pr import *
clas= ['ignore', 'pedestrian', 'people', 'bicycle', 'car', 'van', 'truck', 'tricycle', 'awning_tricycle', 'bus', 'motor'] # 類別
prediction_file=["FRCNN.txt", 'RetinaNet.txt', 'ourResult.txt', 'SSD.txt', "CenterNet.txt"] # 放在Prediction裏面"Faster_RCNN.txt"
prediction_algorithm=["FRCNN", 'RetinaNet', 'ourResult', 'SSD', "CenterNet"] # 對應上面的文件的算法名稱
use_07_metric=[False, False, False, False, False]
ground_truth_file=["gt.txt", 'gt.txt', 'gt.txt', 'gt.txt', "gt.txt"] # 放在Ground Truth裏面

# 繪製出了各個類別的Precision-Recall曲線
draw_pr(prediction_file,prediction_algorithm,ground_truth_file,clas,use_07_metric)

pr.py

# -*- coding: UTF-8 -*-
import numpy as np
import math
import matplotlib.pyplot as plt

def get_pr_data_map(prediction_file,ground_truth_file,cls_name,use_07_metric,ovthresh=0.5):
    with open(ground_truth_file, 'r') as f:####讀取各個算法的gt.txt文件
        lines_gt = f.readlines()          #gt.txt的每一行，line_gt爲一維數組
    with open(prediction_file, 'r') as f:####讀取各個算法的pre.txt文件
        lines_pre = f.readlines()         #讀取pre.txt的每一行，lines_gt爲一維數組
            #根據txt文件中每一行的空格進行劃分，splitlines_gt爲二維數組，行爲每一行的數據，列爲每一行的數據劃分
    splitlines_gt = [x.strip().split(' ') for x in lines_gt]
    #imagenames_gt爲一維數組，當第二列的類別與遍歷的類別數相同時，將第一列加入到imagenames_gt中
    imagenames_gt = [ x[0] for x in splitlines_gt ] 
    class_recs = {}       
#     #BB_gt爲二維數組，當第二列的類別與遍歷的類別數相同時，將第二列之後的加入到BB_gt中
    BB_gt=np.array([[math.ceil(float(z)) for z in x[2:]] for x in splitlines_gt])
    for i in range(len(imagenames_gt)):
        #如果類別中沒有該遍歷的類別，則更新
        if imagenames_gt[i] not in class_recs:
            class_recs.update({imagenames_gt[i]:{"bbox":[],"det":[],"difficult":[]}})
        #否則將BB_gt[i]插入到類別框信息中
        class_recs[imagenames_gt[i]]["bbox"].append(BB_gt[i])
        class_recs[imagenames_gt[i]]["det"].append(False)
        class_recs[imagenames_gt[i]]["difficult"].append(False)
            #npos爲imagenames_gt的長度
    npos=len(imagenames_gt)
    splitlines_pre = [x.strip().split(' ') for x in lines_pre] 
    #image_ids爲一維數組，當第二列的類別與遍歷的類別數相同時，將第一列加入到image_ids中     
    image_ids = [x[0] for x in splitlines_pre] 
    #提取該類別預測框的置信度，當第二列的類別與遍歷的類別數相同時，將第三列加入到confidence中
    confidence = np.array([float(x[2]) for x in splitlines_pre]) 
    #提取該類別預測框的bounging-boxes（二維數組），當第二列的類別與遍歷的類別數相同時,將第三列之後的BBOX座標插入
    BB_pre = np.array([[float(z) for z in x[3:]] for x in splitlines_pre]) 
    #按置信度大小將其索引從小到大排序（生成有順序的一維數組）
    sorted_ind = np.argsort(-confidence)
    #按置信度大小將置信度從小到大排序（生成有順序的一維數組）
    sorted_scores = np.sort(-confidence)
    #根據索引排序相應的bbox的座標值（生成按置信度大小排列的二維數組）
    BB_pre = BB_pre[sorted_ind, :]
    #按置信度大小重新排列image_ids
    image_ids = [image_ids[x] for x in sorted_ind]
    nd = len(image_ids)
    tp = np.zeros(nd)
    fp = np.zeros(nd)
    pr_data_map=[]
    for d in range(nd):
        if class_recs.get(image_ids[d]):
            #print(image_ids[d])
            R = class_recs[image_ids[d]]
            #print(R)
            bb = np.array(BB_pre[d, :]).astype(float)#按置信度順序提取bounding box
            ovmax = -np.inf#ovmax爲負無窮大的數
            BBGT = np.array(R['bbox']).astype(float)#按置信度順序提取groudtruth bbox

            #計算iou
            if BBGT.size > 0:
                ixmin = np.maximum(BBGT[:, 0], bb[0])
                iymin = np.maximum(BBGT[:, 1], bb[1])
                ixmax = np.minimum(BBGT[:, 2], bb[2])
                iymax = np.minimum(BBGT[:, 3], bb[3])
                iw = np.maximum(ixmax - ixmin + 1., 0.)
                ih = np.maximum(iymax - iymin + 1., 0.)
                inters = iw * ih

                uni = ((bb[2] - bb[0] + 1.) * (bb[3] - bb[1] + 1.) +
                       (BBGT[:, 2] - BBGT[:, 0] + 1.) *
                       (BBGT[:, 3] - BBGT[:, 1] + 1.) - inters)

                overlaps = inters / uni#重疊率
                ovmax = np.max(overlaps)#按重疊率的大小從大到小排序重疊率
                jmax = np.argmax(overlaps)#根據重疊率大小重新排序的索引

            if ovmax > ovthresh:#ovthresh=0.5閾值爲0.5，判斷tp和fp
                if not R['difficult'][jmax]:
                    if not R['det'][jmax]:
                        tp[d] = 1.
                        R['det'][jmax] = 1
                    else:
                        fp[d] = 1.
            else:
                fp[d] = 1.
        else:
            fp[d]=1.


        
    fp = np.cumsum(fp)
    tp = np.cumsum(tp)
    #rec=tp/正樣本數
    rec = tp / float(npos)
    print(len(rec))
    # print(type(rec))
    #perc=tp/(tp+fp)
    prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps)
    print(len(prec))
    # print(type(prec))
    ap = voc_ap(rec, prec, use_07_metric)
    print(ap)
    pr_data_map=np.array([rec.tolist(),prec.tolist()])#,ap.tolist()]
    pr_data_map=pr_data_map.tolist()

    #print(pr_data_map)
    return pr_data_map 


def get_pr_data(prediction_file,ground_truth_file,cls_name,use_07_metric,ovthresh=0.5):
    with open(ground_truth_file, 'r') as f:####讀取各個算法的gt.txt文件
        lines_gt = f.readlines()          #gt.txt的每一行，line_gt爲一維數組
    with open(prediction_file, 'r') as f:####讀取各個算法的pre.txt文件
        lines_pre = f.readlines()         #讀取pre.txt的每一行，lines_gt爲一維數組

    pr_data={}
    for cls_num in range(len(cls_name)): #遍歷每個類別數
        # print(cls_num)
    #ground_truth  
        #根據txt文件中每一行的空格進行劃分，splitlines_gt爲二維數組，行爲每一行的數據，列爲每一行的數據劃分
        splitlines_gt = [x.strip().split(' ') for x in lines_gt]
        #imagenames_gt爲一維數組，當第二列的類別與遍歷的類別數相同時，將第一列加入到imagenames_gt中
        imagenames_gt = [ x[0] for x in splitlines_gt if int(x[1])==cls_num] 
        #print(imagenames_gt)           
        #BB_gt爲二維數組，當第二列的類別與遍歷的類別數相同時，將第二列之後的加入到BB_gt中
        BB_gt=np.array([[math.ceil(float(z)) for z in x[2:]] for x in splitlines_gt if int(x[1])==cls_num])
        #創建class_recs數組
        class_recs = {}
        for i in range(len(imagenames_gt)):
            #如果類別中沒有該遍歷的類別，則更新
            if imagenames_gt[i] not in class_recs:
                class_recs.update({imagenames_gt[i]:{"bbox":[],"det":[],"difficult":[]}})
            #否則將BB_gt[i]插入到類別框信息中
            class_recs[imagenames_gt[i]]["bbox"].append(BB_gt[i])
            class_recs[imagenames_gt[i]]["det"].append(False)
            class_recs[imagenames_gt[i]]["difficult"].append(False)
            #print(class_recs[imagenames_gt[i]])
        #npos爲imagenames_gt的長度
        npos=len(imagenames_gt)
    
    #prediction 
        #對預測框的每一行根據空格劃分，splitlines_pre爲二維數組   
        splitlines_pre = [x.strip().split(' ') for x in lines_pre] 
        #image_ids爲一維數組，當第二列的類別與遍歷的類別數相同時，將第一列加入到image_ids中     
        image_ids = [x[0] for x in splitlines_pre if int(x[1])==cls_num] 
        #提取該類別預測框的置信度，當第二列的類別與遍歷的類別數相同時，將第三列加入到confidence中
        confidence = np.array([float(x[2]) for x in splitlines_pre if int(x[1])==cls_num]) 
        #提取該類別預測框的bounging-boxes（二維數組），當第二列的類別與遍歷的類別數相同時,將第三列之後的BBOX座標插入
        BB_pre = np.array([[float(z) for z in x[3:]] for x in splitlines_pre if int(x[1])==cls_num]) 
        #按置信度大小將其索引從小到大排序（生成有順序的一維數組）
        sorted_ind = np.argsort(-confidence)
        #按置信度大小將置信度從小到大排序（生成有順序的一維數組）
        sorted_scores = np.sort(-confidence)
        #根據索引排序相應的bbox的座標值（生成按置信度大小排列的二維數組）
        BB_pre = BB_pre[sorted_ind, :]
        #按置信度大小重新排列image_ids
        image_ids = [image_ids[x] for x in sorted_ind]
        #該類別的預測框的數量
        nd = len(image_ids)
        tp = np.zeros(nd)#將長度爲nd數組置0
        fp = np.zeros(nd)

        # print(nd)###############################################################################################################
        for d in range(nd):
            #按置信度順序提取類別框
            #print(image_ids[d].get(image_ids[d]))
            # if class_recs[image_ids[d]] not in class_recs:
            if class_recs.get(image_ids[d]):
                #print(image_ids[d])
                R = class_recs[image_ids[d]]
                #print(R)
                bb = np.array(BB_pre[d, :]).astype(float)#按置信度順序提取bounding box
                ovmax = -np.inf#ovmax爲負無窮大的數
                BBGT = np.array(R['bbox']).astype(float)#按置信度順序提取groudtruth bbox

                #計算iou
                if BBGT.size > 0:
                    ixmin = np.maximum(BBGT[:, 0], bb[0])
                    iymin = np.maximum(BBGT[:, 1], bb[1])
                    ixmax = np.minimum(BBGT[:, 2], bb[2])
                    iymax = np.minimum(BBGT[:, 3], bb[3])
                    iw = np.maximum(ixmax - ixmin + 1., 0.)
                    ih = np.maximum(iymax - iymin + 1., 0.)
                    inters = iw * ih

                    uni = ((bb[2] - bb[0] + 1.) * (bb[3] - bb[1] + 1.) +
                           (BBGT[:, 2] - BBGT[:, 0] + 1.) *
                           (BBGT[:, 3] - BBGT[:, 1] + 1.) - inters)

                    overlaps = inters / uni#重疊率
                    ovmax = np.max(overlaps)#按重疊率的大小從大到小排序重疊率
                    jmax = np.argmax(overlaps)#根據重疊率大小重新排序的索引

                if ovmax > ovthresh:#ovthresh=0.5閾值爲0.5，判斷tp和fp
                    if not R['difficult'][jmax]:
                        if not R['det'][jmax]:
                            tp[d] = 1.
                            R['det'][jmax] = 1
                        else:
                            fp[d] = 1.
                else:
                    fp[d] = 1.
            else:
                #print("*************************************************************"+str(d))
                fp[d]=1.


            
        fp = np.cumsum(fp)
        tp = np.cumsum(tp)
        #rec=tp/正樣本數
        rec = tp / float(npos)
        #print(rec)
        # print(type(rec))
        #perc=tp/(tp+fp)
        prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps)
        #?print(prec)
        # print(type(prec))
        ap = voc_ap(rec, prec, use_07_metric)
        #print(ap)
        if cls_name[cls_num] not in pr_data:
            pr_data.update({cls_name[cls_num]:[rec,prec,ap]})
    return pr_data #,mAP/7,np.array(mAP_rec/7),np.array(mAP_prec/7)

def voc_ap(rec, prec, use_07_metric):#由於use_07_metric=true時計算結果於實際更接近
    #計算ap，use_07_metric=true，# 2010年以前按recall等間隔取11個不同點處的精度值做平均(0., 0.1, 0.2, …, 0.9, 1.0)
    if use_07_metric:
        ap = 0.
        for t in np.arange(0., 1.1, 0.1):#（[0.0,0.1,0.2,0.3,...,1.0])
            #print(11111111111111111111111111)
            if np.sum(rec >= t) == 0:
                p = 0
            else:
                p = np.max(prec[rec >= t])
            ap = ap + p / 11.
    #use_07_metric=false # 2010年以後取所有不同的recall對應的點處的精度值做平均
    else:
        mrec = np.concatenate(([0.], rec, [1.]))
        mpre = np.concatenate(([0.], prec, [0.]))
        for i in range(mpre.size - 1, 0, -1):
            mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])
        i = np.where(mrec[1:] != mrec[:-1])[0]
        ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])
    return ap

def draw_pr(prediction_file,prediction_algorithm,ground_truth_file,cls,use_07_metric):

    #畫多個算法多個類別的pr曲線
    MAP=[0, 0, 0, 0, 0, 0, 0]#用於統計map
    COLOR = ["#054E9F", "#FFA500", "#B0C4DE", "#008000", "#f6aad0", "#429ffd", "#435055"]
    for cls_name in cls:
        for i in range(len(prediction_file)):
            pr_data=get_pr_data("Prediction/"+prediction_file[i],"Ground_Truth/"+ground_truth_file[i],cls,use_07_metric[i])
            MAP[i]=MAP[i]+pr_data[cls_name][2]
            the_color = COLOR[i]
            plt.plot(pr_data[cls_name][0],pr_data[cls_name][1],label=prediction_algorithm[i],color=the_color)
            title=cls_name+' PR Curve'
            plt.title(title)
            plt.xlabel('Recall')
            plt.ylabel('Precision')
            plt.ylim([0.0, 1.0])
            plt.xlim([0.0, 1.0])
            plt.grid(ls='-.')
            plt.legend()
        plt.savefig("Image_PR/"+title+'.png', dpi=600)
        # plt.show()
        plt.close() 

    #畫各個算法map的pr曲線
    for i in range(len(prediction_file)): 
        pr_data_map=get_pr_data_map("Prediction/"+prediction_file[i],"Ground_Truth/"+ground_truth_file[i],cls,use_07_metric[i])
        the_color = COLOR[i]
        plt.plot(pr_data_map[0],pr_data_map[1],label=prediction_algorithm[i],color=the_color)
        title='Precision-Recall Curve'
        plt.title(title)
        plt.xlabel('Recall')
        plt.ylabel('Precision')
        plt.ylim([0.0, 1.0])
        plt.xlim([0.0, 1.0])
        plt.grid(ls='-.')
        plt.legend()
    plt.savefig("Image_PR/"+title+'.png', dpi=600)
    plt.show()
    plt.close() 


# if __name__ == '__main__':
#     # -*- coding: UTF-8 -*-
#     import matplotlib.pyplot as plt
#     cls=['pedestrian', 'people', 'bicycle', 'car', 'van', 'truck', 'tricycle', 'awning_tricycle', 'bus', 'motor'] # 類別
#     # prediction_file=["FRCNN.txt", 'RetinaNet.txt', 'CornerNet.txt', 'CenterNet.txt', 'tridentnet.txt', 'ourResult.txt', 'SSD.txt'] # 放在Prediction裏面"Faster_RCNN.txt"，記得新建文件夾Prediction
#     # prediction_algorithm=["FRCNN", 'RetinaNet', 'CornerNet', 'CenterNet', 'tridentnet', 'ourResult', 'SSD'] # 對應上面的文件的算法名稱
#     # use_07_metric=[False, False, False, False, False, False, False]
#     # ground_truth_file=["gt.txt", 'gt.txt', 'gt.txt', 'gt.txt', 'gt.txt', 'gt.txt', 'gt.txt']# 放在Ground Truth裏面，記得新建文件夾Ground Truth
#     prediction_file=["FRCNN.txt", 'RetinaNet.txt', 'ourResult.txt', 'SSD.txt'] # 放在Prediction裏面"Faster_RCNN.txt"，記得新建文件夾Prediction
#     prediction_algorithm=["FRCNN", 'RetinaNet', 'ourResult', 'SSD'] # 對應上面的文件的算法名稱
#     use_07_metric=[False, False, False, False]
#     ground_truth_file=["gt.txt", 'gt.txt', 'gt.txt', 'gt.txt']# 放在Ground Truth裏面，記得新建文件夾Ground Truth

#     # 繪製出了各個類別的Precision-Recall曲線
#     draw_pr(prediction_file,prediction_algorithm,ground_truth_file,cls,use_07_metric)

froc.py

# -*- coding: UTF-8 -*-
#以faster和yolo爲例
import numpy as np
import math
import matplotlib.pyplot as plt

def get_pr_data_map(prediction_file,ground_truth_file,cls_name,use_07_metric,ovthresh=0.5):
    with open(ground_truth_file, 'r') as f:####讀取各個算法的gt.txt文件
        lines_gt = f.readlines()          #gt.txt的每一行，line_gt爲一維數組
    with open(prediction_file, 'r') as f:####讀取各個算法的pre.txt文件
        lines_pre = f.readlines()         #讀取pre.txt的每一行，lines_gt爲一維數組
            #根據txt文件中每一行的空格進行劃分，splitlines_gt爲二維數組，行爲每一行的數據，列爲每一行的數據劃分
    splitlines_gt = [x.strip().split(' ') for x in lines_gt]
    #imagenames_gt爲一維數組，當第二列的類別與遍歷的類別數相同時，將第一列加入到imagenames_gt中
    imagenames_gt = [ x[0] for x in splitlines_gt ] 
    #print(imagenames_gt)    
    class_recs = {}       
#     #BB_gt爲二維數組，當第二列的類別與遍歷的類別數相同時，將第二列之後的加入到BB_gt中
    BB_gt=np.array([[math.ceil(float(z)) for z in x[2:]] for x in splitlines_gt])
    for i in range(len(imagenames_gt)):
        #如果類別中沒有該遍歷的類別，則更新
        if imagenames_gt[i] not in class_recs:
            class_recs.update({imagenames_gt[i]:{"bbox":[],"det":[],"difficult":[]}})
        #否則將BB_gt[i]插入到類別框信息中
        class_recs[imagenames_gt[i]]["bbox"].append(BB_gt[i])
        class_recs[imagenames_gt[i]]["det"].append(False)
        class_recs[imagenames_gt[i]]["difficult"].append(False)
            #npos爲imagenames_gt的長度
    npos=len(imagenames_gt)
    splitlines_pre = [x.strip().split(' ') for x in lines_pre] 
    #image_ids爲一維數組，當第二列的類別與遍歷的類別數相同時，將第一列加入到image_ids中     
    image_ids = [x[0] for x in splitlines_pre] 
    #提取該類別預測框的置信度，當第二列的類別與遍歷的類別數相同時，將第三列加入到confidence中
    confidence = np.array([float(x[2]) for x in splitlines_pre]) 
    #提取該類別預測框的bounging-boxes（二維數組），當第二列的類別與遍歷的類別數相同時,將第三列之後的BBOX座標插入
    BB_pre = np.array([[float(z) for z in x[3:]] for x in splitlines_pre]) 
    #按置信度大小將其索引從小到大排序（生成有順序的一維數組）
    sorted_ind = np.argsort(-confidence)
    #按置信度大小將置信度從小到大排序（生成有順序的一維數組）
    sorted_scores = np.sort(-confidence)
    #根據索引排序相應的bbox的座標值（生成按置信度大小排列的二維數組）
    BB_pre = BB_pre[sorted_ind, :]
    #按置信度大小重新排列image_ids
    image_ids = [image_ids[x] for x in sorted_ind]
    nd = len(image_ids)
    tp = np.zeros(nd)
    fp = np.zeros(nd)
    pr_data_map=[]
    # print(nd)###############################################################################################################
    for d in range(nd):
        #按置信度順序提取類別框
        #print(image_ids[d].get(image_ids[d]))
        # if class_recs[image_ids[d]] not in class_recs:
        # print(class_recs)
        if class_recs.get(image_ids[d]):
            #print(image_ids[d])
            R = class_recs[image_ids[d]]
            #print(R)
            bb = np.array(BB_pre[d, :]).astype(float)#按置信度順序提取bounding box
            ovmax = -np.inf#ovmax爲負無窮大的數
            BBGT = np.array(R['bbox']).astype(float)#按置信度順序提取groudtruth bbox

            #計算iou
            if BBGT.size > 0:
                ixmin = np.maximum(BBGT[:, 0], bb[0])
                iymin = np.maximum(BBGT[:, 1], bb[1])
                ixmax = np.minimum(BBGT[:, 2], bb[2])
                iymax = np.minimum(BBGT[:, 3], bb[3])
                iw = np.maximum(ixmax - ixmin + 1., 0.)
                ih = np.maximum(iymax - iymin + 1., 0.)
                inters = iw * ih

                uni = ((bb[2] - bb[0] + 1.) * (bb[3] - bb[1] + 1.) +
                       (BBGT[:, 2] - BBGT[:, 0] + 1.) *
                       (BBGT[:, 3] - BBGT[:, 1] + 1.) - inters)

                overlaps = inters / uni#重疊率
                ovmax = np.max(overlaps)#按重疊率的大小從大到小排序重疊率
                jmax = np.argmax(overlaps)#根據重疊率大小重新排序的索引

            if ovmax > ovthresh:#ovthresh=0.5閾值爲0.5，判斷tp和fp
                if not R['difficult'][jmax]:
                    if not R['det'][jmax]:
                        tp[d] = 1.
                        R['det'][jmax] = 1
                    else:
                        fp[d] = 1.
            else:
                fp[d] = 1.
        else:
            #print("*************************************************************"+str(d))
            fp[d]=1.


        
    fp = np.cumsum(fp)
    tp = np.cumsum(tp)
    #rec=tp/正樣本數
    rec = tp / float(npos)
    # print(len(rec))
    # print(type(rec))
    #perc=tp/(tp+fp)
    prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps)
    # print(len(prec))
    fps = fp / npos
    # print(fps, prec, nd)
    # print(type(prec))
    ap = voc_ap(rec, prec, use_07_metric)
    print(ap)
    ###################################################
    
    # mAP_rec+=rec
    # print("mAP_rec:"+str(mAP_rec)+"\n")
    # mAP_prec+=prec
    # print("mAP_prec:"+str(mAP_prec)+"\n")
    # mAP+=ap
    # print("mAP:"+str(mAP)+"\n")
    ###################################################
    # if not pr_data.has_key(cls_name[cls_num]):
    # rec=np.array(rec)
    # prec=np.array(prec)
    # ap=np.array(ap)
    pr_data_map=np.array([fps.tolist(), rec.tolist()])#,ap.tolist()]
    pr_data_map=pr_data_map.tolist()

    #print(pr_data_map)
    return pr_data_map 


def get_pr_data(prediction_file,ground_truth_file,cls_name,use_07_metric,ovthresh=0.5):
    with open(ground_truth_file, 'r') as f:####讀取各個算法的gt.txt文件
        lines_gt = f.readlines()          #gt.txt的每一行，line_gt爲一維數組
    with open(prediction_file, 'r') as f:####讀取各個算法的pre.txt文件
        lines_pre = f.readlines()         #讀取pre.txt的每一行，lines_gt爲一維數組

    pr_data={}

    for cls_num in range(len(cls_name)): #遍歷每個類別數
    #ground_truth  
    	#根據txt文件中每一行的空格進行劃分，splitlines_gt爲二維數組，行爲每一行的數據，列爲每一行的數據劃分
        splitlines_gt = [x.strip().split(' ') for x in lines_gt]
        #imagenames_gt爲一維數組，當第二列的類別與遍歷的類別數相同時，將第一列加入到imagenames_gt中
        imagenames_gt = [ x[0] for x in splitlines_gt if int(x[1])==cls_num] 
        #print(imagenames_gt)           

        #BB_gt爲二維數組，當第二列的類別與遍歷的類別數相同時，將第二列之後的加入到BB_gt中
        BB_gt=np.array([[math.ceil(float(z)) for z in x[2:]] for x in splitlines_gt if int(x[1])==cls_num])
        #創建class_recs數組
        class_recs = {}
        for i in range(len(imagenames_gt)):
            #如果類別中沒有該遍歷的類別，則更新
            if imagenames_gt[i] not in class_recs:
                class_recs.update({imagenames_gt[i]:{"bbox":[],"det":[],"difficult":[]}})
            #否則將BB_gt[i]插入到類別框信息中
            class_recs[imagenames_gt[i]]["bbox"].append(BB_gt[i])
            class_recs[imagenames_gt[i]]["det"].append(False)
            class_recs[imagenames_gt[i]]["difficult"].append(False)
            #print(class_recs[imagenames_gt[i]])
        #npos爲imagenames_gt的長度
        npos=len(imagenames_gt)
    
    #prediction 
    	#對預測框的每一行根據空格劃分，splitlines_pre爲二維數組   
        splitlines_pre = [x.strip().split(' ') for x in lines_pre] 
        #image_ids爲一維數組，當第二列的類別與遍歷的類別數相同時，將第一列加入到image_ids中     
        image_ids = [x[0] for x in splitlines_pre if int(x[1])==cls_num] 
		#提取該類別預測框的置信度，當第二列的類別與遍歷的類別數相同時，將第三列加入到confidence中
        confidence = np.array([float(x[2]) for x in splitlines_pre if int(x[1])==cls_num]) 
        #提取該類別預測框的bounging-boxes（二維數組），當第二列的類別與遍歷的類別數相同時,將第三列之後的BBOX座標插入
        BB_pre = np.array([[float(z) for z in x[3:]] for x in splitlines_pre if int(x[1])==cls_num]) 
        #按置信度大小將其索引從小到大排序（生成有順序的一維數組）
        sorted_ind = np.argsort(-confidence)
        #按置信度大小將置信度從小到大排序（生成有順序的一維數組）
        sorted_scores = np.sort(-confidence)
        #根據索引排序相應的bbox的座標值（生成按置信度大小排列的二維數組）
        BB_pre = BB_pre[sorted_ind, :]
        #按置信度大小重新排列image_ids
        image_ids = [image_ids[x] for x in sorted_ind]
        #該類別的預測框的數量
        nd = len(image_ids)
        tp = np.zeros(nd)#將長度爲nd數組置0
        fp = np.zeros(nd)

        # print(nd)###############################################################################################################
        for d in range(nd):
        	#按置信度順序提取類別框
        	#print(image_ids[d].get(image_ids[d]))
            # if class_recs[image_ids[d]] not in class_recs:
            if class_recs.get(image_ids[d]):
                #print(image_ids[d])
                R = class_recs[image_ids[d]]
                #print(R)
                bb = np.array(BB_pre[d, :]).astype(float)#按置信度順序提取bounding box
                ovmax = -np.inf#ovmax爲負無窮大的數
                BBGT = np.array(R['bbox']).astype(float)#按置信度順序提取groudtruth bbox

	            #計算iou
                if BBGT.size > 0:
                    ixmin = np.maximum(BBGT[:, 0], bb[0])
                    iymin = np.maximum(BBGT[:, 1], bb[1])
                    ixmax = np.minimum(BBGT[:, 2], bb[2])
                    iymax = np.minimum(BBGT[:, 3], bb[3])
                    iw = np.maximum(ixmax - ixmin + 1., 0.)
                    ih = np.maximum(iymax - iymin + 1., 0.)
                    inters = iw * ih

                    uni = ((bb[2] - bb[0] + 1.) * (bb[3] - bb[1] + 1.) +
                           (BBGT[:, 2] - BBGT[:, 0] + 1.) *
                           (BBGT[:, 3] - BBGT[:, 1] + 1.) - inters)

                    overlaps = inters / uni#重疊率
                    ovmax = np.max(overlaps)#按重疊率的大小從大到小排序重疊率
                    jmax = np.argmax(overlaps)#根據重疊率大小重新排序的索引

                if ovmax > ovthresh:#ovthresh=0.5閾值爲0.5，判斷tp和fp
                    if not R['difficult'][jmax]:
                        if not R['det'][jmax]:
                            tp[d] = 1.
                            R['det'][jmax] = 1
                        else:
                            fp[d] = 1.
                else:
                    fp[d] = 1.
            else:
                #print("*************************************************************"+str(d))
                fp[d]=1.


            
        fp = np.cumsum(fp)
        tp = np.cumsum(tp)
        #rec=tp/正樣本數
        rec = tp / float(npos)
        #print(rec)
        # print(type(rec))
        #perc=tp/(tp+fp)
        prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps)
        #?print(prec)
        # print(type(prec))
        ap = voc_ap(rec, prec, use_07_metric)
        #print(ap)
        ###################################################
        fps = fp / npos
        
        # mAP_rec+=rec
        # print("mAP_rec:"+str(mAP_rec)+"\n")
        # mAP_prec+=prec
        # print("mAP_prec:"+str(mAP_prec)+"\n")
        # mAP+=ap
        # print("mAP:"+str(mAP)+"\n")
        ###################################################
        if cls_name[cls_num] not in pr_data:
            pr_data.update({cls_name[cls_num]:[fps,rec,ap]})
    return pr_data #,mAP/7,np.array(mAP_rec/7),np.array(mAP_prec/7)

def voc_ap(rec, prec, use_07_metric):#由於use_07_metric=true時計算結果於實際更接近
    #計算ap，use_07_metric=true，# 2010年以前按recall等間隔取11個不同點處的精度值做平均(0., 0.1, 0.2, …, 0.9, 1.0)
    if use_07_metric:
        ap = 0.
        for t in np.arange(0., 1.1, 0.1):#（[0.0,0.1,0.2,0.3,...,1.0])
            #print(11111111111111111111111111)
            if np.sum(rec >= t) == 0:
                p = 0
            else:
                p = np.max(prec[rec >= t])
            ap = ap + p / 11.
    #use_07_metric=false # 2010年以後取所有不同的recall對應的點處的精度值做平均
    else:
        mrec = np.concatenate(([0.], rec, [1.]))
        mpre = np.concatenate(([0.], prec, [0.]))
        for i in range(mpre.size - 1, 0, -1):
            mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])
        i = np.where(mrec[1:] != mrec[:-1])[0]
        ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])
    return ap

def draw_pr(prediction_file,prediction_algorithm,ground_truth_file,cls,use_07_metric):
    # for i in range(len(prediction_file)):
    #     pr_data=get_pr_data("Prediction/"+prediction_file[i],"Ground_Truth/"+ground_truth_file,cls)
    #     for cls_name in pr_data:
    #         plt.plot(pr_data[cls_name][0],pr_data[cls_name][1],label=prediction_algorithm[i]+' mAP='+str(round(pr_data[cls_name][2],3)))
    #         title='PR Curve of '+cls_name
    #         plt.title(title)
    #         plt.xlabel('Recall')
    #         plt.ylabel('Precision')
    #         plt.ylim([0.0, 1.0])
    #         plt.xlim([0.0, 1.0])
    #         plt.grid(ls='-.')
    #         plt.legend()
    #         plt.savefig("Images/"+title+'.png', dpi=300)
    #         plt.show()
    #         plt.close()
    # for i in range(len(prediction_file)):
    #     pr_data=get_pr_data("Prediction/"+prediction_file[i],"Ground_Truth/"+ground_truth_file,cls)

    #畫多個算法多個類別的pr曲線
    MAP=[0,0,0,0,0,0]#用於統計map
    for cls_name in cls:
        for i in range(len(prediction_file)):
            pr_data=get_pr_data("Prediction/"+prediction_file[i],"Ground_Truth/"+ground_truth_file[i],cls,use_07_metric[i])
            MAP[i]=MAP[i]+pr_data[cls_name][2]
            # print(pr_data)

            if i==0:
                plt.plot(pr_data[cls_name][0],pr_data[cls_name][1],label=prediction_algorithm[i]+' AP='+str(round(pr_data[cls_name][2],3)),color="#054E9F")
            elif i==1:
                plt.plot(pr_data[cls_name][0],pr_data[cls_name][1],label=prediction_algorithm[i]+' AP='+str(round(pr_data[cls_name][2],3)),color="#FFA500")
            elif i==2:
                plt.plot(pr_data[cls_name][0],pr_data[cls_name][1],label=prediction_algorithm[i]+' AP='+str(round(pr_data[cls_name][2],3)),color="#B0C4DE")
            elif i==3:
                plt.plot(pr_data[cls_name][0],pr_data[cls_name][1],label=prediction_algorithm[i]+' AP='+str(round(pr_data[cls_name][2],3)),color="#008000")
            elif i==4:
                plt.plot(pr_data[cls_name][0],pr_data[cls_name][1],label=prediction_algorithm[i]+' AP='+str(round(pr_data[cls_name][2],3)),color="#BA55D3")
            elif i==5:
                plt.plot(pr_data[cls_name][0],pr_data[cls_name][1],label=prediction_algorithm[i]+' AP='+str(round(pr_data[cls_name][2],3)),color="#FF0000")
            # plt.plot(pr_data[cls_name][0],pr_data[cls_name][1],label=prediction_algorithm[i]+' AP='+str(round(pr_data[cls_name][2],3)),color="#054E9F")
            title=cls_name+' FROC Curve'
            plt.title(title,fontsize=15)
            plt.xlabel('False Positive Per Scan',fontsize=15)
            plt.ylabel('Sensitive',fontsize=15)
            plt.ylim([0.0, 1.0])
            plt.xlim([0.0, 10])
            plt.grid(ls='-.')
            plt.legend()
            plt.savefig("Image_PR/"+title+'.png', dpi=600)
            print("save....ok!!!")
        plt.show()
        plt.close() 

    #畫各個算法map的pr曲線
    for i in range(len(prediction_file)): 
        pr_data_map=get_pr_data_map("Prediction/"+prediction_file[i],"Ground_Truth/"+ground_truth_file[i],cls,use_07_metric[i])
        if i==0:
            plt.plot(pr_data_map[0],pr_data_map[1],label=prediction_algorithm[i]+' mAP='+str(round(MAP[i]/len(cls),3)),color="#054E9F")
        elif i==1:
            plt.plot(pr_data_map[0],pr_data_map[1],label=prediction_algorithm[i]+' mAP='+str(round(MAP[i]/len(cls),3)),color="#FFA500")
        elif i==2:
            plt.plot(pr_data_map[0],pr_data_map[1],label=prediction_algorithm[i]+' mAP='+str(round(MAP[i]/len(cls),3)),color="#B0C4DE")
        elif i==3:
            plt.plot(pr_data_map[0],pr_data_map[1],label=prediction_algorithm[i]+' mAP='+str(round(MAP[i]/len(cls),3)),color="#008000")
        elif i==4:
            plt.plot(pr_data_map[0],pr_data_map[1],label=prediction_algorithm[i]+' mAP='+str(round(MAP[i]/len(cls),3)),color="#BA55D3")
        elif i==5:
            plt.plot(pr_data_map[0],pr_data_map[1],label=prediction_algorithm[i]+' mAP='+str(round(MAP[i]/len(cls),3)),color="#FF0000")
        #plt.plot(pr_data_map[0],pr_data_map[1],label=prediction_algorithm[i]+' mAP='+str(round(MAP[i]/len(cls),3)),color="#054E9F")
        title='FROC Curve'
        plt.title(title,fontsize=15)
        plt.xlabel('False Positive Per Scan',fontsize=15)
        plt.ylabel('Sensitive',fontsize=15)
        plt.ylim([0.0, 1.0])
        plt.xlim([0.0, 10])
        plt.grid(ls='-.')
        plt.legend()
        plt.savefig("Image_PR/"+title+'.png', dpi=600)
        print("save....ok!!!")
    plt.show()
    plt.close()