Merge remote-tracking branch 'origin/master'

.gitignore

@@ -60,6 +60,7 @@ VOC/
 *_saved_model/
 *_web_model/
 *_openvino_model/
+openvino_models/*
 darknet53.conv.74
 yolov3-tiny.conv.15
 

exp1_perf.py

@@ -1,11 +1,12 @@
 from val import run
 from datetime import datetime
+from exp1_speed import run_exp1_speed
 import numpy as np
 import pandas as pd
 
-MODELS = ["yolov5n", "yolov5s", "yolov5m", "yolov5l"]
 
-MODELS_P6 = ["yolov5n6", "yolov5s6", "yolov5m6",  "yolov5l6"]
+MODELS = ["yolov5n", "yolov5s", "yolov5m", "yolov5l",
+          "yolov5n6", "yolov5s6", "yolov5m6",  "yolov5l6"]
 
 PRECISION = ['int8', 'fp16', 'fp32']
 
@@ -84,8 +85,9 @@ def run_exp1_perf(models, precisions):
 
 
 if __name__ == "__main__":
-    run_exp1_perf(MODELS, PRECISION)
-    run_exp1_perf(MODELS_P6, PRECISION)
-    #run_exp1_perf(['yolov5l6'], ['int8'])
+    run_exp1_perf(['yolov5l6'], ['fp16', 'fp32'])
+    run_exp1_speed(['yolov5l6'], ['fp16', 'fp32'])
+
+
 
 

exp1_speed.py

@@ -1,29 +1,34 @@
 from detect import run
 from datetime import datetime
 import pandas as pd
+from pathlib import Path
 
-MODELS = ["yolov5n", "yolov5s", "yolov5m", "yolov5l"
-          # "yolov5n6", "yolov5s6", "yolov5m6", #"yolov5l6"
-          ]
+MODELS = ["yolov5n", "yolov5s", "yolov5m", "yolov5l",
+          "yolov5n6", "yolov5s6", "yolov5m6", "yolov5l6"]
 
 PRECISION = ['int8', 'fp16', 'fp32']
 
 
-def run_exp1_speed():
+def run_exp1_speed(models, precisions):
     column_names = ["model", "precision", "prep_time", "NMS_time", "latency", "inference_time",
                     "total_time",
                     "FPS", "experiment_time"]
     exp1_speed = pd.DataFrame(columns=column_names)
 
     counter = 0
-    for model in MODELS:
+    for model in models:
         imgsize = 1280 if '6' in model else 640
-        for precision in PRECISION:
+        for precision in precisions:
+            if model == 'yolov5l6' and precision == 'int8':
+                break
             start_experiment = datetime.now()
             row = [model, precision]
             print(row)
+            model_path = Path(f'./openvino_models/{model}_{precision}_{imgsize}')
+            print(model_path)
+
             temp = run(
-                weights=model + '_openvino_model_' + precision,
+                weights=model_path,
                 source="../datasets/coco/images/val2017",  # 000000463199.jpg
                 nosave=True,
                 imgsz=(imgsize, imgsize)
@@ -40,12 +45,13 @@ def run_exp1_speed():
             counter += 1
     print(exp1_speed)
     # store results
-    filename = f'exp1_results/exp1_speed_{datetime.now().strftime("%d-%m-%Y_%H-%M")}'
+    filename = Path(f'results/experiments/exp1/{datetime.now().strftime("%y%m%d")}_speed')
+    filename.parent.mkdir(parents=True, exist_ok=True)
     exp1_speed.round(3)
     print(exp1_speed)
-    exp1_speed.to_pickle(filename + '.pkl')
-    exp1_speed.to_csv(filename + '.csv')
+    exp1_speed.to_pickle(str(filename) + '.pkl')
+    exp1_speed.to_csv(str(filename) + '.csv')
 
 
 if __name__ == "__main__":
-    run_exp1_speed()
+    run_exp1_speed(['yolov5l6'], ['fp16', 'fp32'])

exp2_res_speed.py

@@ -3,15 +3,11 @@
 import pandas as pd
 from pathlib import Path
 
-MODELS = ["yolov5n", "yolov5s", "yolov5m", "yolov5l"]
-
-MODELS_P6 = ["yolov5n6", "yolov5s6", "yolov5m6", "yolov5l6"]
+MODELS = ["yolov5n", "yolov5s", "yolov5m", "yolov5l", "yolov5n6", "yolov5s6", "yolov5m6", "yolov5l6"]
 
 PRECISION = ['fp16']
 
-IMAGE_SIZES = [320, 480, 600, 800, 960]
-
-IMAGE_SIZES_P6 = [256, 448, 640, 832, 1024]
+IMAGE_SIZES = [256, 384, 512, 640, 768, 896, 1024]
 
 
 def run_exp2_res_speed(models, precisions, image_sizes):
@@ -56,6 +52,5 @@ def run_exp2_res_speed(models, precisions, image_sizes):
 
 
 if __name__ == "__main__":
-    #run_exp2_res_speed(MODELS, PRECISION, IMAGE_SIZES)
-    run_exp2_res_speed(MODELS_P6, ['fp16'], [1280])
-    #run_exp2_res_speed(['yolov5n'], ['fp16'], [320])
+    run_exp2_res_speed(MODELS, PRECISION, IMAGE_SIZES)
+

export_openvino_models.py

@@ -6,12 +6,12 @@
 
 PRECISION = ['fp16']
 
-
 # try different images sizes, 640 not necessary as already pretrained on 640
 IMAGE_SIZES = [320, 480, 640, 800, 960]
 
 IMAGE_SIZES_P6 = [256, 448, 640, 832, 1024]  # image size for P6 models (multiple of stride 64)
 
+IMAGE_SIZES_EXTRA = [256, 384, 512, 640, 768, 896, 1024]
 
 
 def export_models(models, precisions, image_sizes):
@@ -25,6 +25,8 @@ def export_models(models, precisions, image_sizes):
 
 
 if __name__ == "__main__":
-    export_models(MODELS, PRECISION, IMAGE_SIZES)
-    #export_models(MODELS, ['fp16'], IMAGE_SIZES)
-
+    # export_models(MODELS, PRECISION, IMAGE_SIZES)
+    # export_models(MODELS, ['fp16'], IMAGE_SIZES_P6)
+    # export_models(MODELS_P6, ['fp32'], [1280])
+    export_models(MODELS, ['fp16'], IMAGE_SIZES_EXTRA)
+    export_models(MODELS_P6, ['fp16'], IMAGE_SIZES_EXTRA)

quantize_default.py

@@ -162,5 +162,5 @@ def __getitem__(self, index):
 
 if __name__ == "__main__":
     #export_models()
-    quantize_models(MODELS)
+    #quantize_models(MODELS)
     quantize_models(MODELS_P6[:3])

requirements.txt

@@ -19,6 +19,7 @@ wandb~=0.12.11
 # Plotting ------------------------------------
 pandas>=1.1.4
 seaborn>=0.11.0
+openpyxl
 
 # Export --------------------------------------
 # coremltools>=4.1  # CoreML export

results/experiments/exp1/20220604_exp1_perf.csv

@@ -0,0 +1,22 @@
+,model,img_size,precision,mAP50,mAP,mAP50_8class,mAP_8class,person_mAP.5,car_mAP.5,motorcycle_mAP.5,bus_mAP.5,truck_mAP.5,baseball_bat_mAP.5,knife_mAP.5,cell_phone_mAP.5,person_mAP,car_mAP,motorcycle_mAP,bus_mAP,truck_mAP,baseball_bat_mAP,knife_mAP,cell_phone_mAP,experiment_time
+0,yolov5n,640,int8,0.4314231790001692,0.24787113508957465,0.44083360141276473,0.24436714815165936,0.6793708211297177,0.5025362184275795,0.5801139747564206,0.6622326101099674,0.38428856646459864,0.30931386764379176,0.09652270122416587,0.3122900515458762,0.37804723103460447,0.267131242243343,0.2584006148699476,0.48249025635231596,0.2103767797416003,0.125431161702402,0.04716161937280068,0.18589827989626084,523
+1,yolov5n,640,fp16,0.45486181335672693,0.2707529622038651,0.4596909375405537,0.258513076585943,0.6999077396528962,0.5132597515121207,0.5951781886862397,0.6797055507746579,0.3984619269841213,0.35216091153233664,0.11438555494193574,0.32446787624012124,0.40449227277285155,0.283648180592865,0.27465063275597384,0.503928440712625,0.2211901694381948,0.14229414622091546,0.048771945793746066,0.18912882440037262,382
+2,yolov5n,640,fp32,0.45526599098321785,0.2707861004490829,0.4590413789650507,0.25854134177827903,0.7005021872894625,0.5136383012550454,0.5960522143627862,0.6798647771212062,0.39790708764594973,0.3442623494173893,0.11542367870857899,0.32468043591998763,0.40491218729647177,0.28356069354804236,0.2763462752151591,0.5039140333929752,0.2214126578300705,0.14179479429319997,0.04863925333243175,0.18775083931788183,434
+3,yolov5s,640,int8,0.5541858113039659,0.3491968312148317,0.5619838666372421,0.3384407866802551,0.7544593357223895,0.6128128522017561,0.6653445623113713,0.7691352127986419,0.5099490323837728,0.4967093082834019,0.23623147898938643,0.45122915040721645,0.47135840765521786,0.3556965358396609,0.36556303793713985,0.6049268769209662,0.3030631139878573,0.2226917897099494,0.12446552778654422,0.2597610036047051,847
+4,yolov5s,640,fp16,0.5645432682266455,0.363698641628669,0.571108152578838,0.35006803085230453,0.7628935427006011,0.6244165019414359,0.6769278079773049,0.7737022414787458,0.5176135846375015,0.5020525655007146,0.24049691218275165,0.4707620642116479,0.4873924527186255,0.3710039083401534,0.3699427607350578,0.6100052737194045,0.31635534604492876,0.22932849919310022,0.12922538373260822,0.2872906223345577,464
+5,yolov5s,640,fp32,0.5647636469369322,0.36419639452296904,0.5713870224710926,0.35092798041782014,0.7629392090822446,0.6249599551700238,0.6763895815320802,0.773758758460585,0.5177216421774546,0.5008692991300256,0.24487603283495393,0.469581701381373,0.4879464249047042,0.37144110133946323,0.37034914621998427,0.6113645563933637,0.3170884478537019,0.22948505487672838,0.1309029449620392,0.28884616679257646,720
+6,yolov5m,640,int8,0.6293221532487827,0.4277831505557422,0.656779847553475,0.42734865051789517,0.8031778308443034,0.6914636778729424,0.7450582165613004,0.8328889286280176,0.5897514226957639,0.6451725761464735,0.3558064158968575,0.5909197117821408,0.5412025348117713,0.4328577856239274,0.45028612889302577,0.6793854527314613,0.38838017189785273,0.35173092460474026,0.20393868055470316,0.37100752502567935,1660
+7,yolov5m,640,fp16,0.6357043635213712,0.4403215036123669,0.6606575539697976,0.4386370796239435,0.8070052528864594,0.7010515378498446,0.746281643485571,0.836767791222733,0.5906150206094708,0.6400036959774322,0.35415751461152195,0.6093779751153484,0.5553559604767373,0.45155058738045745,0.45918389983372687,0.6965452359720511,0.3978792251818622,0.3585475152129492,0.20596311959198826,0.38407109334177575,866
+8,yolov5m,640,fp32,0.6354700390642277,0.4403623722508371,0.6599687978956588,0.438284515469473,0.8068219356531682,0.7020519026233364,0.7476705376455017,0.8367546854503104,0.5908705394641256,0.635539349176066,0.35151732004888814,0.6085241131038732,0.5557391757571921,0.4501588719092072,0.4600929057191691,0.6974698799933015,0.3987241025499283,0.3555492381923115,0.20409802484839518,0.38444392478627903,1385
+9,yolov5l,640,int8,0.6600940308352422,0.460373750489769,0.6837101912133877,0.4565224980685859,0.8176839228399039,0.7225613686667797,0.7621413513557338,0.8557296370719534,0.6329282079056064,0.6853555718737215,0.3703364580792363,0.6229450119141673,0.5683499171613474,0.4630990521700344,0.469617491626788,0.7020913385136124,0.4249759459214614,0.38865691253797846,0.22203341699101395,0.4133559096264513,3146
+10,yolov5l,640,fp16,0.6665367902092766,0.47789390703235773,0.6898390735821485,0.471667935347832,0.8256948289734023,0.734441493189033,0.7524269864099606,0.8547283670111554,0.6382927548140075,0.6965918672006282,0.3908614667326936,0.6256748243263068,0.5871755369835433,0.4865698415078093,0.4874399054578185,0.7203807755746207,0.43882361352608157,0.39985244228202566,0.23275881886365118,0.4203425485871056,1427
+11,yolov5l,640,fp32,0.6664181307991731,0.478635600145122,0.6894984725918721,0.4729973059433904,0.825632442010252,0.7342926030528893,0.7532994349426476,0.8546580553675527,0.6389686157885038,0.6930848537021895,0.3894079320993114,0.6266438437716303,0.5874312312577594,0.4873386916087468,0.48766992330425296,0.7220311523463784,0.4401978352941912,0.40307775552807756,0.23402032238909332,0.4222115358186234,2994
+12,yolov5n6,1280,int8,0.450798943824819,0.2770467000897089,0.4875456534368053,0.2912662087400323,0.7288970365281424,0.5884788098528929,0.5643143161209007,0.7363965927395504,0.42623711312671364,0.3357948120825753,0.14827768964262647,0.37196885740104074,0.43747732906604453,0.33724843206111227,0.28654061028921807,0.5672939692324647,0.2608422691245019,0.15100642496558098,0.06776351552505203,0.2219571196562835,1816
+13,yolov5n6,1280,fp16,0.5400530473306253,0.3469462387511327,0.5514125951379049,0.34019907702465524,0.7763624221820323,0.6380425507016312,0.6590395930461523,0.7814670854939122,0.4881668181946613,0.4187403233566344,0.20972858024700305,0.43975338788121177,0.4970080358971506,0.3821795908120972,0.3611094050060574,0.6196541515662732,0.3048615853513525,0.19057632968957577,0.10214551616642775,0.26405800170830734,1089
+14,yolov5n6,1280,fp32,0.5401285615505159,0.34712347893384893,0.551489456509668,0.33977481401231957,0.7772950537191546,0.638019405863828,0.6614776817521646,0.7803467966864597,0.4883418140858524,0.4177848278420754,0.20751914058201532,0.44113093154579314,0.4973847520838521,0.3835887672185281,0.36126521829005,0.6193831315322063,0.3045619715341881,0.18731086087833335,0.10116788135532837,0.2635359292060703,1552
+15,yolov5s6,1280,int8,0.6241496303074341,0.4168397958075743,0.6482978654248087,0.41286016076443205,0.8151412127848457,0.705270614141835,0.7387345498912761,0.8285584170859971,0.582006099761898,0.5767382679481105,0.34887716565406,0.5910565961304478,0.5393327613721826,0.44257204474222667,0.43949791148462836,0.6636408149889623,0.38535837272201273,0.27607625276973147,0.19075037231407196,0.36565275572164035,3039
+16,yolov5s6,1280,fp16,0.6323960790510746,0.43420618394367827,0.6569917557402419,0.43145563869909653,0.8206833758039995,0.7150693358004645,0.7509916373851308,0.8316057209096914,0.5906168500273201,0.590837307604964,0.3493616384371493,0.6067681799532161,0.5617352470988275,0.4585890019336457,0.4551180939164049,0.684514140132925,0.39961043385793743,0.3118185919925624,0.1979364150804021,0.38232318558006734,1688
+17,yolov5s6,1280,fp32,0.6324991306171344,0.4343550349293713,0.6559187180400157,0.431612205894493,0.8207687384602991,0.7156447544793143,0.7464121036780811,0.8311842013990133,0.5889221352254326,0.5891904418532479,0.3471871933408957,0.6080401758838414,0.5619703383136276,0.4596882461060957,0.45418402532982294,0.684539292879407,0.3982810845469938,0.3126999568126403,0.19797830930549903,0.3835563938618577,3433
+18,yolov5m6,1280,int8,0.681199208802166,0.4847140745445867,0.7072789950451037,0.4866692722266781,0.8460871279011278,0.7609395816477315,0.7842403532289051,0.8697336268315163,0.62569036919707,0.6624618196379387,0.45952782051702884,0.6495512613995114,0.6016885968502462,0.5069393586191412,0.5069939844097241,0.7311927913400679,0.4447905070025806,0.3773621733829418,0.28398552081682155,0.44040124539190106,5771
+19,yolov5m6,1280,fp16,0.6867168349816257,0.4982664657841841,0.7166502575241687,0.5028562581020642,0.8507114800281481,0.7671696460270029,0.7882269089981908,0.8737055168155524,0.632929956475395,0.685159078872597,0.4693268566686746,0.665972616307788,0.6178298439488159,0.5244355658433183,0.5231347283069913,0.7452594698461851,0.45233258625216183,0.3947943432556957,0.2991228275040415,0.46594069985930436,3292
+20,yolov5m6,1280,fp32,0.6869069795931797,0.49844172187683017,0.7166566241619927,0.5033602299961913,0.8506847897246519,0.7673610063135161,0.7914834550321095,0.874124604199124,0.633171570390417,0.6812626618880426,0.46864988839592614,0.6665150173521548,0.6183516375056428,0.5245307549251312,0.5236923780360254,0.7474170851700175,0.451999298853148,0.3967630982952229,0.2987306430734117,0.46539694411093147,5781

results/experiments/exp1/20220609_exp1_perf.csv

@@ -0,0 +1,3 @@
+,model,img_size,precision,mAP50,mAP,mAP50_8class,mAP_8class,person_mAP.5,car_mAP.5,motorcycle_mAP.5,bus_mAP.5,truck_mAP.5,baseball_bat_mAP.5,knife_mAP.5,cell_phone_mAP.5,person_mAP,car_mAP,motorcycle_mAP,bus_mAP,truck_mAP,baseball_bat_mAP,knife_mAP,cell_phone_mAP,experiment_time
+0,yolov5l6,1280,fp16,0.7061042757231983,0.5227280554509581,0.7434818681153712,0.5324955592698563,0.8598390698011174,0.787859037238027,0.8079996242721972,0.8850437886752031,0.6673509598451254,0.7169808653401972,0.521954379672731,0.7008272200783721,0.6350552259668647,0.5494377454819125,0.5568775414703249,0.7704109662868025,0.4866806525207699,0.4243228408447344,0.34254745134870773,0.49463205023873397,6032
+1,yolov5l6,1280,fp32,0.7059118815833638,0.5228428252450639,0.7431273980514019,0.5321621208044109,0.859570592146744,0.7882380991650857,0.8070124688242202,0.8860877959175956,0.6669472257287786,0.7179466397444454,0.5192162308709506,0.7000001320133953,0.6359903312155086,0.5497617085113294,0.5565396262999778,0.7719261950705294,0.48700068198104296,0.4183031495369353,0.34314265535611216,0.4946326184638516,11163

results/experiments/exp1/220605_speed.csv

@@ -0,0 +1,22 @@
+,model,precision,prep_time,NMS_time,latency,inference_time,total_time,FPS,experiment_time
+0,yolov5n,int8,2.337548112869263,1.768181276321411,4.105729389190674,68.88695788383484,72.99268727302551,13.700002525726307,410
+1,yolov5n,fp16,2.374600076675415,1.430059242248535,3.80465931892395,34.67108125686646,38.475740575790404,25.990402914537196,253
+2,yolov5n,fp32,2.3621745586395266,2.0693668365478515,4.431541395187378,45.092685079574586,49.52422647476196,20.19213769062319,311
+3,yolov5s,int8,2.309380531311035,2.0558454513549806,4.365225982666016,146.81233563423157,151.17756161689758,6.61473825417374,800
+4,yolov5s,fp16,2.549609851837158,2.1765974998474125,4.72620735168457,60.3297658443451,65.05597319602967,15.371378689344233,392
+5,yolov5s,fp32,2.756035280227661,2.809509468078613,5.565544748306274,104.74162607192994,110.30717082023622,9.065593764793999,639
+6,yolov5m,int8,2.3247382640838623,1.6931139469146728,4.0178522109985355,309.2857801437378,313.3036323547363,3.1917919127977283,1610
+7,yolov5m,fp16,3.0497063159942623,2.878777551651001,5.928483867645263,127.29497909545897,133.22346296310425,7.506185305188669,763
+8,yolov5m,fp32,2.906248760223389,2.848623561859131,5.7548723220825195,254.92730817794802,260.68218050003054,3.836088826945664,1397
+9,yolov5l,int8,2.1997583866119386,1.755626630783081,3.9553850173950194,552.891881942749,556.8472669601441,1.7958245632757575,2829
+10,yolov5l,fp16,2.709670877456665,2.6728721141815184,5.382542991638184,247.74582324028017,253.12836623191836,3.950564746598931,1371
+11,yolov5l,fp32,2.758587741851807,2.929982089996338,5.688569831848145,524.281589460373,529.9701592922211,1.8868986913065202,2749
+12,yolov5n6,int8,9.233469152450562,4.684622764587402,13.918091917037964,315.48181357383726,329.3999054908752,3.035823578970946,1727
+13,yolov5n6,fp16,9.615099811553954,5.032219171524048,14.647318983078002,153.3368145942688,167.9841335773468,5.95294316614467,956
+14,yolov5n6,fp32,9.389935970306396,5.156487560272217,14.546423530578615,268.75860691070557,283.3050304412842,3.5297643619048023,1531
+15,yolov5s6,int8,8.927440929412843,5.030945777893066,13.958386707305909,566.1847447395326,580.1431314468384,1.723712556082611,2987
+16,yolov5s6,fp16,9.705197525024413,5.759062337875366,15.46425986289978,283.1850588798523,298.6493187427521,3.3484087765871355,1628
+17,yolov5s6,fp32,9.564798974990845,5.033862733840942,14.598661708831788,679.2221281528473,693.820789861679,1.4412943725704173,3592
+18,yolov5m6,int8,8.399726963043213,4.753249883651733,13.152976846694946,1191.6705033302305,1204.8234801769254,0.8299971045162171,6101
+19,yolov5m6,fp16,10.250411748886108,7.218113374710083,17.46852512359619,595.1217665672302,612.5902916908265,1.632412419138857,3218
+20,yolov5m6,fp32,10.374418830871582,7.458432197570801,17.832851028442384,1087.1622262477877,1104.99507727623,0.9049814072158233,5701

results/experiments/exp1/220609_speed.csv

@@ -0,0 +1,3 @@
+,model,precision,prep_time,NMS_time,latency,inference_time,total_time,FPS,experiment_time
+0,yolov5l6,fp16,10.344834518432616,6.876286220550536,17.221120738983153,1136.2903309345245,1153.5114516735077,0.8669181381330942,5939
+1,yolov5l6,fp32,10.763452577590941,7.065650749206543,17.829103326797483,2182.0223227500915,2199.8514260768893,0.45457615371023147,11185