diff --git a/.gitignore b/.gitignore index 69a00843ea42..4ef995cbb6a4 100755 --- a/.gitignore +++ b/.gitignore @@ -60,6 +60,7 @@ VOC/ *_saved_model/ *_web_model/ *_openvino_model/ +openvino_models/* darknet53.conv.74 yolov3-tiny.conv.15 diff --git a/exp1_perf.py b/exp1_perf.py index f699944ae9e9..1df4114eb911 100644 --- a/exp1_perf.py +++ b/exp1_perf.py @@ -1,11 +1,12 @@ from val import run from datetime import datetime +from exp1_speed import run_exp1_speed import numpy as np import pandas as pd -MODELS = ["yolov5n", "yolov5s", "yolov5m", "yolov5l"] -MODELS_P6 = ["yolov5n6", "yolov5s6", "yolov5m6", "yolov5l6"] +MODELS = ["yolov5n", "yolov5s", "yolov5m", "yolov5l", + "yolov5n6", "yolov5s6", "yolov5m6", "yolov5l6"] PRECISION = ['int8', 'fp16', 'fp32'] @@ -84,8 +85,9 @@ def run_exp1_perf(models, precisions): if __name__ == "__main__": - run_exp1_perf(MODELS, PRECISION) - run_exp1_perf(MODELS_P6, PRECISION) - #run_exp1_perf(['yolov5l6'], ['int8']) + run_exp1_perf(['yolov5l6'], ['fp16', 'fp32']) + run_exp1_speed(['yolov5l6'], ['fp16', 'fp32']) + + diff --git a/exp1_speed.py b/exp1_speed.py index 8c541e6f4f2e..247194792b97 100644 --- a/exp1_speed.py +++ b/exp1_speed.py @@ -1,29 +1,34 @@ from detect import run from datetime import datetime import pandas as pd +from pathlib import Path -MODELS = ["yolov5n", "yolov5s", "yolov5m", "yolov5l" - # "yolov5n6", "yolov5s6", "yolov5m6", #"yolov5l6" - ] +MODELS = ["yolov5n", "yolov5s", "yolov5m", "yolov5l", + "yolov5n6", "yolov5s6", "yolov5m6", "yolov5l6"] PRECISION = ['int8', 'fp16', 'fp32'] -def run_exp1_speed(): +def run_exp1_speed(models, precisions): column_names = ["model", "precision", "prep_time", "NMS_time", "latency", "inference_time", "total_time", "FPS", "experiment_time"] exp1_speed = pd.DataFrame(columns=column_names) counter = 0 - for model in MODELS: + for model in models: imgsize = 1280 if '6' in model else 640 - for precision in PRECISION: + for precision in precisions: + if model == 'yolov5l6' and precision == 'int8': + break start_experiment = datetime.now() row = [model, precision] print(row) + model_path = Path(f'./openvino_models/{model}_{precision}_{imgsize}') + print(model_path) + temp = run( - weights=model + '_openvino_model_' + precision, + weights=model_path, source="../datasets/coco/images/val2017", # 000000463199.jpg nosave=True, imgsz=(imgsize, imgsize) @@ -40,12 +45,13 @@ def run_exp1_speed(): counter += 1 print(exp1_speed) # store results - filename = f'exp1_results/exp1_speed_{datetime.now().strftime("%d-%m-%Y_%H-%M")}' + filename = Path(f'results/experiments/exp1/{datetime.now().strftime("%y%m%d")}_speed') + filename.parent.mkdir(parents=True, exist_ok=True) exp1_speed.round(3) print(exp1_speed) - exp1_speed.to_pickle(filename + '.pkl') - exp1_speed.to_csv(filename + '.csv') + exp1_speed.to_pickle(str(filename) + '.pkl') + exp1_speed.to_csv(str(filename) + '.csv') if __name__ == "__main__": - run_exp1_speed() + run_exp1_speed(['yolov5l6'], ['fp16', 'fp32']) diff --git a/exp2_res_speed.py b/exp2_res_speed.py index b2d04c718874..471b22170639 100644 --- a/exp2_res_speed.py +++ b/exp2_res_speed.py @@ -3,15 +3,11 @@ import pandas as pd from pathlib import Path -MODELS = ["yolov5n", "yolov5s", "yolov5m", "yolov5l"] - -MODELS_P6 = ["yolov5n6", "yolov5s6", "yolov5m6", "yolov5l6"] +MODELS = ["yolov5n", "yolov5s", "yolov5m", "yolov5l", "yolov5n6", "yolov5s6", "yolov5m6", "yolov5l6"] PRECISION = ['fp16'] -IMAGE_SIZES = [320, 480, 600, 800, 960] - -IMAGE_SIZES_P6 = [256, 448, 640, 832, 1024] +IMAGE_SIZES = [256, 384, 512, 640, 768, 896, 1024] def run_exp2_res_speed(models, precisions, image_sizes): @@ -56,6 +52,5 @@ def run_exp2_res_speed(models, precisions, image_sizes): if __name__ == "__main__": - #run_exp2_res_speed(MODELS, PRECISION, IMAGE_SIZES) - run_exp2_res_speed(MODELS_P6, ['fp16'], [1280]) - #run_exp2_res_speed(['yolov5n'], ['fp16'], [320]) \ No newline at end of file + run_exp2_res_speed(MODELS, PRECISION, IMAGE_SIZES) + diff --git a/export_openvino_models.py b/export_openvino_models.py index 961daa239c13..a08dd0edc0f1 100644 --- a/export_openvino_models.py +++ b/export_openvino_models.py @@ -6,12 +6,12 @@ PRECISION = ['fp16'] - # try different images sizes, 640 not necessary as already pretrained on 640 IMAGE_SIZES = [320, 480, 640, 800, 960] IMAGE_SIZES_P6 = [256, 448, 640, 832, 1024] # image size for P6 models (multiple of stride 64) +IMAGE_SIZES_EXTRA = [256, 384, 512, 640, 768, 896, 1024] def export_models(models, precisions, image_sizes): @@ -25,6 +25,8 @@ def export_models(models, precisions, image_sizes): if __name__ == "__main__": - export_models(MODELS, PRECISION, IMAGE_SIZES) - #export_models(MODELS, ['fp16'], IMAGE_SIZES) - + # export_models(MODELS, PRECISION, IMAGE_SIZES) + # export_models(MODELS, ['fp16'], IMAGE_SIZES_P6) + # export_models(MODELS_P6, ['fp32'], [1280]) + export_models(MODELS, ['fp16'], IMAGE_SIZES_EXTRA) + export_models(MODELS_P6, ['fp16'], IMAGE_SIZES_EXTRA) diff --git a/quantize_default.py b/quantize_default.py index 4034ac6b632c..8e93994e3ef5 100644 --- a/quantize_default.py +++ b/quantize_default.py @@ -162,5 +162,5 @@ def __getitem__(self, index): if __name__ == "__main__": #export_models() - quantize_models(MODELS) + #quantize_models(MODELS) quantize_models(MODELS_P6[:3]) diff --git a/requirements.txt b/requirements.txt index 89926e97d1a8..87f756796a2d 100755 --- a/requirements.txt +++ b/requirements.txt @@ -19,6 +19,7 @@ wandb~=0.12.11 # Plotting ------------------------------------ pandas>=1.1.4 seaborn>=0.11.0 +openpyxl # Export -------------------------------------- # coremltools>=4.1 # CoreML export diff --git a/results/experiments/exp1/20220604_exp1_perf.csv b/results/experiments/exp1/20220604_exp1_perf.csv new file mode 100644 index 000000000000..2efe69b86b83 --- /dev/null +++ b/results/experiments/exp1/20220604_exp1_perf.csv @@ -0,0 +1,22 @@ +,model,img_size,precision,mAP50,mAP,mAP50_8class,mAP_8class,person_mAP.5,car_mAP.5,motorcycle_mAP.5,bus_mAP.5,truck_mAP.5,baseball_bat_mAP.5,knife_mAP.5,cell_phone_mAP.5,person_mAP,car_mAP,motorcycle_mAP,bus_mAP,truck_mAP,baseball_bat_mAP,knife_mAP,cell_phone_mAP,experiment_time +0,yolov5n,640,int8,0.4314231790001692,0.24787113508957465,0.44083360141276473,0.24436714815165936,0.6793708211297177,0.5025362184275795,0.5801139747564206,0.6622326101099674,0.38428856646459864,0.30931386764379176,0.09652270122416587,0.3122900515458762,0.37804723103460447,0.267131242243343,0.2584006148699476,0.48249025635231596,0.2103767797416003,0.125431161702402,0.04716161937280068,0.18589827989626084,523 +1,yolov5n,640,fp16,0.45486181335672693,0.2707529622038651,0.4596909375405537,0.258513076585943,0.6999077396528962,0.5132597515121207,0.5951781886862397,0.6797055507746579,0.3984619269841213,0.35216091153233664,0.11438555494193574,0.32446787624012124,0.40449227277285155,0.283648180592865,0.27465063275597384,0.503928440712625,0.2211901694381948,0.14229414622091546,0.048771945793746066,0.18912882440037262,382 +2,yolov5n,640,fp32,0.45526599098321785,0.2707861004490829,0.4590413789650507,0.25854134177827903,0.7005021872894625,0.5136383012550454,0.5960522143627862,0.6798647771212062,0.39790708764594973,0.3442623494173893,0.11542367870857899,0.32468043591998763,0.40491218729647177,0.28356069354804236,0.2763462752151591,0.5039140333929752,0.2214126578300705,0.14179479429319997,0.04863925333243175,0.18775083931788183,434 +3,yolov5s,640,int8,0.5541858113039659,0.3491968312148317,0.5619838666372421,0.3384407866802551,0.7544593357223895,0.6128128522017561,0.6653445623113713,0.7691352127986419,0.5099490323837728,0.4967093082834019,0.23623147898938643,0.45122915040721645,0.47135840765521786,0.3556965358396609,0.36556303793713985,0.6049268769209662,0.3030631139878573,0.2226917897099494,0.12446552778654422,0.2597610036047051,847 +4,yolov5s,640,fp16,0.5645432682266455,0.363698641628669,0.571108152578838,0.35006803085230453,0.7628935427006011,0.6244165019414359,0.6769278079773049,0.7737022414787458,0.5176135846375015,0.5020525655007146,0.24049691218275165,0.4707620642116479,0.4873924527186255,0.3710039083401534,0.3699427607350578,0.6100052737194045,0.31635534604492876,0.22932849919310022,0.12922538373260822,0.2872906223345577,464 +5,yolov5s,640,fp32,0.5647636469369322,0.36419639452296904,0.5713870224710926,0.35092798041782014,0.7629392090822446,0.6249599551700238,0.6763895815320802,0.773758758460585,0.5177216421774546,0.5008692991300256,0.24487603283495393,0.469581701381373,0.4879464249047042,0.37144110133946323,0.37034914621998427,0.6113645563933637,0.3170884478537019,0.22948505487672838,0.1309029449620392,0.28884616679257646,720 +6,yolov5m,640,int8,0.6293221532487827,0.4277831505557422,0.656779847553475,0.42734865051789517,0.8031778308443034,0.6914636778729424,0.7450582165613004,0.8328889286280176,0.5897514226957639,0.6451725761464735,0.3558064158968575,0.5909197117821408,0.5412025348117713,0.4328577856239274,0.45028612889302577,0.6793854527314613,0.38838017189785273,0.35173092460474026,0.20393868055470316,0.37100752502567935,1660 +7,yolov5m,640,fp16,0.6357043635213712,0.4403215036123669,0.6606575539697976,0.4386370796239435,0.8070052528864594,0.7010515378498446,0.746281643485571,0.836767791222733,0.5906150206094708,0.6400036959774322,0.35415751461152195,0.6093779751153484,0.5553559604767373,0.45155058738045745,0.45918389983372687,0.6965452359720511,0.3978792251818622,0.3585475152129492,0.20596311959198826,0.38407109334177575,866 +8,yolov5m,640,fp32,0.6354700390642277,0.4403623722508371,0.6599687978956588,0.438284515469473,0.8068219356531682,0.7020519026233364,0.7476705376455017,0.8367546854503104,0.5908705394641256,0.635539349176066,0.35151732004888814,0.6085241131038732,0.5557391757571921,0.4501588719092072,0.4600929057191691,0.6974698799933015,0.3987241025499283,0.3555492381923115,0.20409802484839518,0.38444392478627903,1385 +9,yolov5l,640,int8,0.6600940308352422,0.460373750489769,0.6837101912133877,0.4565224980685859,0.8176839228399039,0.7225613686667797,0.7621413513557338,0.8557296370719534,0.6329282079056064,0.6853555718737215,0.3703364580792363,0.6229450119141673,0.5683499171613474,0.4630990521700344,0.469617491626788,0.7020913385136124,0.4249759459214614,0.38865691253797846,0.22203341699101395,0.4133559096264513,3146 +10,yolov5l,640,fp16,0.6665367902092766,0.47789390703235773,0.6898390735821485,0.471667935347832,0.8256948289734023,0.734441493189033,0.7524269864099606,0.8547283670111554,0.6382927548140075,0.6965918672006282,0.3908614667326936,0.6256748243263068,0.5871755369835433,0.4865698415078093,0.4874399054578185,0.7203807755746207,0.43882361352608157,0.39985244228202566,0.23275881886365118,0.4203425485871056,1427 +11,yolov5l,640,fp32,0.6664181307991731,0.478635600145122,0.6894984725918721,0.4729973059433904,0.825632442010252,0.7342926030528893,0.7532994349426476,0.8546580553675527,0.6389686157885038,0.6930848537021895,0.3894079320993114,0.6266438437716303,0.5874312312577594,0.4873386916087468,0.48766992330425296,0.7220311523463784,0.4401978352941912,0.40307775552807756,0.23402032238909332,0.4222115358186234,2994 +12,yolov5n6,1280,int8,0.450798943824819,0.2770467000897089,0.4875456534368053,0.2912662087400323,0.7288970365281424,0.5884788098528929,0.5643143161209007,0.7363965927395504,0.42623711312671364,0.3357948120825753,0.14827768964262647,0.37196885740104074,0.43747732906604453,0.33724843206111227,0.28654061028921807,0.5672939692324647,0.2608422691245019,0.15100642496558098,0.06776351552505203,0.2219571196562835,1816 +13,yolov5n6,1280,fp16,0.5400530473306253,0.3469462387511327,0.5514125951379049,0.34019907702465524,0.7763624221820323,0.6380425507016312,0.6590395930461523,0.7814670854939122,0.4881668181946613,0.4187403233566344,0.20972858024700305,0.43975338788121177,0.4970080358971506,0.3821795908120972,0.3611094050060574,0.6196541515662732,0.3048615853513525,0.19057632968957577,0.10214551616642775,0.26405800170830734,1089 +14,yolov5n6,1280,fp32,0.5401285615505159,0.34712347893384893,0.551489456509668,0.33977481401231957,0.7772950537191546,0.638019405863828,0.6614776817521646,0.7803467966864597,0.4883418140858524,0.4177848278420754,0.20751914058201532,0.44113093154579314,0.4973847520838521,0.3835887672185281,0.36126521829005,0.6193831315322063,0.3045619715341881,0.18731086087833335,0.10116788135532837,0.2635359292060703,1552 +15,yolov5s6,1280,int8,0.6241496303074341,0.4168397958075743,0.6482978654248087,0.41286016076443205,0.8151412127848457,0.705270614141835,0.7387345498912761,0.8285584170859971,0.582006099761898,0.5767382679481105,0.34887716565406,0.5910565961304478,0.5393327613721826,0.44257204474222667,0.43949791148462836,0.6636408149889623,0.38535837272201273,0.27607625276973147,0.19075037231407196,0.36565275572164035,3039 +16,yolov5s6,1280,fp16,0.6323960790510746,0.43420618394367827,0.6569917557402419,0.43145563869909653,0.8206833758039995,0.7150693358004645,0.7509916373851308,0.8316057209096914,0.5906168500273201,0.590837307604964,0.3493616384371493,0.6067681799532161,0.5617352470988275,0.4585890019336457,0.4551180939164049,0.684514140132925,0.39961043385793743,0.3118185919925624,0.1979364150804021,0.38232318558006734,1688 +17,yolov5s6,1280,fp32,0.6324991306171344,0.4343550349293713,0.6559187180400157,0.431612205894493,0.8207687384602991,0.7156447544793143,0.7464121036780811,0.8311842013990133,0.5889221352254326,0.5891904418532479,0.3471871933408957,0.6080401758838414,0.5619703383136276,0.4596882461060957,0.45418402532982294,0.684539292879407,0.3982810845469938,0.3126999568126403,0.19797830930549903,0.3835563938618577,3433 +18,yolov5m6,1280,int8,0.681199208802166,0.4847140745445867,0.7072789950451037,0.4866692722266781,0.8460871279011278,0.7609395816477315,0.7842403532289051,0.8697336268315163,0.62569036919707,0.6624618196379387,0.45952782051702884,0.6495512613995114,0.6016885968502462,0.5069393586191412,0.5069939844097241,0.7311927913400679,0.4447905070025806,0.3773621733829418,0.28398552081682155,0.44040124539190106,5771 +19,yolov5m6,1280,fp16,0.6867168349816257,0.4982664657841841,0.7166502575241687,0.5028562581020642,0.8507114800281481,0.7671696460270029,0.7882269089981908,0.8737055168155524,0.632929956475395,0.685159078872597,0.4693268566686746,0.665972616307788,0.6178298439488159,0.5244355658433183,0.5231347283069913,0.7452594698461851,0.45233258625216183,0.3947943432556957,0.2991228275040415,0.46594069985930436,3292 +20,yolov5m6,1280,fp32,0.6869069795931797,0.49844172187683017,0.7166566241619927,0.5033602299961913,0.8506847897246519,0.7673610063135161,0.7914834550321095,0.874124604199124,0.633171570390417,0.6812626618880426,0.46864988839592614,0.6665150173521548,0.6183516375056428,0.5245307549251312,0.5236923780360254,0.7474170851700175,0.451999298853148,0.3967630982952229,0.2987306430734117,0.46539694411093147,5781 diff --git a/results/experiments/exp1/20220604_exp1_perf.pkl b/results/experiments/exp1/20220604_exp1_perf.pkl new file mode 100644 index 000000000000..128443e7f054 Binary files /dev/null and b/results/experiments/exp1/20220604_exp1_perf.pkl differ diff --git a/results/experiments/exp1/20220609_exp1_perf.csv b/results/experiments/exp1/20220609_exp1_perf.csv new file mode 100644 index 000000000000..bb2702556663 --- /dev/null +++ b/results/experiments/exp1/20220609_exp1_perf.csv @@ -0,0 +1,3 @@ +,model,img_size,precision,mAP50,mAP,mAP50_8class,mAP_8class,person_mAP.5,car_mAP.5,motorcycle_mAP.5,bus_mAP.5,truck_mAP.5,baseball_bat_mAP.5,knife_mAP.5,cell_phone_mAP.5,person_mAP,car_mAP,motorcycle_mAP,bus_mAP,truck_mAP,baseball_bat_mAP,knife_mAP,cell_phone_mAP,experiment_time +0,yolov5l6,1280,fp16,0.7061042757231983,0.5227280554509581,0.7434818681153712,0.5324955592698563,0.8598390698011174,0.787859037238027,0.8079996242721972,0.8850437886752031,0.6673509598451254,0.7169808653401972,0.521954379672731,0.7008272200783721,0.6350552259668647,0.5494377454819125,0.5568775414703249,0.7704109662868025,0.4866806525207699,0.4243228408447344,0.34254745134870773,0.49463205023873397,6032 +1,yolov5l6,1280,fp32,0.7059118815833638,0.5228428252450639,0.7431273980514019,0.5321621208044109,0.859570592146744,0.7882380991650857,0.8070124688242202,0.8860877959175956,0.6669472257287786,0.7179466397444454,0.5192162308709506,0.7000001320133953,0.6359903312155086,0.5497617085113294,0.5565396262999778,0.7719261950705294,0.48700068198104296,0.4183031495369353,0.34314265535611216,0.4946326184638516,11163 diff --git a/results/experiments/exp1/20220609_exp1_perf.pkl b/results/experiments/exp1/20220609_exp1_perf.pkl new file mode 100644 index 000000000000..9ff815254f80 Binary files /dev/null and b/results/experiments/exp1/20220609_exp1_perf.pkl differ diff --git a/results/experiments/exp1/220530_exp1_1.xlsx b/results/experiments/exp1/220530_exp1_1.xlsx new file mode 100644 index 000000000000..a44fb1ea08e3 Binary files /dev/null and b/results/experiments/exp1/220530_exp1_1.xlsx differ diff --git a/results/experiments/exp1/220605_speed.csv b/results/experiments/exp1/220605_speed.csv new file mode 100644 index 000000000000..45c73ac14500 --- /dev/null +++ b/results/experiments/exp1/220605_speed.csv @@ -0,0 +1,22 @@ +,model,precision,prep_time,NMS_time,latency,inference_time,total_time,FPS,experiment_time +0,yolov5n,int8,2.337548112869263,1.768181276321411,4.105729389190674,68.88695788383484,72.99268727302551,13.700002525726307,410 +1,yolov5n,fp16,2.374600076675415,1.430059242248535,3.80465931892395,34.67108125686646,38.475740575790404,25.990402914537196,253 +2,yolov5n,fp32,2.3621745586395266,2.0693668365478515,4.431541395187378,45.092685079574586,49.52422647476196,20.19213769062319,311 +3,yolov5s,int8,2.309380531311035,2.0558454513549806,4.365225982666016,146.81233563423157,151.17756161689758,6.61473825417374,800 +4,yolov5s,fp16,2.549609851837158,2.1765974998474125,4.72620735168457,60.3297658443451,65.05597319602967,15.371378689344233,392 +5,yolov5s,fp32,2.756035280227661,2.809509468078613,5.565544748306274,104.74162607192994,110.30717082023622,9.065593764793999,639 +6,yolov5m,int8,2.3247382640838623,1.6931139469146728,4.0178522109985355,309.2857801437378,313.3036323547363,3.1917919127977283,1610 +7,yolov5m,fp16,3.0497063159942623,2.878777551651001,5.928483867645263,127.29497909545897,133.22346296310425,7.506185305188669,763 +8,yolov5m,fp32,2.906248760223389,2.848623561859131,5.7548723220825195,254.92730817794802,260.68218050003054,3.836088826945664,1397 +9,yolov5l,int8,2.1997583866119386,1.755626630783081,3.9553850173950194,552.891881942749,556.8472669601441,1.7958245632757575,2829 +10,yolov5l,fp16,2.709670877456665,2.6728721141815184,5.382542991638184,247.74582324028017,253.12836623191836,3.950564746598931,1371 +11,yolov5l,fp32,2.758587741851807,2.929982089996338,5.688569831848145,524.281589460373,529.9701592922211,1.8868986913065202,2749 +12,yolov5n6,int8,9.233469152450562,4.684622764587402,13.918091917037964,315.48181357383726,329.3999054908752,3.035823578970946,1727 +13,yolov5n6,fp16,9.615099811553954,5.032219171524048,14.647318983078002,153.3368145942688,167.9841335773468,5.95294316614467,956 +14,yolov5n6,fp32,9.389935970306396,5.156487560272217,14.546423530578615,268.75860691070557,283.3050304412842,3.5297643619048023,1531 +15,yolov5s6,int8,8.927440929412843,5.030945777893066,13.958386707305909,566.1847447395326,580.1431314468384,1.723712556082611,2987 +16,yolov5s6,fp16,9.705197525024413,5.759062337875366,15.46425986289978,283.1850588798523,298.6493187427521,3.3484087765871355,1628 +17,yolov5s6,fp32,9.564798974990845,5.033862733840942,14.598661708831788,679.2221281528473,693.820789861679,1.4412943725704173,3592 +18,yolov5m6,int8,8.399726963043213,4.753249883651733,13.152976846694946,1191.6705033302305,1204.8234801769254,0.8299971045162171,6101 +19,yolov5m6,fp16,10.250411748886108,7.218113374710083,17.46852512359619,595.1217665672302,612.5902916908265,1.632412419138857,3218 +20,yolov5m6,fp32,10.374418830871582,7.458432197570801,17.832851028442384,1087.1622262477877,1104.99507727623,0.9049814072158233,5701 diff --git a/results/experiments/exp1/220605_speed.pkl b/results/experiments/exp1/220605_speed.pkl new file mode 100644 index 000000000000..aa76427f7bc5 Binary files /dev/null and b/results/experiments/exp1/220605_speed.pkl differ diff --git a/results/experiments/exp1/220609_speed.csv b/results/experiments/exp1/220609_speed.csv new file mode 100644 index 000000000000..dfd328dcded1 --- /dev/null +++ b/results/experiments/exp1/220609_speed.csv @@ -0,0 +1,3 @@ +,model,precision,prep_time,NMS_time,latency,inference_time,total_time,FPS,experiment_time +0,yolov5l6,fp16,10.344834518432616,6.876286220550536,17.221120738983153,1136.2903309345245,1153.5114516735077,0.8669181381330942,5939 +1,yolov5l6,fp32,10.763452577590941,7.065650749206543,17.829103326797483,2182.0223227500915,2199.8514260768893,0.45457615371023147,11185 diff --git a/results/experiments/exp1/220609_speed.pkl b/results/experiments/exp1/220609_speed.pkl new file mode 100644 index 000000000000..5923d49c9c04 Binary files /dev/null and b/results/experiments/exp1/220609_speed.pkl differ diff --git a/results/experiments/exp2/220511_res_speed.csv b/results/experiments/exp2/220511_res_speed.csv deleted file mode 100644 index 4d3dfd8c5807..000000000000 --- a/results/experiments/exp2/220511_res_speed.csv +++ /dev/null @@ -1,21 +0,0 @@ -,model,precision,resolution,prep_time,NMS_time,latency,inference_time,total_time,FPS,experiment_time -0,yolov5n6,fp16,fp16,0.30963797569274903,0.8105006217956543,1.1201385974884033,10.254688215255737,11.37482681274414,87.91342641627027,109 -1,yolov5n6,fp16,fp16,1.4461627960205077,1.2085473537445068,2.6547101497650143,19.784381580352782,22.4390917301178,44.56508365076996,167 -2,yolov5n6,fp16,fp16,2.312229061126709,1.7049466133117677,4.017175674438477,35.13287453651428,39.150050210952756,25.542751404191982,258 -3,yolov5n6,fp16,fp16,3.3494174480438232,2.349052095413208,5.698469543457032,69.2811872959137,74.97965683937073,13.336950876453123,452 -4,yolov5n6,fp16,fp16,5.800476980209351,3.5913053512573243,9.391782331466676,114.94408721923828,124.33586955070496,8.0427313824527,707 -5,yolov5s6,fp16,fp16,0.28116641044616697,0.9249234199523926,1.2060898303985597,14.287695598602296,15.493785429000855,64.54200650850802,129 -6,yolov5s6,fp16,fp16,1.3308261394500733,1.1025200366973875,2.433346176147461,31.2643168926239,33.697663068771355,29.675648366450975,229 -7,yolov5s6,fp16,fp16,2.6464440345764157,2.16440806388855,4.810852098464966,61.95250544548035,66.7633575439453,14.978276060214244,403 -8,yolov5s6,fp16,fp16,4.202495527267456,3.002540969848633,7.205036497116089,112.20104517936706,119.40608167648315,8.374782808043088,689 -9,yolov5s6,fp16,fp16,6.283166599273682,4.588887357711792,10.872053956985475,169.53574104309084,180.4077950000763,5.542997740200622,1009 -10,yolov5m6,fp16,fp16,0.26869707107543944,0.9650455951690673,1.2337426662445068,25.832505989074708,27.066248655319214,36.94638339929217,196 -11,yolov5m6,fp16,fp16,1.6467161655426026,1.5306586742401125,3.177374839782715,63.002799177169805,66.18017401695252,15.110265496489069,415 -12,yolov5m6,fp16,fp16,3.0558205127716063,3.3823890686035156,6.438209581375122,133.49495692253114,139.93316650390628,7.146268643696308,802 -13,yolov5m6,fp16,fp16,4.452916622161865,3.982505464553833,8.435422086715699,238.22406587600707,246.65948796272278,4.054172042030381,1358 -14,yolov5m6,fp16,fp16,6.714513063430786,4.864035272598267,11.578548336029053,361.00072875022886,372.5792770862579,2.683992539307237,2000 -15,yolov5l6,fp16,fp16,0.25929031372070316,1.0871392726898192,1.3464295864105225,46.90703558921814,48.25346517562866,20.72390027038036,320 -16,yolov5l6,fp16,fp16,1.7951343059539795,2.0226609230041506,3.81779522895813,124.47034282684326,128.2881380558014,7.794952948533954,743 -17,yolov5l6,fp16,fp16,3.4624338150024414,2.8446065902709963,6.307040405273438,259.83829374313353,266.14533414840696,3.757345599161937,1442 -18,yolov5l6,fp16,fp16,4.879588937759399,3.487671518325806,8.367260456085205,548.2367875099183,556.6040479660035,1.7966092838424317,2906 -19,yolov5l6,fp16,fp16,6.757919263839722,5.00304455757141,11.760963821411131,696.5305470943451,708.2915109157564,1.4118480662108899,3690 diff --git a/results/experiments/exp2/220511_res_speed.pkl b/results/experiments/exp2/220511_res_speed.pkl deleted file mode 100644 index aea9f5dc118b..000000000000 Binary files a/results/experiments/exp2/220511_res_speed.pkl and /dev/null differ diff --git a/results/experiments/exp2/220512_res_speed.csv b/results/experiments/exp2/220512_res_speed.csv deleted file mode 100644 index ced8340c8d52..000000000000 --- a/results/experiments/exp2/220512_res_speed.csv +++ /dev/null @@ -1,21 +0,0 @@ -,model,precision,resolution,prep_time,NMS_time,latency,inference_time,total_time,FPS,experiment_time -0,yolov5n6,fp16,256,0.24371924400329592,0.8404638767242432,1.084183120727539,10.141628408432007,11.225811529159547,89.08041947813352,110 -1,yolov5n6,fp16,448,1.0932781219482421,1.018265390396118,2.11154351234436,19.167963075637818,21.279506587982176,46.99357082672022,167 -2,yolov5n6,fp16,640,2.243425130844116,1.9454683780670166,4.188893508911133,35.8370707988739,40.02596430778504,24.983782834321378,264 -3,yolov5n6,fp16,832,3.5901308059692383,2.507836389541626,6.097967195510864,68.1953230857849,74.29329028129577,13.460165732513829,450 -4,yolov5n6,fp16,1024,5.580936670303345,4.029936170578003,9.610872840881347,105.97678842544556,115.5876612663269,8.651442455400911,666 -5,yolov5s6,fp16,256,0.29055061340332033,0.9028595924377442,1.1934102058410645,14.572713375091553,15.766123580932618,63.42713190510503,131 -6,yolov5s6,fp16,448,1.2744545936584473,1.3087448120117189,2.5831994056701664,31.54699683189392,34.130196237564085,29.29956783838789,231 -7,yolov5s6,fp16,640,2.462086009979248,2.1359723567962647,4.598058366775513,62.19229373931884,66.79035210609435,14.972222311562781,405 -8,yolov5s6,fp16,832,4.184008979797364,3.0083611965179444,7.192370176315308,111.86833968162536,119.06070985794067,8.399076413983817,688 -9,yolov5s6,fp16,1024,5.708690214157104,4.804820728302001,10.513510942459106,169.59895315170286,180.11246409416196,5.552086608937873,1008 -10,yolov5m6,fp16,256,0.3092641830444336,1.0183655738830566,1.3276297569274902,26.237813854217528,27.565443611145017,36.27730480621356,199 -11,yolov5m6,fp16,448,1.6060508728027343,1.6209608078002928,3.2270116806030273,63.78585457801819,67.01286625862122,14.922507509837333,418 -12,yolov5m6,fp16,640,3.0997354984283447,3.3723194122314455,6.47205491065979,134.69117631912232,141.1632312297821,7.083997662055668,810 -13,yolov5m6,fp16,832,4.7852356910705565,3.5309593200683596,8.316195011138916,241.8856906890869,250.20188570022583,3.996772435193111,1378 -14,yolov5m6,fp16,1024,6.6518336772918705,4.947465944290162,11.599299621582032,360.04673557281495,371.646035194397,2.6907323240430365,1996 -15,yolov5l6,fp16,256,0.2904427528381348,0.9902376651763916,1.2806804180145264,47.17461166381836,48.45529208183289,20.63758068594793,322 -16,yolov5l6,fp16,448,1.7154144287109376,2.03649787902832,3.751912307739258,123.78862175941467,127.54053406715393,7.840644602236572,739 -17,yolov5l6,fp16,640,3.603149127960205,3.1653121948242187,6.7684613227844235,258.8982938766479,265.6667551994323,3.764114178491449,1439 -18,yolov5l6,fp16,832,4.399629974365235,2.91133828163147,7.310968255996705,538.379247713089,545.6902159690857,1.8325415606437256,2850 -19,yolov5l6,fp16,1024,6.6425714015960695,4.650054550170898,11.292625951766968,693.9196030139924,705.2122289657594,1.4180128462414305,3673 diff --git a/results/experiments/exp2/220512_res_speed.pkl b/results/experiments/exp2/220512_res_speed.pkl deleted file mode 100644 index 31be09f17c9a..000000000000 Binary files a/results/experiments/exp2/220512_res_speed.pkl and /dev/null differ diff --git a/results/experiments/exp2/220530_results_res-fps.xlsx b/results/experiments/exp2/220530_results_res-fps.xlsx new file mode 100644 index 000000000000..4cd18ce4551c Binary files /dev/null and b/results/experiments/exp2/220530_results_res-fps.xlsx differ diff --git a/results/experiments/exp2/220607_res_speed.csv b/results/experiments/exp2/220607_res_speed.csv new file mode 100644 index 000000000000..d09b27da594b --- /dev/null +++ b/results/experiments/exp2/220607_res_speed.csv @@ -0,0 +1,57 @@ +,model,precision,resolution,prep_time,NMS_time,latency,inference_time,total_time,FPS,experiment_time +0,yolov5n,fp16,256,0.2874142169952393,0.8247872829437256,1.1122014999389649,9.366648864746095,10.47885036468506,95.43031584553549,110 +1,yolov5n,fp16,384,1.0620476245880126,1.0748423576354982,2.1368899822235106,15.647989463806152,17.78487944602966,56.22753885032616,141 +2,yolov5n,fp16,512,1.712149429321289,1.6394833087921141,3.3516327381134032,23.452619361877442,26.804252099990848,37.30751360901957,189 +3,yolov5n,fp16,640,2.3433992385864255,1.6207120418548584,3.964111280441284,34.883061742782594,38.84717302322388,25.741898886752278,256 +4,yolov5n,fp16,768,3.018115615844726,2.1696663379669188,5.187781953811645,52.715145874023435,57.90292782783508,17.27028386152315,359 +5,yolov5n,fp16,896,4.483811521530152,2.958504581451416,7.442316102981568,71.14632511138916,78.58864121437074,12.724485174291827,468 +6,yolov5n,fp16,1024,6.03074631690979,3.427300310134888,9.458046627044677,107.71961088180541,117.17765750885007,8.534050101867527,668 +7,yolov5s,fp16,256,0.2968314647674561,1.0435479164123536,1.3403793811798097,12.653345918655395,13.993725299835205,71.4605995597024,121 +8,yolov5s,fp16,384,0.9246787548065186,1.199285125732422,2.1239638805389407,22.460319566726685,24.584283447265626,40.676394011850874,179 +9,yolov5s,fp16,512,1.7559356212615969,1.617551612854004,3.3734872341156006,38.108002519607545,41.481489753723146,24.107138049694697,266 +10,yolov5s,fp16,640,2.6431338787078857,2.338999700546265,4.982133579254151,60.18285059928894,65.1649841785431,15.345664740133099,393 +11,yolov5s,fp16,768,3.5305919647216797,2.8173383235931397,6.347930288314819,88.49074501991272,94.83867530822754,10.544221508261062,558 +12,yolov5s,fp16,896,4.821472930908203,3.673910522460938,8.49538345336914,127.65168604850768,136.14706950187684,7.344998343766882,775 +13,yolov5s,fp16,1024,6.046475839614868,4.476853466033936,10.523329305648804,169.1071916103363,179.63052091598507,5.566982687021821,1004 +14,yolov5m,fp16,256,0.2686610698699951,1.0932804584503173,1.3619415283203125,22.687984037399293,24.049925565719604,41.580170269856666,181 +15,yolov5m,fp16,384,1.1966259956359864,1.3866671085357667,2.5832931041717533,46.26801695823669,48.851310062408444,20.470280095303107,316 +16,yolov5m,fp16,512,2.218581533432007,2.4894436836242675,4.708025217056274,78.31252193450928,83.02054715156555,12.045210906335765,502 +17,yolov5m,fp16,640,2.9838796615600587,3.010322618484497,5.994202280044556,128.20988903045654,134.2040913105011,7.451337662175674,771 +18,yolov5m,fp16,768,3.768339538574219,4.373612403869629,8.141951942443848,188.35892543792724,196.5008773803711,5.089035801424326,1104 +19,yolov5m,fp16,896,5.049262285232544,4.450817060470581,9.500079345703124,264.67071704864503,274.17079639434814,3.6473614737642217,1493 +20,yolov5m,fp16,1024,6.904968357086182,4.978753232955933,11.883721590042114,358.25344400405885,370.13716559410096,2.7017011339429176,1994 +21,yolov5l,fp16,256,0.4061430931091308,1.0526859760284424,1.4588290691375732,40.814834117889404,42.273663187026976,23.65539025032687,285 +22,yolov5l,fp16,384,1.5247634410858153,1.9523205280303955,3.4770839691162108,86.07312726974487,89.55021123886108,11.166919498745319,542 +23,yolov5l,fp16,512,2.0531357765197753,2.142653751373291,4.195789527893066,155.768688249588,159.96447777748108,6.2513878949491035,909 +24,yolov5l,fp16,640,2.918727159500122,2.7774035930633545,5.696130752563477,247.78869500160218,253.48482575416566,3.9450093196892926,1373 +25,yolov5l,fp16,768,4.124243831634521,2.829124593734741,6.953368425369263,367.6669966697693,374.62036509513854,2.6693690284190503,1997 +26,yolov5l,fp16,896,4.4959934711456295,3.512740135192871,8.0087336063385,509.0635277748108,517.0722613811492,1.9339656653190112,2725 +27,yolov5l,fp16,1024,6.339571619033813,5.512308740615844,11.851880359649657,691.0329309463501,702.8848113059997,1.4227082217667264,3650 +28,yolov5n6,fp16,256,0.2811727523803711,0.8779641628265381,1.159136915206909,9.891167926788329,11.050304841995239,90.4952410181148,107 +29,yolov5n6,fp16,384,1.0370928764343263,0.8592153072357177,1.896308183670044,16.08849573135376,17.984803915023804,55.60249668135881,145 +30,yolov5n6,fp16,512,1.543254518508911,1.652123785018921,3.195378303527832,24.30231246948242,27.49769077301025,36.366690143360245,195 +31,yolov5n6,fp16,640,2.243247652053833,1.8142452239990234,4.057492876052857,36.95196237564087,41.00945525169372,24.384620421377072,267 +32,yolov5n6,fp16,768,3.289968061447144,2.2763389110565186,5.5663069725036625,57.37600269317627,62.94230966567993,15.887564427036944,390 +33,yolov5n6,fp16,896,4.380832481384277,2.964733409881592,7.345565891265869,74.5955671787262,81.94113306999206,12.203882989338519,488 +34,yolov5n6,fp16,1024,5.934005546569824,3.639643335342407,9.57364888191223,115.2300078868866,124.80365676879883,8.01258573578913,708 +35,yolov5s6,fp16,256,0.2311494827270508,0.7686600685119629,0.9998095512390137,14.93731107711792,15.937120628356935,62.74659164094541,133 +36,yolov5s6,fp16,384,0.8340030193328857,1.3399819374084472,2.1739849567413327,24.66955409049988,26.84353904724121,37.25291207840097,195 +37,yolov5s6,fp16,512,1.6184547424316404,1.567635679244995,3.1860904216766355,39.192254590988156,42.37834501266479,23.596957354071982,277 +38,yolov5s6,fp16,640,2.4246670246124267,2.3047451496124265,4.729412174224853,61.47041630744934,66.19982848167419,15.105779319002702,404 +39,yolov5s6,fp16,768,3.5118714332580567,3.0814196586608884,6.593291091918945,89.91971220970153,96.51300330162047,10.361298123474826,573 +40,yolov5s6,fp16,896,4.602912712097169,3.458353090286255,8.061265802383424,132.13516368865965,140.1964294910431,7.1328492717704215,796 +41,yolov5s6,fp16,1024,5.959108686447143,4.620459365844726,10.579568052291869,170.00369777679444,180.58326582908632,5.537611668549917,1009 +42,yolov5m6,fp16,256,0.2842877388000489,1.0838107585906982,1.368098497390747,26.163379049301145,27.531477546691892,36.322060750428456,200 +43,yolov5m6,fp16,384,1.112209415435791,1.2744161128997802,2.3866255283355713,48.117578124999994,50.504203653335566,19.80033200531328,329 +44,yolov5m6,fp16,512,2.2310540676116943,2.1550845623016355,4.386138629913329,83.19363651275636,87.57977514266969,11.418161309171833,527 +45,yolov5m6,fp16,640,2.97467303276062,3.43458399772644,6.40925703048706,140.33581190109254,146.7450689315796,6.8145390320832755,832 +46,yolov5m6,fp16,768,4.086995983123779,3.805197525024414,7.892193508148193,204.94035229682922,212.83254580497743,4.698529523376182,1187 +47,yolov5m6,fp16,896,5.414488887786865,3.885143995285034,9.299632883071899,273.48132133483887,282.7809542179108,3.536306052738618,1546 +48,yolov5m6,fp16,1024,6.7466327190399165,4.628524446487427,11.375157165527344,373.45547666549686,384.83063383102416,2.598545729181974,2064 +49,yolov5l6,fp16,256,0.3123368740081787,0.9809344768524171,1.2932713508605957,47.230714559555054,48.523985910415654,20.60836473422004,323 +50,yolov5l6,fp16,384,1.3153974056243896,2.0336001873016354,3.348997592926025,90.96768040657044,94.31667799949646,10.602578687147345,573 +51,yolov5l6,fp16,512,2.1343494415283204,1.967482852935791,4.101832294464112,156.30720176696778,160.4090340614319,6.234062849708513,906 +52,yolov5l6,fp16,640,3.4279452323913575,3.4958573818206786,6.923802614212036,259.3668434143067,266.2906460285187,3.7552952569460656,1442 +53,yolov5l6,fp16,768,4.162003755569458,2.801045274734497,6.963049030303955,380.6446837902069,387.6077328205109,2.579927889269095,2063 +54,yolov5l6,fp16,896,4.802863550186157,3.8313127994537353,8.634176349639892,520.1301951408386,528.7643714904785,1.8912015519903596,2786 +55,yolov5l6,fp16,1024,6.673413610458375,4.9158134937286375,11.589227104187012,699.5085344791412,711.0977615833282,1.4062763997082524,3707 diff --git a/results/experiments/exp2/220607_res_speed.pkl b/results/experiments/exp2/220607_res_speed.pkl new file mode 100644 index 000000000000..2f7c8ce8d323 Binary files /dev/null and b/results/experiments/exp2/220607_res_speed.pkl differ diff --git a/results/experiments/exp2/220607_res_speed_excel.xlsx b/results/experiments/exp2/220607_res_speed_excel.xlsx new file mode 100644 index 000000000000..c3e85f08effb Binary files /dev/null and b/results/experiments/exp2/220607_res_speed_excel.xlsx differ diff --git a/visualize.py b/visualize.py index 7e131fe4aed3..2ce1acd5dd95 100644 --- a/visualize.py +++ b/visualize.py @@ -2,11 +2,25 @@ import matplotlib.pyplot as plt import seaborn as sns import pandas as pd +import os +from pathlib import Path + + +def read_file(path, fields=None): + if os.path.splitext(path)[-1] == '.csv': + df = pd.read_csv(path, usecols=fields) + elif os.path.splitext(path)[-1] == '.xls' or '.xlsx': + df = pd.read_excel(path, engine='openpyxl', usecols=fields) + else: + return print('Not a valid file') + return df # %% -def plot_speed_det(df_path): - df = pd.read_csv(df_path) +# plots different scatter plots +def col_plot_speed_det(path): + df = read_file(path) + g = sns.relplot( data=df, x='FPS', y='mAP50', col='device', hue='model', style='precision', kind='scatter' @@ -19,4 +33,33 @@ def plot_speed_det(df_path): # g.savefig("results/plots/relplot.png") plt.show() + +def plot_exp1_results(path): + fields = ['model', 'precision', 'FPS', 'mAP50'] + df = read_file(path, fields) + + pivot = df.pivot(index='model', columns='precision', values=['FPS', 'mAP50']) + + # calculate percentage of mAP and FPS of INT8 and FP 16 compared to baseline FP32 + pivot.loc[:, ('FPS', 'fp16')] = (pivot['FPS']['fp16'] / pivot['FPS']['fp32'][:]) * 100 + pivot.loc[:, ('FPS', 'int8')] = (pivot['FPS']['int8'] / pivot['FPS']['fp32'][:]) * 100 + pivot.loc[:, ('FPS', 'fp32')] = 100 + pivot.loc[:, ('mAP50', 'fp16')] = (pivot['mAP50']['fp16'] / pivot['mAP50']['fp32'][:]) * 100 + pivot.loc[:, ('mAP50', 'int8')] = (pivot['mAP50']['int8'] / pivot['mAP50']['fp32'][:]) * 100 + pivot.loc[:, ('mAP50', 'fp32')] = 100 + + df = pivot.stack().reset_index() + + sns.scatterplot( + data=df, x='mAP50', y='FPS', hue='model', style='precision' + ) + + # g.set_axis_labels("Inference speed in FPS", "Detection performance in mAP:@.5") + # g.set(xlim=(0, 60), ylim=(0, 12), xticks=[10, 30, 50], yticks=[2, 6, 10]) + # g.tight_layout() + # g.savefig("results/plots/relplot.png") + plt.show() + + # %% +plot_exp1_results("./results/experiments/exp1/220530_exp1_1.xlsx")