From c2a27a314012f087bb0147d19f4ab5369f0f104b Mon Sep 17 00:00:00 2001 From: Frank Liu Date: Tue, 5 Jul 2022 12:25:08 -0700 Subject: [PATCH] Use image from S3 bucket Change-Id: Iec3943455cb58dc0d8644841446cb3d91242bb20 --- .../SemanticSegmentationTranslator.java | 68 +++++++++--------- examples/docs/img/semantic_instance_bikes.png | Bin 3069 -> 0 bytes .../img/semantic_instance_dog_bike_car.png | Bin 5936 -> 0 bytes examples/docs/semantic_segmentation.md | 4 +- .../inference/SemanticSegmentation.java | 36 ++++------ 5 files changed, 49 insertions(+), 59 deletions(-) delete mode 100644 examples/docs/img/semantic_instance_bikes.png delete mode 100644 examples/docs/img/semantic_instance_dog_bike_car.png diff --git a/api/src/main/java/ai/djl/modality/cv/translator/SemanticSegmentationTranslator.java b/api/src/main/java/ai/djl/modality/cv/translator/SemanticSegmentationTranslator.java index 4ff0353cc19..5baaec52f0c 100644 --- a/api/src/main/java/ai/djl/modality/cv/translator/SemanticSegmentationTranslator.java +++ b/api/src/main/java/ai/djl/modality/cv/translator/SemanticSegmentationTranslator.java @@ -37,9 +37,6 @@ public class SemanticSegmentationTranslator extends BaseImageTranslator { private final int shortEdge; private final int maxEdge; - private int rescaledWidth; - private int rescaledHeight; - private static final int CHANNEL = 3; private static final int CLASSNUM = 21; private static final int BIKE = 2; @@ -79,43 +76,44 @@ public Image processOutput(TranslatorContext ctx, NDList list) { float[] scores = list.get(1).toFloatArray(); // get dimensions of image - final int width = (int) ctx.getAttachment("originalWidth"); - final int height = (int) ctx.getAttachment("originalHeight"); + int width = (int) ctx.getAttachment("originalWidth"); + int height = (int) ctx.getAttachment("originalHeight"); // build image array - NDManager manager = NDManager.newBaseManager(); - ByteBuffer bb = manager.allocateDirect(CHANNEL * height * width); - NDArray intRet = manager.create(bb, new Shape(CHANNEL, height, width), DataType.UINT8); - - // change color of pixels in image array where objects have been detected - for (int j = 0; j < height; j++) { - for (int k = 0; k < width; k++) { - int maxi = 0; - double maxnum = -Double.MAX_VALUE; - for (int i = 0; i < CLASSNUM; i++) { - - // get score for each i at the k,j pixel of the image - float score = scores[i * (width * height) + j * width + k]; - if (score > maxnum) { - maxnum = score; - maxi = i; + try (NDManager manager = NDManager.newBaseManager()) { + ByteBuffer bb = manager.allocateDirect(CHANNEL * height * width); + NDArray intRet = manager.create(bb, new Shape(CHANNEL, height, width), DataType.UINT8); + + // change color of pixels in image array where objects have been detected + for (int j = 0; j < height; j++) { + for (int k = 0; k < width; k++) { + int maxi = 0; + double maxnum = -Double.MAX_VALUE; + for (int i = 0; i < CLASSNUM; i++) { + + // get score for each i at the k,j pixel of the image + float score = scores[i * (width * height) + j * width + k]; + if (score > maxnum) { + maxnum = score; + maxi = i; + } } - } - // color pixel if object was found, otherwise leave as is (black) - if (maxi == PERSON || maxi == BIKE) { - NDIndex index = new NDIndex(0, j, k); - intRet.set(index, 0xFF00FF); - } else if (maxi == CAT || maxi == SHEEP || maxi == 13) { - NDIndex index = new NDIndex(1, j, k); - intRet.set(index, 0xFF00FF); - } else if (maxi == CAR || maxi == DOG) { - NDIndex index = new NDIndex(2, j, k); - intRet.set(index, 0xFF00FF); + // color pixel if object was found, otherwise leave as is (black) + if (maxi == PERSON || maxi == BIKE) { + NDIndex index = new NDIndex(0, j, k); + intRet.set(index, 0xFF00FF); + } else if (maxi == CAT || maxi == SHEEP || maxi == 13) { + NDIndex index = new NDIndex(1, j, k); + intRet.set(index, 0xFF00FF); + } else if (maxi == CAR || maxi == DOG) { + NDIndex index = new NDIndex(2, j, k); + intRet.set(index, 0xFF00FF); + } } } + return BufferedImageFactory.getInstance().fromNDArray(intRet); } - return BufferedImageFactory.getInstance().fromNDArray(intRet); } /** @@ -156,8 +154,8 @@ public NDArray transform(NDArray array) { if (Math.round(scale * max) > maxEdge) { scale = maxEdge / (float) max; } - rescaledHeight = Math.round(height * scale); - rescaledWidth = Math.round(width * scale); + int rescaledHeight = Math.round(height * scale); + int rescaledWidth = Math.round(width * scale); return NDImageUtils.resize(array, rescaledWidth, rescaledHeight); } diff --git a/examples/docs/img/semantic_instance_bikes.png b/examples/docs/img/semantic_instance_bikes.png deleted file mode 100644 index 016fcc19cecb2b6f6b2d92ba5507231f2e570819..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3069 zcmc&$XXg}T6 zMyp{km@euE?_d~AGaUxgF!@Fk+WBLg*$;#1JVJSUVopcQ4i5h|zQ@4l!Lq8fPR|b+ z{^#g=n>CMeH_tF&7rlZOU#K9H|NpOy6}b);7`Z$?v>Ntli)z=%%0Thj)$u}=&|Slo z(P`bys=Yu~;AOkh(-f3jgg(q!P_Xg@emUP!S1x*s=cV^566O_4?25?^O}Q_c+%?Vt z#-!W2;CBKabNnHf&SNyqvmN?5BYq|AF_;3cF`}G0t@ofqL8}^6D1?7iu!|E~+FVJO zqIM|t9El0kFf+c;&ddxcSu9Sik1o95cQc? zH0{^JFyq~IJ^n1pPifthh^0!GgKYg$@3N3Gqk0Bkm1*(%|cdpQ6E`KTR8OY`LyPcMWrE8{o5d(WK=_2E-1 zj7^1~$(eA{U0#@12`A`&q+)b3RG*(Pv2UHqJ5=9?~PilVtm*#orbwjHd9we%b9>?U#Mwbc5tSXEOq&@)?pqqs%Ba*2^7)YBL1 z7Wuj$6P$EES-i-l2@o=J<{i>YqR~6Qj;+Rf-uxKr5L0(YL&ex3`t>Sub)on?C*{7j zvSxB27Oq`4q)`fvlv64oe~T;5u#++)vdAR)^oDoXj-9D?a#0*7;66_IaFPqf!7b5< z1a&Hw`_X8;n^uR|(DyI3UFs?m5d{(%Ce8b_&#-o6MPy8qgwy6HEMHu|X#A%TYJR9H z-9p(@D}Pyer>^*J#J8W|k=GU$ZFi)(L1mieemNXC-}Y&=$|{D#)65238iTtPilhut3zp&-Y`LEwuwa{CAI`)(*~Sm;;>Of(^lkh85mDA@mhXejS!} zDTF$j^PG4y@kx3o(AsdA2Ph23%S~={)w2!K4y;v%*2*0COQS8!Zkhas*eZv99xyyL zI43rZIKF{;Vm4$VNwdwqGe2o?(ZVh7bDJ8m!I#RimlB^!p)im10`5Cn?z+|vAn$GM z-2a|zUw^J|H@~-)Ytw7nu5ThrdSV25rKH*OQh6+3>XwYI64ncUCgkhXe>|M%bRb@M z5=DPBr*LT?LXl-cSFDi|at!w{ZJ4XFW`TmJSqLv^$lRzhUuN?0%F-sOu%YA3bkT<& z=T1J4<`32^Fg~*GTwV!Ak8vSz(9I-Vc71i*4aXq+ZRWErkV6y9Hp<9N0|i2Q>U-Xq z@}iHBWnjkh1P)yM^foHmUAb{#W+JyvOw)riT4#^&pNACgdz~AAR?XEte|2v?f8;gf zt2p?!Vz`top z%QOU{`x>*sg-Qs~R#W&7?B4x3E)Wf>0~2jg+Wi6ONGr&3fL%eFztr;-vjRtb0kA4r z5;v=tdsIsu(%-iJ~ z`%C{88LcSks4Z*v;6|2k)N(2@UwoZ-1?r9El&4x`mxySVQ%5$6&J)>CgDt0$T??KK zc)R5m{Sb_D&9C^U7!DQKs(PoHIEYx6C<2)nvjD)viC_pGK-H}OB0_~n)#&rU@0{Yg z6{)Bi-V>hJd(~?VeQ`~!7yKPX?#;5%fk_` z?qyhnl)7rp4mneA4_)4X+?65>=}R<~%rx&W7BZ^6pfIFr4uow{n+de24NdziNf`!e z)#f=2g7JE`JiKan+&lY6ZtFH2Dvun57AlSN!83J|X6Y%Y z7)~iBB5HMt3=E4CKm{pu?g0k|n(!pPBaR1oI^`fj)|QOz zLC-=gL0PDmx01cG1FpiMuK#lh0p@oybefeMfDXK>hFvrnH&?cKyv3wE4C9k%r~3yK z+p(+>(&d^62Y#bt6a~~PM3e}THL3dGYiX=Hj%ZiLNz8(ppf^*uIQNIo2WAHka%`Oo zx~3YQ`)_!B5_t$)TlYNDs1jmKSI#zMAhuS}cG2+w^duM|uK@e84buvPu(DzE?xy45UUe%Kk2q zT{h!CP!KXS9wu=iN+Cwmo36Q*b@@6|()&Ft`? z-gbr2LZem)6V3Px{K9{Z$;t>`u)&niw}Fd%dr+Qz>vYI6{E(40DZEH~=f9Sy zg$Oj**PnO8P{`y1N)WjfsgLmjT9ZyyoOr7(ryNlY9!h_ z+n3^I4D`X%hI;893Y-G($gGIi1>1yPCO8X5L0ftE!JVi&5}a)p@c@3 z>Gp=1$!;k7_x73WOA~3by`NF{v2Aa|(ameWrKnc}C0g+nuXx8ueBoFB1p6_K9wxmX;p= zXSs5E+g?%JV&gi(|NrOHeDP=%0uOL8A2Q{D;dZ_`Hx7X8jJ? zMRE|BBKQ{D1IE4GfBEhJ6{%nBR#9M$Oty%vH3xi@b(WV^l{9QnIj;tn%V`yL=UR8YYYt{)^4c0{ zTVV&kV$WasTdpgmn+(tI)J~`iv)pE(7`U!;%A8o@(RYm3|7fz_wOk`s7p`Ad@;Mav zty|Wn;cmvj0S_AbwGgjqU3!7Cw6YYhG14ub<;VUU-1^7s-MrzEZ3488c1qyv6Eh}z zYR@}`xz+enfy8$ClH4e&vZJHBZ=WZM_;F5LhAkWIB-Ns`dfBLj9k9bgE!x;5lJjzy3jW%WFfJCeeUaQ%yXI(O{z z)1$mHaT$RQh3}u-81BTKHy$)vE+@lfhtFuT@FR!X7&NbJ+{+*}LQSPY_xJ_sCTR5J6 zINe$w`+d*K!MQC9FtcJL zeS&pgN&5L=$pBH3EjNy~bXZLX&0r9u`i9fVCm>JHz@wj6H0tk((!~L$`r2Guy?ad4 zAA@Ls9;gZ*8|^DkPl^PdcGbqtCwkTO4TSp$Ed-qE`{t2+1EkxZ)t_i? zjHbwM12&vz%&bZe1Gc0nD4cu`w|yvo#JwW~7E*p9eUyK4c9{Rr0CK25Gy}sEx@*GV z2=(I+@$m!RKXi6jC9MB56vPFF!Mbaz;q7nHV3n&udOj5P1I4BgS`R`$aP`Ue8}Cx0 zD#6Nu7^r7D-gU+D1srJd_q)G? z`iTQ^WRE9-QY1-&SLUz@KV_O>F41hroBHB_sxVE64<1nQ`1a`-ztQQ;HGbMDOq^&Rd4<2f~^PnBw=^u55Pt8(^kZ)07yI;s!CU={gNBuly?p zylFY?SdhUL_6gj$wQ(Cu)79c|W(7z2lQ1E_l9IrBR zTOR{1wd@KDFk;u_B8CG3DeIghzhOZi6si~W9b6>O*#@FWU4)Pu9+8nF1ll77P^dcA zOMQkjDBOi2)e-{sBrNfy{N%weX+~4${8Q!K=0MJq%yZa_5L~ZbAxC}qk5|gv+DB~CREaRCf=eSt zl=#o>A2!iyvhVw0z->4uqT8GQu-gweGVY6Yw^Ifx`>_sR&o_MPYNF0vi32(JGVPX? z92-7W$E8pWyn)Tz*UwVl%p|ke=Gl5c+P>{YbBS`fZ43q&F>M|5WO%g;IPlFp;hxhF z&f23alFJX>3!YM9F9hynk<(@(h*mdtfo|r7c}r?bH-6TPc+_nt5a)))Tr6N3Np@=U z^kfbIIf~1d&^^><8QLb;v}Z5!G!M+4oFSLX*9hQWK|@;P)aum~4Zn^e-uy#laFi*& z@L`zAxq93|&Rq-3@F3)tcGSA|m4AVs<$FLDYBg`|)VTL3S;~oOYN&Mc*;rxOxqI!C z`-z?t4oJEUJ5_$BeJgDvH&gU69hS2pX|lX0))RU}|0|JT4?OEwr=amYRX~;y(##ud zi}T?x;j{Kj9@-<}xu1SNzKb_kblh6b<+~CIflD!pvM$BmY;K`sIb?34K>s)SDb z47EAQOgP9UK;^1)^8Kg2je2o-HIgDhXQH##RmWkOdpPeFG)-PUn}H}R60F79?0IMK zO#h30J6n6BiJOc3K?eM_vuHcrr&wTV8v5IzrG%qp>`?nE4>=>890Nj%e6VwbzOWuB3uqZggi+C_SM1Ym1ZlCE<$H6ULRu zHbn;xdEmr%B5{@B(~DSkK^DHBcMbi`dUk88Gj&R7y3AEwTVzz8e&UE9vR0)w-OP6f zgWxgVCt&s?r1fZ!xD6eph~~Cl7*}+e(L=Bw?=h!`w4GK@uEkiW>D)eApK4y925&36 z#;7Bh_;e%0euOB4wZPTH3W(lq-nkQKuhhs>8Sbont+C3^1xSlkpJSrI?P7lUe3m^1 z#?`}b3O3IZHT?<6A%e;p@U|;)Bs#3#ksC_4QlOsI5_)$a{3ZtB6g-5DFb*M~hrf&n z$jE~?L3KUQx<&JT)SB!I^}Y!wWCKS`4P*GQ4swXybI1;^q>$El%)pL&u6YKL3OmiT zgchjQYvW}t#?A|;)l-#3ebnWbJ6%}YS?(>jyY@q^+)D-_=JeAg^%TGSU8v)1@92ex zjrCIzYJG@o2&H{xJTy9x`qfdL>Luc%D*x@V-*QvHIO5jUEgg>f^B;S^xDE9x^Tmp8 z7`pHoi7szyLR8!YmvcfR4VG#w8?1cgJ8bkPW2;wO zA_XV7DB+_bzqr+9tqgsYqLCJXJ(8OEJxFc6_nF|h@%$_vu;qCfT%NBWtM>}D5v*HEko)nSYWN>zuZ(DpAF|1P`#7TM zZnK-@tK(eDSSN8BCb(@3gRtpM^yET9B8#1j6Pl;CFP{$BU+gNVeLh0dsj_!*PDbw3 z1%D6;t3P2mq&O*`+|H1r-!HuW78W*36sj6mWrO`6 z4y8|ZNQ&V_zg>7ewq+AlUEDKq&=_BL4X!fp6RtQPpnw}r=iN9vhS^Xv+`=0=>-`h| zjd#*?1p7n%PJUb7n;+Kn;Tvp`hxG*aw+|iEW1NGeRf2|X)iI-J+T>P- z0zIn`pQ&u%7B9!JrRNq}WvUrmikD^VB-q;~zx*w_OXe_11Ah8M?Ao)H7Hvl`!n|SJ z-W1RK*rx(TDKBk5xlNK;etHr8&1zP))*zcQaux z;s)J9g7GOMiIYM>*@Krtf`1`&>=5DOxEWz}i^P zIuuLY;MNTNN-8Ea+ZFk!HO|XHsZK$4>%3)0P|LDRx?PYALD|wc65(b)yMO0 z>j;6m)L46${sSJ-MjPnGXK3DDDhAY%EA8h_C?y8F0i?pwUwj$}cdCnlZ+n{yh8Obn zMkgSoz!Bnb39LK39At?7^N z0Sw;Ksbv5tjgtLXP_c5OLm!Z0;PO9c5Li)-Q3(+TqaJ*t$Cm+EXO4)ep}{xnHAkNh zVRt-4z=QoU>hC&k)<}YdWU_l83LJ0QRh|feOY8|+JOt3}PQj4NqSM?U@R50!b_qgi z$Wt^8G{7V&x)un5o;0${7zW@|wOk6YpeLore*~+-veYKmuGhO!$3hH<%~w zX8w<$H(+QHa8uO)DI7d|PeMSF{kgOR4YH|H&PenM7Dm>`CkHzt=l4rWuJcc$ewv6x ziooCzEWlI0xFE&)1BHT@3BrgUyy;=WVDrH{wbckxt>D$L5IB;2$$b~-uxyK5l>p<1 zyw_m}*tf_-!5Cnn9JA$}6cSP^*E&uh?&eC~MtaSq>=ALKANd{^WBNcEFi!E`Jc$N? z+VW`6b`*H&vbK+|3`jU0l|_d@D#gG<0t+;=S1i9A{y* arguments = new ConcurrentHashMap<>(); + arguments.put("toTensor", "true"); + arguments.put("normalize", "true"); + SemanticSegmentationTranslator translator = + SemanticSegmentationTranslator.builder(arguments).build(); Criteria criteria = Criteria.builder() - .optApplication(Application.CV.SEMANTIC_SEGMENTATION) .setTypes(Image.class, Image.class) - .optModelPath(Paths.get("src/test/resources/deeplabv3_scripted.pt")) - .optTranslator(SemanticSegmentationTranslator.builder().addTransform(new Resize(width, height)).addTransform(new ToTensor()).addTransform(new Normalize(MEAN, STD)).optSynsetUrl("https://mlrepo.djl.ai/model/cv/instance_segmentation/ai/djl/mxnet/mask_rcnn/classes.txt").build()) + .optModelUrls(url) + .optTranslator(translator) .optEngine("PyTorch") .optProgress(new ProgressBar()) .build();