fix(TRT): make refinedet great again, also upgrades to TRT8.0.0/TRT-O…

…SS21.08
jolibrain · Aug 26, 2021 · bdff2ae · bdff2ae
1 parent 825d348
commit bdff2ae
Show file tree

Hide file tree

Showing 11 changed files with 616 additions and 405 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -918,26 +918,19 @@ if (USE_TENSORRT)
     set(TENSORRT_INC_DIR /usr/include/x86_64-linux-gnu)
   endif()
 
-  if (NOT EXISTS "${TRTTESTDIR}/libnvinfer.so.7")
-    message(FATAL_ERROR "Could not find TensorRT ${TENSORRT_LIB_DIR}/libnvinfer.so.7, please provide tensorRT location as TENSORRT_DIR or (TENSORRT_LIB_DIR _and_ TENSORRT_INC_DIR)")
+  if (NOT EXISTS "${TRTTESTDIR}/libnvinfer.so.8")
+    message(FATAL_ERROR "Could not find TensorRT ${TENSORRT_LIB_DIR}/libnvinfer.so.8, please provide tensorRT location as TENSORRT_DIR or (TENSORRT_LIB_DIR _and_ TENSORRT_INC_DIR)")
   else()
     message(STATUS "Found TensorRT libraries : ${TRTTESTDIR}/libnvinfer.so")
   endif()
 
     set(TENSORRT_LIB_DIR ${CMAKE_BINARY_DIR}/tensorrt-oss/bin/ ${TENSORRT_LIB_DIR})
-	set(TENSORRT_LIBS /usr/lib/x86_64-linux-gnu/libnvinfer.so.7 nvcaffeparser nvinfer_plugin nvonnxparser )
+	set(TENSORRT_LIBS nvinfer nvcaffeparser nvinfer_plugin nvonnxparser )
 
     if (NOT TENSORRT_VERSION)
-        if (EXISTS "${TRTTESTDIR}/libnvinfer.so.7.2.3")
-            set(TENSORRT_VERSION 21.04)
-            message(STATUS "Found TensorRT libraries version 7.2.3")
-        # TODO(sileht): Update trt-oss patches for 7.2.2
-        # elseif (EXISTS "${TRTTESTDIR}/libnvinfer.so.7.2.2")
-        #     set(TENSORRT_VERSION 20.12)
-        #     message(STATUS "Found TensorRT libraries version 7.2.2")
-        elseif (EXISTS "${TRTTESTDIR}/libnvinfer.so.7")
-            set(TENSORRT_VERSION 21.04)
-            message(WARNING "No supported TensorRT version found assuming 21.04/7.2.3")
+        if (EXISTS "${TRTTESTDIR}/libnvinfer.so.8")
+            set(TENSORRT_VERSION 21.08)
+            message(STATUS "Found TensorRT libraries version 8.x")
         else()
             message(FATAL_ERROR "No supported TensorRT version found")
         endif()
@@ -958,9 +951,9 @@ if (USE_TENSORRT)
 	set(TRT_PATCHES
       ${TRT_PATCHES_PATH}/0001-support-for-refinedet.patch
       ${TRT_PATCHES_PATH}/0003-c++14.patch
-      ${TRT_PATCHES_PATH}/0002-find-myelin.patch
-      ${TRT_PATCHES_PATH}/0006-find-7.patch
       ${TRT_PATCHES_PATH}/0008-cuda13.patch
+      ${TRT_PATCHES_PATH}/0009-fix_compil_order.patch
+      ${TRT_PATCHES_PATH}/0010-fix_plugin_detect.patch
       )
 
     # Tensorrt uses its own protobuf version (3.0.0) and this cannot be overrided

diff --git a/docker/gpu_tensorrt.Dockerfile b/docker/gpu_tensorrt.Dockerfile
@@ -1,5 +1,5 @@
 # syntax = docker/dockerfile:1.0-experimental
-FROM nvcr.io/nvidia/tensorrt:21.04-py3 AS build
+FROM nvcr.io/nvidia/tensorrt:21.07-py3 AS build
 
 ARG DEEPDETECT_RELEASE=OFF
 ARG DEEPDETECT_ARCH=gpu
@@ -110,7 +110,7 @@ RUN --mount=type=cache,target=/ccache/ mkdir build && cd build && ../build.sh
 RUN ./docker/get_libs.sh
 
 # Build final Docker image
-FROM nvcr.io/nvidia/tensorrt:21.04-py3 AS runtime
+FROM nvcr.io/nvidia/tensorrt:21.07-py3 AS runtime
 
 ARG DEEPDETECT_ARCH=gpu
 

diff --git a/patches/trt/0001-support-for-refinedet.patch b/patches/trt/0001-support-for-refinedet.patch
diff --git a/patches/trt/0008-cuda13.patch b/patches/trt/0008-cuda13.patch
@@ -2,16 +2,7 @@ diff --git a/CMakeLists.txt b/CMakeLists.txt
 index a18cbd0..d23c936 100644
 --- a/CMakeLists.txt
 +++ b/CMakeLists.txt
-@@ -80,7 +80,7 @@ endif()
- ############################################################################################
- # Dependencies
-
--set(DEFAULT_CUDA_VERSION 11.1)
-+set(DEFAULT_CUDA_VERSION 11.3)
- set(DEFAULT_CUDNN_VERSION 8.0)
- set(DEFAULT_PROTOBUF_VERSION 3.0.0)
- set(DEFAULT_CUB_VERSION 1.8.0)
-@@ -152,7 +152,7 @@ else()
+@@ -155,7 +155,7 @@ else()
 
    if (CUDA_VERSION VERSION_GREATER_EQUAL 11.0)
      # Ampere GPU (SM80) support is only available in CUDA versions > 11.0

diff --git a/patches/trt/0009-fix_compil_order.patch b/patches/trt/0009-fix_compil_order.patch
@@ -0,0 +1,17 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index ba3af18..8c8ede2 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -185,7 +185,11 @@ else()
+ endif()
+
+ if(BUILD_PARSERS)
+-    add_subdirectory(parsers)
++  add_subdirectory(parsers)
++  if (BUILD_PLUGINS)
++    add_dependencies(nvonnxparser nvinfer_plugin)
++    add_dependencies(nvcaffeparserlibs nvinfer_plugin)
++  endif()
+ else()
+     find_library_create_target(nvcaffeparser nvparsers SHARED ${TRT_OUT_DIR} ${TRT_LIB_DIR})
+     find_library_create_target(nvonnxparser nvonnxparser SHARED ${TRT_OUT_DIR} ${TRT_LIB_DIR})
diff --git a/patches/trt/0010-fix_plugin_detect.patch b/patches/trt/0010-fix_plugin_detect.patch
@@ -0,0 +1,13 @@
+diff --git a/parsers/onnx/CMakeLists.txt b/parsers/onnx/CMakeLists.txt
+index 4ea7fe6..2c2ad82 100644
+--- a/parsers/onnx/CMakeLists.txt
++++ b/parsers/onnx/CMakeLists.txt
+@@ -99,7 +99,7 @@ find_library(TENSORRT_LIBRARY_INFER nvinfer
+ find_library(TENSORRT_LIBRARY_INFER_PLUGIN nvinfer_plugin
+   HINTS  ${TENSORRT_ROOT} ${TENSORRT_BUILD} ${CUDA_TOOLKIT_ROOT_DIR}
+   PATH_SUFFIXES lib lib64 lib/x64)
+-set(TENSORRT_LIBRARY ${TENSORRT_LIBRARY_INFER} ${TENSORRT_LIBRARY_INFER_PLUGIN})
++set(TENSORRT_LIBRARY nvinfer nvinfer_plugin)
+ MESSAGE(STATUS "Find TensorRT libs at ${TENSORRT_LIBRARY}")
+ find_package_handle_standard_args(
+   TENSORRT DEFAULT_MSG TENSORRT_INCLUDE_DIR TENSORRT_LIBRARY)
diff --git a/src/backends/tensorrt/protoUtils.cc b/src/backends/tensorrt/protoUtils.cc
@@ -107,6 +107,22 @@ namespace dd
     return -1;
   }
 
+  bool isRefinedet(const std::string source)
+  {
+    caffe::NetParameter net;
+    if (!TRTReadProtoFromTextFile(source.c_str(), &net))
+      return -1;
+    int nlayers = net.layer_size();
+
+    for (int i = nlayers - 1; i >= 0; --i)
+      {
+        caffe::LayerParameter lparam = net.layer(i);
+        if (lparam.type() == "DetectionOutput" && lparam.bottom_size() > 3)
+          return true;
+      }
+    return false;
+  }
+
   int fixProto(const std::string dest, const std::string source)
   {
     caffe::NetParameter source_net;

diff --git a/src/backends/tensorrt/protoUtils.h b/src/backends/tensorrt/protoUtils.h
@@ -40,5 +40,6 @@ namespace dd
                                 google::protobuf::Message *proto);
   bool TRTWriteProtoToTextFile(const google::protobuf::Message &proto,
                                const char *filename);
+  bool isRefinedet(const std::string source);
 }
 #endif
diff --git a/src/backends/tensorrt/tensorrtlib.cc b/src/backends/tensorrt/tensorrtlib.cc
@@ -109,6 +109,7 @@ namespace dd
     _floatOut = tl._floatOut;
     _keepCount = tl._keepCount;
     _dims = tl._dims;
+    _runtime = tl._runtime;
   }
 
   template <class TInputConnectorStrategy, class TOutputConnectorStrategy,
@@ -125,6 +126,8 @@ namespace dd
   {
     trtLogger.setLogger(this->_logger);
     initLibNvInferPlugins(&trtLogger, "");
+    _runtime = std::shared_ptr<nvinfer1::IRuntime>(
+        nvinfer1::createInferRuntime(trtLogger));
 
     if (ad.has("tensorRTEngineFile"))
       _engineFileName = ad.get("tensorRTEngineFile").get<std::string>();
@@ -168,6 +171,15 @@ namespace dd
           _datatype = nvinfer1::DataType::kINT8;
       }
 
+    if (this->_mlmodel.is_caffe_source() && isRefinedet(this->_mlmodel._def))
+      {
+        if (_datatype != nvinfer1::DataType::kFLOAT)
+          {
+            this->_logger->warn("refinedet detected : forcing fp32");
+            _datatype = nvinfer1::DataType::kFLOAT;
+          }
+      }
+
     if (ad.has("gpuid"))
       _gpuid = ad.get("gpuid").get<int>();
     cudaSetDevice(_gpuid);
@@ -182,11 +194,9 @@ namespace dd
       }
 
     _builder = std::shared_ptr<nvinfer1::IBuilder>(
-        nvinfer1::createInferBuilder(trtLogger),
-        [=](nvinfer1::IBuilder *b) { b->destroy(); });
+        nvinfer1::createInferBuilder(trtLogger));
     _builderc = std::shared_ptr<nvinfer1::IBuilderConfig>(
-        _builder->createBuilderConfig(),
-        [=](nvinfer1::IBuilderConfig *b) { b->destroy(); });
+        _builder->createBuilderConfig());
 
     if (_dla != -1)
       {
@@ -354,14 +364,10 @@ namespace dd
       }
     // force output to be float32
     outl->setPrecision(nvinfer1::DataType::kFLOAT);
-    nvinfer1::ICudaEngine *engine
-        = _builder->buildEngineWithConfig(*network, *_builderc);
-
-    network->destroy();
-    if (caffeParser != nullptr)
-      caffeParser->destroy();
+    nvinfer1::IHostMemory *n
+        = _builder->buildSerializedNetwork(*network, *_builderc);
 
-    return engine;
+    return _runtime->deserializeCudaEngine(n->data(), n->size());
   }
 
   template <class TInputConnectorStrategy, class TOutputConnectorStrategy,
@@ -396,14 +402,10 @@ namespace dd
     _builder->setMaxBatchSize(_max_batch_size);
     _builderc->setMaxWorkspaceSize(_max_workspace_size);
 
-    nvinfer1::ICudaEngine *engine
-        = _builder->buildEngineWithConfig(*network, *_builderc);
-
-    network->destroy();
-    if (onnxParser != nullptr)
-      onnxParser->destroy();
+    nvinfer1::IHostMemory *n
+        = _builder->buildSerializedNetwork(*network, *_builderc);
 
-    return engine;
+    return _runtime->deserializeCudaEngine(n->data(), n->size());
   }
 
   template <class TInputConnectorStrategy, class TOutputConnectorStrategy,
@@ -523,16 +525,9 @@ namespace dd
                 trtModelStream.resize(size);
                 file.read(trtModelStream.data(), size);
                 file.close();
-                nvinfer1::IRuntime *runtime
-                    = nvinfer1::createInferRuntime(trtLogger);
                 _engine = std::shared_ptr<nvinfer1::ICudaEngine>(
-                    runtime->deserializeCudaEngine(
-                        trtModelStream.data(), trtModelStream.size(), nullptr),
-                    [=](nvinfer1::ICudaEngine *e) {
-                      if (e != nullptr)
-                        e->destroy();
-                    });
-                runtime->destroy();
+                    _runtime->deserializeCudaEngine(trtModelStream.data(),
+                                                    trtModelStream.size()));
 
                 if (_engine == nullptr)
                   throw MLLibInternalException(
@@ -562,8 +557,7 @@ namespace dd
                     "No model to parse for conversion to TensorRT");
               }
 
-            _engine = std::shared_ptr<nvinfer1::ICudaEngine>(
-                le, [=](nvinfer1::ICudaEngine *e) { e->destroy(); });
+            _engine = std::shared_ptr<nvinfer1::ICudaEngine>(le);
 
             if (_writeEngine)
               {
@@ -574,7 +568,6 @@ namespace dd
                 nvinfer1::IHostMemory *trtModelStream = _engine->serialize();
                 p.write(reinterpret_cast<const char *>(trtModelStream->data()),
                         trtModelStream->size());
-                trtModelStream->destroy();
               }
           }
         else
@@ -585,11 +578,7 @@ namespace dd
           }
 
         _context = std::shared_ptr<nvinfer1::IExecutionContext>(
-            _engine->createExecutionContext(),
-            [=](nvinfer1::IExecutionContext *e) {
-              if (e != nullptr)
-                e->destroy();
-            });
+            _engine->createExecutionContext());
         _TRTContextReady = true;
 
         try

diff --git a/src/backends/tensorrt/tensorrtlib.h b/src/backends/tensorrt/tensorrtlib.h
@@ -51,7 +51,8 @@ namespace dd
     {
     }
 
-    void log(nvinfer1::ILogger::Severity severity, const char *msg) override
+    void log(nvinfer1::ILogger::Severity severity,
+             const char *msg) noexcept override
     {
       switch (severity)
         {
@@ -126,6 +127,7 @@ namespace dd
     std::shared_ptr<nvinfer1::IBuilder> _builder = nullptr;
     std::shared_ptr<nvinfer1::IExecutionContext> _context = nullptr;
     std::shared_ptr<nvinfer1::IBuilderConfig> _builderc = nullptr;
+    std::shared_ptr<nvinfer1::IRuntime> _runtime = nullptr;
 
     bool _bbox = false;
     bool _ctc = false;

diff --git a/tests/ut-tensorrtapi.cc b/tests/ut-tensorrtapi.cc
@@ -105,7 +105,7 @@ TEST(tensorrtapi, service_predict_best)
         + "\"},\"parameters\":{\"input\":{\"connector\":\"image\",\"height\":"
           "224,\"width\":224},\"mllib\":{\"datatype\":\"fp32\","
           "\"maxBatchSize\":1,\"maxWorkspaceSize\":256,"
-          "\"tensorRTEngineFile\":\"TRTengine_bs\",\"gpuid\":0}}}";
+          "\"tensorRTEngineFile\":\"TRTengine\",\"gpuid\":0}}}";
   std::string joutstr = japi.jrender(japi.service_create(sname, jstr));
   ASSERT_EQ(created_str, joutstr);
 
@@ -125,6 +125,8 @@ TEST(tensorrtapi, service_predict_best)
   std::string age
       = jd["body"]["predictions"][0]["classes"][0]["cat"].GetString();
   ASSERT_TRUE(age == "29");
+  std::cout << "looking for " << age_repo << "TRTengine_arch"
+            << get_trt_archi() << "_bs1" << std::endl;
   ASSERT_TRUE(fileops::file_exists(age_repo + "TRTengine_arch"
                                    + get_trt_archi() + "_bs1"));
   /*ASSERT_TRUE(!fileops::remove_file(age_repo, "net_tensorRT.proto"));
@@ -176,14 +178,14 @@ TEST(tensorrtapi, service_predict_refinedet)
   ASSERT_TRUE(!jd.HasParseError());
   ASSERT_EQ(400, jd["status"]["code"]);
 
-  ASSERT_TRUE(fileops::file_exists(squeez_repo + "TRTengine_arch"
-                                   + get_trt_archi() + "_bs48"));
+  ASSERT_TRUE(fileops::file_exists(refinedet_repo + "TRTengine_arch"
+                                   + get_trt_archi() + "_bs1"));
   jstr = "{\"clear\":\"lib\"}";
   joutstr = japi.jrender(japi.service_delete(sname, jstr));
   ASSERT_EQ(ok_str, joutstr);
-  ASSERT_TRUE(!fileops::file_exists(squeez_repo + "net_tensorRT.proto"));
-  ASSERT_TRUE(!fileops::file_exists(squeez_repo + "TRTengine_arch"
-                                    + get_trt_archi() + "_bs48"));
+  ASSERT_TRUE(!fileops::file_exists(refinedet_repo + "net_tensorRT.proto"));
+  ASSERT_TRUE(!fileops::file_exists(refinedet_repo + "TRTengine_arch"
+                                    + get_trt_archi() + "_bs1"));
 }
 
 TEST(tensorrtapi, service_predict_onnx)