notebook/onnx-pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Copyright (c) Microsoft Corporation. All rights reserved.\n",
    "\n",
    "Licensed under the MIT License."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ONNX Pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This repository shows how to deploy and use ONNX pipeline with dockers including convert model, generate input and performance test."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prerequisites"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pull dockers from Azure. It should take several minutes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# !sh build.sh # For Linux\n",
    "!build.sh # For Windows"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Install the onnxpipeline SDK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../utils')\n",
    "import onnxpipeline\n",
    "\n",
    "# Initiate ONNX pipeline with local directory \"model\"\n",
    "pipeline = onnxpipeline.Pipeline('model')\n",
    "\n",
    "# onnx\n",
    "#pipeline = onnxpipeline.Pipeline('onnx')\n",
    "\n",
    "# tensorflow\n",
    "#pipeline = onnxpipeline.Pipeline('mnist/model')\n",
    "\n",
    "# pytorch\n",
    "#pipeline = onnxpipeline.Pipeline('pytorch')\n",
    "\n",
    "# cntk \n",
    "#pipeline = onnxpipeline.Pipeline('cntk')\n",
    "\n",
    "# keras\n",
    "#pipeline = onnxpipeline.Pipeline('KerasToONNX')\n",
    "\n",
    "# sklearn\n",
    "#pipeline = onnxpipeline.Pipeline('sklearn')\n",
    "\n",
    "# caffe\n",
    "#pipeline = onnxpipeline.Pipeline('caffe')\n",
    "\n",
    "# current directory\n",
    "#pipeline = onnxpipeline.Pipeline()\n",
    "\n",
    "# test mxnet fail\n",
    "#pipeline = onnxpipeline.Pipeline('mxnet')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run parameters\n",
    "\n",
    "(1) local_directory: string\n",
    "\n",
    "    Required. The path of local directory where would be mounted to the docker. All operations will be executed from this path.\n",
    "\n",
    "(2) mount_path: string\n",
    "\n",
    "    Optional. The path where the local_directory will be mounted in the docker. Default is \"/mnt/model\".\n",
    "\n",
    "(3) print_logs: boolean\n",
    "\n",
    "    Optional. Whether print the logs from the docker. Default is True.\n",
    "    \n",
    "(4) convert_directory: string\n",
    "\n",
    "    Optional. The directory path for converting model. Default is test/.    \n",
    "\n",
    "(5) convert_name: string\n",
    "\n",
    "    Optional. The model name for converting model. Default is model.onnx.\n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Config information for ONNX pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------config----------------\n",
      "           Container information: <docker.client.DockerClient object at 0x000001E5C77FD6D8>\n",
      " Local directory path for volume: E:\\onnx-pipeline\\notebook/model\n",
      "Volume directory path in dockers: /mnt/model\n",
      "                     Result path: result\n",
      "        Converted directory path: test\n",
      "        Converted model filename: model.onnx\n",
      "            Converted model path: test/model.onnx\n",
      "        Print logs in the docker: True\n"
     ]
    }
   ],
   "source": [
    "pipeline.config()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert model to ONNX"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This image is used to convert model from major model frameworks to onnx. Supported frameworks are - caffe, cntk, coreml, keras, libsvm, mxnet, scikit-learn, tensorflow and pytorch.\n",
    "\n",
    "\n",
    "You can run the docker image with customized parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r\n",
      "\n",
      "-------------\r\n",
      "\n",
      "Model Conversion\r\n",
      "\n",
      "\r\n",
      "\n",
      "Input model is already ONNX model. Skipping conversion.\r\n",
      "\n",
      "\r\n",
      "\n",
      "-------------\r\n",
      "\n",
      "MODEL INPUT GENERATION(if needed)\r\n",
      "\n",
      "\r\n",
      "\n",
      "Input.pb already exists. Skipping dummy input generation. \r\n",
      "\n",
      "\r\n",
      "\n",
      "-------------\r\n",
      "\n",
      "MODEL CORRECTNESS VERIFICATION\r\n",
      "\n",
      "\r\n",
      "\n",
      "\r\n",
      "\n",
      "Check the ONNX model for validity \r\n",
      "\n",
      "The ONNX model is valid.\r\n",
      "\n",
      "\r\n",
      "\n",
      "The original model is already onnx. Skipping correctness test. \r\n",
      "\n",
      "\r\n",
      "\n",
      "-------------\r\n",
      "\n",
      "MODEL CONVERSION SUMMARY (.json file generated at /mnt/model/test/output.json )\r\n",
      "\n",
      "\r\n",
      "\n",
      "{'conversion_status': 'SUCCESS',\r\n",
      "\n",
      " 'correctness_verified': 'SKIPPED',\r\n",
      "\n",
      " 'error_message': '',\r\n",
      "\n",
      " 'input_folder': '/mnt/model/test/test_data_set_0',\r\n",
      "\n",
      " 'output_onnx_path': '/mnt/model/test/model.onnx'}\r\n",
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "model = pipeline.convert_model(model='model.onnx', model_type='onnx')\n",
    "\n",
    "# test tensorflow\n",
    "#model = pipeline.convert_model(model_type='tensorflow')\n",
    "\n",
    "# test pytorch\n",
    "#model = pipeline.convert_model(model_type='pytorch', model='saved_model.pb', model_input_shapes='(1,3,224,224)')\n",
    "\n",
    "# test cntk\n",
    "#model = pipeline.convert_model(model_type='cntk', model='ResNet50_ImageNet_Caffe.model')\n",
    "\n",
    "# test keras\n",
    "#model = pipeline.convert_model(model_type='keras', model='keras_Average_ImageNet.keras')\n",
    "\n",
    "# test sklearn\n",
    "#model = pipeline.convert_model(model_type='scikit-learn', model='sklearn_svc.joblib', initial_types=(\"float_input\", \"FloatTensorType([1,4])\"))\n",
    "\n",
    "# test caffe\n",
    "#model = pipeline.convert_model(model_type='caffe', model='bvlc_alexnet.caffemodel')\n",
    "\n",
    "# test mxnet\n",
    "#model = pipeline.convert_model(model_type='mxnet', model='resnet.json', model_params='resnet.params', model_input_shapes='(1,3,224,224)')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run parameters\n",
    "\n",
    "(1) model: string\n",
    "\n",
    "    Required. The path of the model that needs to be converted.\n",
    "    \n",
    "    **IMPORTANT Only support the model path which is under the mounting directory (while initialization by Pipeline()).\n",
    "\n",
    "(2) output_onnx_path: string\n",
    "\n",
    "    Required. The path to store the converted onnx model. Should end with \".onnx\". e.g. output.onnx\n",
    "\n",
    "(3) model_type: string\n",
    "\n",
    "    Required. The name of original model framework. Available types are caffe, cntk, coreml, keras, libsvm, mxnet, scikit-learn, tensorflow and pytorch.\n",
    "\n",
    "(4) model_inputs_names: string\n",
    "\n",
    "    (tensorflow) Optional. The model's input names. Required for tensorflow frozen models and checkpoints.\n",
    "\n",
    "(5) model_outputs_names: string\n",
    "\n",
    "    (tensorflow) Optional. The model's output names. Required for tensorflow frozen models checkpoints.\n",
    "\n",
    "(6) model_params: string \n",
    "\n",
    "    (mxnet) Optional. The params of the model if needed.\n",
    "\n",
    "(7) model_input_shapes: list of tuple \n",
    "\n",
    "    (pytorch, mxnet) Optional. List of tuples. The input shape(s) of the model. Each dimension separated by ','.\n",
    "\n",
    "(8) target_opset: String\n",
    "\n",
    "    Optional. Specifies the opset for ONNX, for example, 7 for ONNX 1.2, and 8 for ONNX 1.3. Defaults to '7'.\n",
    "    \n",
    "(9) caffe_model_prototxt: string\n",
    "\n",
    "    (caffe) Optional. The filename of deploy prototxt for the caffe madel. \n",
    "\n",
    "(10) initial_types: tuple (string, string)\n",
    "\n",
    "    (scikit-learn) Optional. A tuple consist two strings. The first is data type and the second is the size of tensor type e.g., ('float_input', 'FloatTensorType([1,4])')\n",
    "\n",
    "(11) input_json: string\n",
    "\n",
    "    Optional. Use JSON file as input parameters.\n",
    "    \n",
    "    **IMPORTANT Only support the path which is under the mounting directory (while initialization by Pipeline()).\n",
    "    \n",
    "(12) model_inputs_names: string\n",
    "\n",
    "    (tensorflow) Optional.\n",
    "    \n",
    "(13) model_outputs_names: string\n",
    "\n",
    "    (tensorflow) Optional.    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performance test tool"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can run perf_tuning using command python perf_tuning.py [Your model path] [Output path on the docker]. You can use the same arguments as for onnxruntime_pert_test tool, e.g. -m for mode, -e to specify execution provider etc. By default it will try all providers available."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-08-30 18:21:00.936705200 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:00.936788600 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:00.936795600 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:00.936800300 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 1\n",
      "\n",
      "Total time cost:0.0617255\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:3.08627 ms\n",
      "\n",
      "2019-08-30 18:21:01.180742700 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:01.180851500 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:01.180858800 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:01.180863100 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 1\n",
      "\n",
      "Total time cost:0.0253154\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:1.26577 ms\n",
      "\n",
      "2019-08-30 18:21:01.398435500 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:01.398513000 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:01.398533400 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:01.398537800 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.0255808\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:1.27904 ms\n",
      "\n",
      "2019-08-30 18:21:01.705239300 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:01.705295800 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:01.705302800 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:01.705307200 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.058074\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:2.9037 ms\n",
      "\n",
      "2019-08-30 18:21:01.955712300 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:01.955766600 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:01.955773400 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:01.955777700 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.0248691\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:1.24345 ms\n",
      "\n",
      "2019-08-30 18:21:02.209768100 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:02.209824700 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:02.209846600 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:02.209850800 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.060125\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:3.00625 ms\n",
      "\n",
      "2019-08-30 18:21:02.458939400 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:02.458994400 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:02.459016100 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:02.459020300 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.0262749\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:1.31375 ms\n",
      "\n",
      "2019-08-30 18:21:02.701319700 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:02.701414200 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:02.701434200 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:02.701438300 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.0529148\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:2.64574 ms\n",
      "\n",
      "2019-08-30 18:21:02.941667600 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:02.941721200 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:02.941728100 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:02.941732600 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.0249799\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:1.24899 ms\n",
      "\n",
      "2019-08-30 18:21:03.158992100 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:03.159077300 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:03.159098000 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:03.159102300 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.0293959\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:1.4698 ms\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-08-30 18:21:03.379854600 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:03.379912600 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:03.379919300 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:03.379923600 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.0280829\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:1.40415 ms\n",
      "\n",
      "2019-08-30 18:21:03.596514100 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:03.596614700 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:03.596621800 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:03.596639100 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.026462\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:1.3231 ms\n",
      "\n",
      "2019-08-30 18:21:03.922145200 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:03.922199800 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:03.922206400 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:03.922210700 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 1\n",
      "\n",
      "Total time cost:0.115866\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:5.79332 ms\n",
      "\n",
      "2019-08-30 18:21:04.239855700 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:04.239909800 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:04.239931500 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:04.239948800 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 1\n",
      "\n",
      "Total time cost:0.0151312\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:0.75656 ms\n",
      "\n",
      "2019-08-30 18:21:04.441897900 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:04.441953100 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:04.441960100 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:04.441964700 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.0148912\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:0.74456 ms\n",
      "\n",
      "2019-08-30 18:21:04.750284900 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:04.750339200 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:04.750346000 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:04.750350400 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.113686\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:5.6843 ms\n",
      "\n",
      "2019-08-30 18:21:05.066663100 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:05.066716800 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:05.066723500 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:05.066728100 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.0251954\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:1.25977 ms\n",
      "\n",
      "2019-08-30 18:21:05.399165500 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:05.399222900 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:05.399244900 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:05.399249100 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.113965\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:5.69827 ms\n",
      "\n",
      "2019-08-30 18:21:05.725907600 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:05.725963000 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:05.725969700 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:05.725974000 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.0249359\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:1.2468 ms\n",
      "\n",
      "2019-08-30 18:21:06.046263700 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:06.046318200 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:06.046325000 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:06.046329300 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.115368\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:5.76841 ms\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-08-30 18:21:06.376067800 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:06.376137600 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:06.376144800 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:06.376148900 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.0301617\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:1.50808 ms\n",
      "\n",
      "2019-08-30 18:21:06.592665100 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:06.592721800 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:06.592728800 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:06.592733300 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.0155429\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:0.777145 ms\n",
      "\n",
      "2019-08-30 18:21:06.838994400 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:06.839049000 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:06.839055800 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:06.839060300 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.0510955\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:2.55478 ms\n",
      "\n",
      "2019-08-30 18:21:07.077309800 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:07.077364400 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:07.077371000 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:07.077375400 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.0195002\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:0.97501 ms\n",
      "\n",
      "2019-08-30 18:21:07.334269400 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:07.334324500 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:07.334331200 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:07.334335600 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 1\n",
      "\n",
      "Total time cost:0.0608857\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:3.04428 ms\n",
      "\n",
      "2019-08-30 18:21:07.585076800 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:07.585132900 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:07.585139500 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:07.585143700 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 1\n",
      "\n",
      "Total time cost:0.0252169\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:1.26085 ms\n",
      "\n",
      "2019-08-30 18:21:07.792479000 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:07.792582900 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:07.792589900 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:07.792607400 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.02608\n",
      "\n",
      "Total iterations:20\n",
      "\n",
      "Average time cost:1.304 ms\n",
      "\n",
      "/home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:97 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] /home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:91 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] CUDA failure 35: CUDA driver version is insufficient for CUDA runtime version ; GPU=32609 ; hostname=72b03b9a3bef ; expr=cudaSetDevice(device_id_); \n",
      "\n",
      "Stacktrace:\n",
      "\n",
      "\n",
      "\n",
      "Stacktrace:\n",
      "\n",
      "\n",
      "\n",
      "Setting thread pool size to 1\n",
      "\n",
      "/home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:97 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] /home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:91 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] CUDA failure 35: CUDA driver version is insufficient for CUDA runtime version ; GPU=32685 ; hostname=72b03b9a3bef ; expr=cudaSetDevice(device_id_); \n",
      "\n",
      "Stacktrace:\n",
      "\n",
      "\n",
      "\n",
      "Stacktrace:\n",
      "\n",
      "\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "/home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:97 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] /home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:91 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] CUDA failure 35: CUDA driver version is insufficient for CUDA runtime version ; GPU=32617 ; hostname=72b03b9a3bef ; expr=cudaSetDevice(device_id_); \n",
      "\n",
      "Stacktrace:\n",
      "\n",
      "\n",
      "\n",
      "Stacktrace:\n",
      "\n",
      "\n",
      "\n",
      "Setting thread pool size to 1\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:97 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] /home/ziyl/onnxruntime/onnxruntime/core/providers/cuda/cuda_call.cc:91 bool onnxruntime::CudaCall(ERRTYPE, const char*, const char*, ERRTYPE, const char*) [with ERRTYPE = cudaError; bool THRW = true] CUDA failure 35: CUDA driver version is insufficient for CUDA runtime version ; GPU=32566 ; hostname=72b03b9a3bef ; expr=cudaSetDevice(device_id_); \n",
      "\n",
      "Stacktrace:\n",
      "\n",
      "\n",
      "\n",
      "Stacktrace:\n",
      "\n",
      "\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "2019-08-30 18:21:08.635924500 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:08.636045000 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:08.636067200 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:08.636071600 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 1\n",
      "\n",
      "Total time cost:0.256121\n",
      "\n",
      "Total iterations:200\n",
      "\n",
      "Average time cost:1.28061 ms\n",
      "\n",
      "Cores:  1\n",
      "\n",
      "No GPU found on current device. Cuda and TensorRT performance tuning might not be available. \n",
      "\n",
      "providers  ['mklml', 'cpu_openmp', 'mkldnn', 'mkldnn_openmp', 'cpu', 'tensorrt', 'ngraph', 'cuda']\n",
      "\n",
      "\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/mklml/onnxruntime_perf_test -P -x 1 -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/4402ea3b-bf1d-41e1-b29c-5366fc0539be\n",
      "\n",
      "mklml_parallel_1_threads 3.0862749999999997\n",
      "\n",
      "\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/mklml/onnxruntime_perf_test -x 1 -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/01047ecb-090f-4497-960f-71d35ac55d59\n",
      "\n",
      "mklml_1_threads 1.26577\n",
      "\n",
      "\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/mklml/onnxruntime_perf_test -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/aadb677d-84b9-4c8f-965a-6e353aba716b\n",
      "\n",
      "mklml 1.27904\n",
      "\n",
      "\n",
      "\n",
      "OMP_WAIT_POLICY=active\n",
      "\n",
      "OMP_NUM_THREADS=1\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -P -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/9a09422d-6bb1-4911-b710-e6302adab9f7\n",
      "\n",
      "cpu_openmp_parallel_1_threads_OMP_WAIT_POLICY_active 2.9036999999999997\n",
      "\n",
      "\n",
      "\n",
      "OMP_WAIT_POLICY=active\n",
      "\n",
      "OMP_NUM_THREADS=1\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/38097b84-2694-4454-be29-dad2b2315e20\n",
      "\n",
      "cpu_openmp_1_threads_OMP_WAIT_POLICY_active 1.243455\n",
      "\n",
      "\n",
      "\n",
      "OMP_WAIT_POLICY=passive\n",
      "\n",
      "OMP_NUM_THREADS=1\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -P -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/6151aa03-f7c5-4b46-869a-2d347c46bd19\n",
      "\n",
      "cpu_openmp_parallel_1_threads_OMP_WAIT_POLICY_passive 3.0062499999999996\n",
      "\n",
      "\n",
      "\n",
      "OMP_WAIT_POLICY=passive\n",
      "\n",
      "OMP_NUM_THREADS=1\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/72dbf715-8b3b-4b9a-8b42-7a0e440b89bc\n",
      "\n",
      "cpu_openmp_1_threads_OMP_WAIT_POLICY_passive 1.3137450000000002\n",
      "\n",
      "\n",
      "\n",
      "OMP_NUM_THREADS=1\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -P -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/bfd07168-8e05-4701-8b45-34d8c872a0dd\n",
      "\n",
      "cpu_openmp_parallel_1_threads 2.64574\n",
      "\n",
      "\n",
      "\n",
      "OMP_NUM_THREADS=1\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/8dd9526c-c60d-41a4-8baf-ef3383fe421f\n",
      "\n",
      "cpu_openmp_1_threads 1.2489949999999999\n",
      "\n",
      "\n",
      "\n",
      "OMP_WAIT_POLICY=active\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/8f63dde5-b5ab-4877-8951-4950a49a78c9\n",
      "\n",
      "cpu_openmp_OMP_WAIT_POLICY_active 1.4697950000000002\n",
      "\n",
      "\n",
      "\n",
      "OMP_WAIT_POLICY=passive\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/17a50b73-ef31-4bdd-8bb2-39450cbebd58\n",
      "\n",
      "cpu_openmp_OMP_WAIT_POLICY_passive 1.4041450000000002\n",
      "\n",
      "\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/b332a535-1299-4820-a89d-eab7fe57463c\n",
      "\n",
      "cpu_openmp 1.3231\n",
      "\n",
      "\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -P -x 1 -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/50ff53f2-a55c-4553-b47d-5c17357c1c0d\n",
      "\n",
      "mkldnn_parallel_1_threads 5.793315000000001\n",
      "\n",
      "\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -x 1 -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/ca8f0016-678e-4523-93e6-520e92cca22d\n",
      "\n",
      "mkldnn_1_threads 0.75656\n",
      "\n",
      "\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/d4af9b27-ecd7-4e7f-b5d5-17ad75121b29\n",
      "\n",
      "mkldnn 0.74456\n",
      "\n",
      "\n",
      "\n",
      "OMP_WAIT_POLICY=active\n",
      "\n",
      "OMP_NUM_THREADS=1\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -P -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/4bee89b6-75d4-4220-b530-eee3d94a921a\n",
      "\n",
      "mkldnn_openmp_parallel_1_threads_OMP_WAIT_POLICY_active 5.6842999999999995\n",
      "\n",
      "\n",
      "\n",
      "OMP_WAIT_POLICY=active\n",
      "\n",
      "OMP_NUM_THREADS=1\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/e69ba203-c3fc-4bb4-9225-2b5ba178c8b9\n",
      "\n",
      "mkldnn_openmp_1_threads_OMP_WAIT_POLICY_active 1.2597699999999998\n",
      "\n",
      "\n",
      "\n",
      "OMP_WAIT_POLICY=passive\n",
      "\n",
      "OMP_NUM_THREADS=1\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -P -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/cb8233d1-ad43-4242-96c8-913c077e5984\n",
      "\n",
      "mkldnn_openmp_parallel_1_threads_OMP_WAIT_POLICY_passive 5.69827\n",
      "\n",
      "\n",
      "\n",
      "OMP_WAIT_POLICY=passive\n",
      "\n",
      "OMP_NUM_THREADS=1\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/81510efa-478b-4cc3-bbd6-38672a19a0d0\n",
      "\n",
      "mkldnn_openmp_1_threads_OMP_WAIT_POLICY_passive 1.2467949999999999\n",
      "\n",
      "\n",
      "\n",
      "OMP_NUM_THREADS=1\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -P -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/aed44390-d6cc-477c-89c6-480c63be3d78\n",
      "\n",
      "mkldnn_openmp_parallel_1_threads 5.76841\n",
      "\n",
      "\n",
      "\n",
      "OMP_NUM_THREADS=1\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/88967d47-0bab-4a23-9fbc-96b0b598ad92\n",
      "\n",
      "mkldnn_openmp_1_threads 1.508085\n",
      "\n",
      "\n",
      "\n",
      "OMP_WAIT_POLICY=active\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/71d9d5f4-3f1b-4589-bc7d-fa55e7b554e5\n",
      "\n",
      "mkldnn_openmp_OMP_WAIT_POLICY_active 0.777145\n",
      "\n",
      "\n",
      "\n",
      "OMP_WAIT_POLICY=passive\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/63c7b7eb-2897-46f5-a0d0-d60f4f49532a\n",
      "\n",
      "mkldnn_openmp_OMP_WAIT_POLICY_passive 2.554775\n",
      "\n",
      "\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/ffb9d8f3-fa16-4635-9329-65c822ddb3b6\n",
      "\n",
      "mkldnn_openmp 0.97501\n",
      "\n",
      "\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -P -x 1 -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/76f0cec6-eb24-42ee-ac81-f6da7829c047\n",
      "\n",
      "cpu_parallel_1_threads 3.044285\n",
      "\n",
      "\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -x 1 -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/8ff91ef5-1b17-4799-b709-a5f1e67e1d59\n",
      "\n",
      "cpu_1_threads 1.260845\n",
      "\n",
      "\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/3df68986-ce41-4c18-9eb0-ec09f294c2ba\n",
      "\n",
      "cpu 1.304\n",
      "\n",
      "\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e tensorrt -x 1 -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/7142d4af-c130-492e-9778-ae6a36be6d73\n",
      "\n",
      "tensorrt_1_threads None\n",
      "\n",
      "\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e tensorrt -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/5f910f6f-2fd4-4e3b-853e-23b8224a043e\n",
      "\n",
      "tensorrt None\n",
      "\n",
      "\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/ngraph/onnxruntime_perf_test -e ngraph -P -x 1 -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/f14f99b5-d998-4684-a2f7-1208dea09372\n",
      "\n",
      "ngraph_parallel_1_threads None\n",
      "\n",
      "\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/ngraph/onnxruntime_perf_test -e ngraph -x 1 -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/40577dce-808e-4e47-9de8-87d203929fef\n",
      "\n",
      "ngraph_1_threads None\n",
      "\n",
      "\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/ngraph/onnxruntime_perf_test -e ngraph -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/def16a31-b4db-48a0-b188-029cb99588ec\n",
      "\n",
      "ngraph None\n",
      "\n",
      "\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e cuda -x 1 -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/23cebcb8-4b73-4403-bae4-7d175b3821e1\n",
      "\n",
      "cuda_1_threads None\n",
      "\n",
      "\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e cuda -o 3 -m times -r 20 /mnt/model/test/model.onnx /mnt/model/result/0d0bf070-bffa-487f-914f-40bb9ffa76f3\n",
      "\n",
      "cuda None\n",
      "\n",
      "\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/mklml/onnxruntime_perf_test -x 1 -o 3 -m times -r 200 /mnt/model/test/model.onnx /mnt/model/result/40f14513-9eff-49f6-948c-8196a090238a\n",
      "\n",
      "mklml_1_threads 1.280607\n",
      "\n",
      "\n",
      "\n",
      "OMP_WAIT_POLICY=active\n",
      "\n",
      "OMP_NUM_THREADS=1\n",
      "\n",
      "2019-08-30 18:21:09.483851900 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:09.483906700 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:09.483913300 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:09.483917700 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -o 3 -m times -r 200 /mnt/model/test/model.onnx /mnt/model/result/b5a55ba9-4308-4e75-bcec-7ef266766504Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.258109\n",
      "\n",
      "Total iterations:200\n",
      "\n",
      "Average time cost:1.29055 ms\n",
      "\n",
      "2019-08-30 18:21:10.227264500 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:10.227321100 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:10.227343000 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:10.227347200 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.180762\n",
      "\n",
      "Total iterations:200\n",
      "\n",
      "Average time cost:0.903809 ms\n",
      "\n",
      "2019-08-30 18:21:10.878917300 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:10.878971600 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:10.878978300 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:10.878982700 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 0\n",
      "\n",
      "Total time cost:0.14335\n",
      "\n",
      "Total iterations:200\n",
      "\n",
      "Average time cost:0.716752 ms\n",
      "\n",
      "2019-08-30 18:21:11.617281100 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:11.617331200 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:11.617338100 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm1/cell_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "2019-08-30 18:21:11.617342500 [W:onnxruntime:Default, graph.cc:2263 CleanUnusedInitializers] Removing initializer 'lstm2/hidden_init'. It is not used by any node and should be removed from the model.\n",
      "\n",
      "Setting thread pool size to 1\n",
      "\n",
      "Total time cost:0.290206\n",
      "\n",
      "Total iterations:200\n",
      "\n",
      "Average time cost:1.45103 ms\n",
      "\n",
      "\n",
      "\n",
      "cpu_openmp_1_threads_OMP_WAIT_POLICY_active 1.2905449999999998\n",
      "\n",
      "\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -o 3 -m times -r 200 /mnt/model/test/model.onnx /mnt/model/result/dfcafe60-b02e-49cf-b581-5f3200f5921b\n",
      "\n",
      "mkldnn 0.903808\n",
      "\n",
      "\n",
      "\n",
      "OMP_WAIT_POLICY=active\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -e mkldnn -o 3 -m times -r 200 /mnt/model/test/model.onnx /mnt/model/result/ce436aed-7088-4ba8-bc1f-bff59c56e049\n",
      "\n",
      "mkldnn_openmp_OMP_WAIT_POLICY_active 0.7167509999999999\n",
      "\n",
      "\n",
      "\n",
      "/perf_tuning/bin/RelWithDebInfo/all_eps/onnxruntime_perf_test -x 1 -o 3 -m times -r 200 /mnt/model/test/model.onnx /mnt/model/result/bcad2365-69d8-44ae-8e9b-e30fefd23821\n",
      "\n",
      "cpu_1_threads 1.451031\n",
      "\n",
      "\n",
      "\n",
      "Results:\n",
      "\n",
      "mkldnn_openmp_OMP_WAIT_POLICY_active 0.7167509999999999 ms\n",
      "\n",
      "mkldnn_openmp 0.97501 ms\n",
      "\n",
      "mkldnn_openmp_1_threads_OMP_WAIT_POLICY_passive 1.2467949999999999 ms\n",
      "\n",
      "mkldnn 0.903808 ms\n",
      "\n",
      "mkldnn_1_threads 0.75656 ms\n",
      "\n",
      "mkldnn_parallel_1_threads 5.793315000000001 ms\n",
      "\n",
      "mklml_1_threads 1.280607 ms\n",
      "\n",
      "mklml 1.27904 ms\n",
      "\n",
      "mklml_parallel_1_threads 3.0862749999999997 ms\n",
      "\n",
      "cpu_openmp_1_threads_OMP_WAIT_POLICY_active 1.2905449999999998 ms\n",
      "\n",
      "cpu_openmp_1_threads 1.2489949999999999 ms\n",
      "\n",
      "cpu_openmp_1_threads_OMP_WAIT_POLICY_passive 1.3137450000000002 ms\n",
      "\n",
      "cpu_1_threads 1.451031 ms\n",
      "\n",
      "cpu 1.304 ms\n",
      "\n",
      "cpu_parallel_1_threads 3.044285 ms\n",
      "\n",
      "ngraph_parallel_1_threads error\n",
      "\n",
      "ngraph_1_threads error\n",
      "\n",
      "ngraph error\n",
      "\n"
     ]
    }
   ],
   "source": [
    "result = pipeline.perf_tuning(model=model)\n",
    "#result = pipeline.perf_tuning()   # is ok, too"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run parameters\n",
    "\n",
    "(1) model: string\n",
    "\n",
    "    Required. The path of the model that wants to be performed.\n",
    "    \n",
    "(2) result: string\n",
    "\n",
    "    Optional. The path of the result.\n",
    "    \n",
    "(3) config: string (choices=[\"Debug\", \"MinSizeRel\", \"Release\", \"RelWithDebInfo\"])\n",
    "\n",
    "    Optional. Configuration to run. Default is \"RelWithDebInfo\".\n",
    "    \n",
    "(4) test_mode: string (choices=[\"duration\", \"times\"])\n",
    "\n",
    "    Optional. Specifies the test mode. Value could be 'duration' or 'times'. Default is \"times\".\n",
    "\n",
    "(5) execution_provider: string (choices=[\"cpu\", \"cuda\", \"mkldnn\"])\n",
    "\n",
    "    Optional. help=\"Specifies the provider 'cpu','cuda','mkldnn'. Default is ''.\n",
    "    \n",
    "(6) repeated_times: integer\n",
    "\n",
    "    Optional. Specifies the repeated times if running in 'times' test mode. Default:20.\n",
    "    \n",
    "(7) duration_times: integer\n",
    "\n",
    "    Optional. Specifies the seconds to run for 'duration' mode. Default:10.\n",
    "    \n",
    "(8) parallel: boolean\n",
    "\n",
    "    Optional. Use parallel executor, default (without -x): sequential executor.\n",
    "    \n",
    "(9) intra_op_num_threads: integer\n",
    "\n",
    "    Optional. Sets the number of threads used to parallelize the execution within nodes, A value of 0 means ORT will pick a default. Must >=0.\n",
    "    \n",
    "(10) inter_op_num_threads: integer\n",
    "\n",
    "    Optional. Sets the number of threads used to parallelize the execution of the graph (across nodes), A value of 0 means ORT will pick a default. Must >=0.\n",
    "    \n",
    "(11) top_n: integer\n",
    "\n",
    "    Optional. Show percentiles for top n runs. Default:5.\n",
    "    \n",
    "(12) runtime: boolean\n",
    "\n",
    "    Optional. Use this boolean flag to enable GPU if you have one.\n",
    "    \n",
    "(13) input_json: string\n",
    "\n",
    "    Optional. Use JSON file as input parameters.\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mkldnn_openmp_OMP_WAIT_POLICY_active 0.7167509999999999 ms\n",
      "\n",
      "mkldnn_openmp 0.97501 ms\n",
      "\n",
      "mkldnn_openmp_1_threads_OMP_WAIT_POLICY_passive 1.2467949999999999 ms\n",
      "\n",
      "mkldnn 0.903808 ms\n",
      "\n",
      "mkldnn_1_threads 0.75656 ms\n",
      "\n",
      "mkldnn_parallel_1_threads 5.793315000000001 ms\n",
      "\n",
      "mklml_1_threads 1.280607 ms\n",
      "\n",
      "mklml 1.27904 ms\n",
      "\n",
      "mklml_parallel_1_threads 3.0862749999999997 ms\n",
      "\n",
      "cpu_openmp_1_threads_OMP_WAIT_POLICY_active 1.2905449999999998 ms\n",
      "\n",
      "cpu_openmp_1_threads 1.2489949999999999 ms\n",
      "\n",
      "cpu_openmp_1_threads_OMP_WAIT_POLICY_passive 1.3137450000000002 ms\n",
      "\n",
      "cpu_1_threads 1.451031 ms\n",
      "\n",
      "cpu 1.304 ms\n",
      "\n",
      "cpu_parallel_1_threads 3.044285 ms\n",
      "\n",
      "ngraph_parallel_1_threads error\n",
      "\n",
      "ngraph_1_threads error\n",
      "\n",
      "ngraph error\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pipeline.print_performance(result)\n",
    "#pipeline.print_result() # is ok, too"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Result Visualization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After performance test, there would be a directory for results. \n",
    "\n",
    "This libray use Pandas.read_json to visualize JSON file. (orient is changeable.)\n",
    "\n",
    "\"latency.json\" contains the raw data of results ordered by the average time. \n",
    "\n",
    "Use .latency to obtain the original latency JSON; Use .profiling to obtain original top 5 profiling JSON."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "E:\\onnx-pipeline\\notebook/model/result\n"
     ]
    }
   ],
   "source": [
    "r = pipeline.get_result(result)\n",
    "#r.latency\n",
    "#r.profiling\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Print latency.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Provide parameters for top 5 performace. Use the parameter \"top\" to visualize more results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>avg</th>\n",
       "      <th>p90</th>\n",
       "      <th>p95</th>\n",
       "      <th>cpu_usage</th>\n",
       "      <th>gpu_usage</th>\n",
       "      <th>memory_util</th>\n",
       "      <th>code_snippet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>mkldnn_openmp_OMP_WAIT_POLICY_active</td>\n",
       "      <td>0.716751</td>\n",
       "      <td>0.8413</td>\n",
       "      <td>0.9128</td>\n",
       "      <td>0.94217</td>\n",
       "      <td>0</td>\n",
       "      <td>0.28349</td>\n",
       "      <td>OrderedDict([('execution_provider', 'mkldnn_op...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>mkldnn</td>\n",
       "      <td>0.903808</td>\n",
       "      <td>1.2916</td>\n",
       "      <td>1.4089</td>\n",
       "      <td>0.97900</td>\n",
       "      <td>0</td>\n",
       "      <td>0.28289</td>\n",
       "      <td>OrderedDict([('execution_provider', 'mkldnn'),...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>mklml_1_threads</td>\n",
       "      <td>1.280607</td>\n",
       "      <td>1.4607</td>\n",
       "      <td>1.6955</td>\n",
       "      <td>0.90200</td>\n",
       "      <td>0</td>\n",
       "      <td>0.27916</td>\n",
       "      <td>OrderedDict([('execution_provider', 'mklml'), ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cpu_openmp_1_threads_OMP_WAIT_POLICY_active</td>\n",
       "      <td>1.290545</td>\n",
       "      <td>1.4693</td>\n",
       "      <td>1.6791</td>\n",
       "      <td>0.95833</td>\n",
       "      <td>0</td>\n",
       "      <td>0.28105</td>\n",
       "      <td>OrderedDict([('execution_provider', 'cpu_openm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cpu_1_threads</td>\n",
       "      <td>1.451031</td>\n",
       "      <td>2.0321</td>\n",
       "      <td>2.0950</td>\n",
       "      <td>0.89473</td>\n",
       "      <td>0</td>\n",
       "      <td>0.28135</td>\n",
       "      <td>OrderedDict([('execution_provider', 'cpu'), ('...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          name       avg     p90     p95  \\\n",
       "0         mkldnn_openmp_OMP_WAIT_POLICY_active  0.716751  0.8413  0.9128   \n",
       "1                                       mkldnn  0.903808  1.2916  1.4089   \n",
       "2                              mklml_1_threads  1.280607  1.4607  1.6955   \n",
       "3  cpu_openmp_1_threads_OMP_WAIT_POLICY_active  1.290545  1.4693  1.6791   \n",
       "4                                cpu_1_threads  1.451031  2.0321  2.0950   \n",
       "\n",
       "   cpu_usage  gpu_usage  memory_util  \\\n",
       "0    0.94217          0      0.28349   \n",
       "1    0.97900          0      0.28289   \n",
       "2    0.90200          0      0.27916   \n",
       "3    0.95833          0      0.28105   \n",
       "4    0.89473          0      0.28135   \n",
       "\n",
       "                                        code_snippet  \n",
       "0  OrderedDict([('execution_provider', 'mkldnn_op...  \n",
       "1  OrderedDict([('execution_provider', 'mkldnn'),...  \n",
       "2  OrderedDict([('execution_provider', 'mklml'), ...  \n",
       "3  OrderedDict([('execution_provider', 'cpu_openm...  \n",
       "4  OrderedDict([('execution_provider', 'cpu'), ('...  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.prints()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Print profiling.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Only provide profiling JSON for top 5 performace by giving certain index of the result. The file name is profile_[name].json\n",
    "(1) index: integer\n",
    "    Required. The index for top 5 profiling files.\n",
    "(2) top: integer\n",
    "    The number for top Ops.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat</th>\n",
       "      <th>pid</th>\n",
       "      <th>tid</th>\n",
       "      <th>dur</th>\n",
       "      <th>ts</th>\n",
       "      <th>ph</th>\n",
       "      <th>name</th>\n",
       "      <th>args</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Node</td>\n",
       "      <td>12569</td>\n",
       "      <td>12569</td>\n",
       "      <td>1404</td>\n",
       "      <td>34769</td>\n",
       "      <td>X</td>\n",
       "      <td></td>\n",
       "      <td>{'provider': 'CPUExecutionProvider', 'op_name'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Node</td>\n",
       "      <td>12569</td>\n",
       "      <td>12569</td>\n",
       "      <td>272</td>\n",
       "      <td>33080</td>\n",
       "      <td>X</td>\n",
       "      <td>distill_conv1_1_nchwc_9</td>\n",
       "      <td>{'provider': 'CPUExecutionProvider', 'op_name'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Node</td>\n",
       "      <td>12569</td>\n",
       "      <td>12569</td>\n",
       "      <td>230</td>\n",
       "      <td>40891</td>\n",
       "      <td>X</td>\n",
       "      <td>distill_conv2/bn_1_nchwc_14</td>\n",
       "      <td>{'provider': 'CPUExecutionProvider', 'op_name'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Node</td>\n",
       "      <td>12569</td>\n",
       "      <td>12569</td>\n",
       "      <td>215</td>\n",
       "      <td>56671</td>\n",
       "      <td>X</td>\n",
       "      <td>distill_conv2/bn_1_nchwc_14</td>\n",
       "      <td>{'provider': 'CPUExecutionProvider', 'op_name'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Node</td>\n",
       "      <td>12569</td>\n",
       "      <td>12569</td>\n",
       "      <td>205</td>\n",
       "      <td>49507</td>\n",
       "      <td>X</td>\n",
       "      <td>distill_conv2/bn_1_nchwc_14</td>\n",
       "      <td>{'provider': 'CPUExecutionProvider', 'op_name'...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    cat    pid    tid   dur     ts ph                         name  \\\n",
       "0  Node  12569  12569  1404  34769  X                                \n",
       "1  Node  12569  12569   272  33080  X      distill_conv1_1_nchwc_9   \n",
       "2  Node  12569  12569   230  40891  X  distill_conv2/bn_1_nchwc_14   \n",
       "3  Node  12569  12569   215  56671  X  distill_conv2/bn_1_nchwc_14   \n",
       "4  Node  12569  12569   205  49507  X  distill_conv2/bn_1_nchwc_14   \n",
       "\n",
       "                                                args  \n",
       "0  {'provider': 'CPUExecutionProvider', 'op_name'...  \n",
       "1  {'provider': 'CPUExecutionProvider', 'op_name'...  \n",
       "2  {'provider': 'CPUExecutionProvider', 'op_name'...  \n",
       "3  {'provider': 'CPUExecutionProvider', 'op_name'...  \n",
       "4  {'provider': 'CPUExecutionProvider', 'op_name'...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.print_profiling(index=4, top=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get code snippets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "import onnxruntime as ort\n",
      "so = rt.SessionOptions()\n",
      "so.set_graph_optimization_level(99)\n",
      "so.enable_sequential_execution = False\n",
      "so.session_thread_pool_size(0)\n",
      "session = rt.Session(\"/mnt/model/test/model.onnx\", so)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(r.get_code(ep='cpu'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# netron"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Serving 'model/test/model.onnx' at http://localhost:8080\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"1000\"\n",
       "            src=\"http://localhost:8080\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x1e5d8f7d978>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# only workable for notebook in the local server \n",
    "import netron\n",
    "netron.start(\"model/test/model.onnx\", browse=False) # 'model.onnx'\n",
    "from IPython.display import IFrame\n",
    "IFrame('http://localhost:8080', width=\"100%\", height=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Stopping http://localhost:8080\n"
     ]
    }
   ],
   "source": [
    "netron.stop()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}