From cfbe5faffdbcd590debac06581ae2c56d1357c81 Mon Sep 17 00:00:00 2001 From: Aaron Markham Date: Wed, 15 Aug 2018 13:55:57 -0700 Subject: [PATCH 1/9] fixing Python 3 print issue --- docs/mxdoc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/mxdoc.py b/docs/mxdoc.py index dabfd3bbb644..1ca19868735c 100644 --- a/docs/mxdoc.py +++ b/docs/mxdoc.py @@ -41,8 +41,7 @@ for section in [ _DOC_SET ]: print("Document sets to generate:") for candidate in [ 'scala_docs', 'clojure_docs', 'doxygen_docs', 'r_docs' ]: - print '%-12s : %s' % (candidate, parser.get(section, candidate)) - print + print('%-12s : %s' % (candidate, parser.get(section, candidate))) _MXNET_DOCS_BUILD_MXNET = parser.getboolean('mxnet', 'build_mxnet') _SCALA_DOCS = parser.getboolean(_DOC_SET, 'scala_docs') From 607c3a2b9df7f6700d6743103c534a92e5eac67c Mon Sep 17 00:00:00 2001 From: Aaron Markham Date: Wed, 15 Aug 2018 13:56:41 -0700 Subject: [PATCH 2/9] test setup Jenkinsfile --- docs/Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/Jenkinsfile b/docs/Jenkinsfile index 9c7b857615b5..41e73653079e 100644 --- a/docs/Jenkinsfile +++ b/docs/Jenkinsfile @@ -23,12 +23,12 @@ // timeout in minutes max_time = 60 -node('restricted-mxnetlinux-cpu') { +node('mxnetlinux-cpu') { // Loading the utilities requires a node context unfortunately checkout scm utils = load('ci/Jenkinsfile_utils.groovy') } -utils.assign_node_labels(linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu') +utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu') utils.main_wrapper( core_logic: { @@ -39,7 +39,7 @@ core_logic: { timeout(time: max_time, unit: 'MINUTES') { sh "ci/build.py -p ubuntu_cpu --docker-registry ${env.DOCKER_CACHE_REGISTRY} --docker-build-retries 3 /work/runtime_functions.sh build_docs ${params.tags_to_build} ${params.tag_list} ${params.fork} ${params.tag_default} ${params.domain}" archiveArtifacts 'docs/build_version_doc/artifacts.tgz' - build 'restricted-website-publish' + build 'test-website-publish' } } } From f2f47f83498975249803779b046f86ae8de4fe35 Mon Sep 17 00:00:00 2001 From: Aaron Markham Date: Wed, 15 Aug 2018 15:09:50 -0700 Subject: [PATCH 3/9] moved python reqs for docs to ci zone; added six; using six in for sphinx python 3 compatibility --- ci/docker/install/ubuntu_docs.sh | 12 ++---------- .../requirements.txt => ci/docs_requirements | 15 ++++++++------- docs/build_version_doc/setup_docs_ubuntu.sh | 4 ++-- docs/mxdoc.py | 8 ++++---- 4 files changed, 16 insertions(+), 23 deletions(-) rename docs/build_version_doc/requirements.txt => ci/docs_requirements (51%) diff --git a/ci/docker/install/ubuntu_docs.sh b/ci/docker/install/ubuntu_docs.sh index ee121962ee08..ec8a285ba0b1 100755 --- a/ci/docker/install/ubuntu_docs.sh +++ b/ci/docker/install/ubuntu_docs.sh @@ -27,15 +27,7 @@ apt-get install -y \ doxygen \ pandoc -echo 'Installing python packages...' -pip install --upgrade pip && pip install \ - beautifulsoup4 \ - breathe \ - CommonMark==0.5.4 \ - h5py \ - mock==1.0.1 \ - pypandoc \ - recommonmark==0.4.0 \ - sphinx==1.5.6 +pip3 install -r ../../docs_requirements +pip2 install -r ../../docs_requirements echo 'Dependency installation complete.' diff --git a/docs/build_version_doc/requirements.txt b/ci/docs_requirements similarity index 51% rename from docs/build_version_doc/requirements.txt rename to ci/docs_requirements index 4f3f4d065c9b..7407223b3eed 100644 --- a/docs/build_version_doc/requirements.txt +++ b/ci/docs_requirements @@ -1,15 +1,16 @@ -beautifulsoup4 -breathe +beautifulsoup4==4.6.3 +breathe==4.10.0 cpplint==1.3.0 CommonMark==0.5.4 h5py==2.8.0rc1 -mock==1.0.1 -nose -nose-timer +mock==2.0.0 +nose==1.3.7 +nose-timer==0.7.3 numpy<1.15.0,>=1.8.2 pylint==1.8.3 -pypandoc -recommonmark==0.4.0 +pypandoc==1.4 +recommonmark==0.4.0 requests<2.19.0,>=2.18.4 scipy==1.0.1 +six==1.11.0 sphinx==1.5.6 diff --git a/docs/build_version_doc/setup_docs_ubuntu.sh b/docs/build_version_doc/setup_docs_ubuntu.sh index d00f00c502fe..7105ee561759 100755 --- a/docs/build_version_doc/setup_docs_ubuntu.sh +++ b/docs/build_version_doc/setup_docs_ubuntu.sh @@ -110,10 +110,10 @@ sudo apt-get update && sudo apt-get install -y \ wget -nv https://bootstrap.pypa.io/get-pip.py echo "Installing for Python 3..." sudo python3 get-pip.py -pip3 install --user -r requirements.txt +pip3 install --user -r ../../ci/docs_requirements echo "Installing for Python 2..." sudo python2 get-pip.py -pip2 install --user -r requirements.txt +pip2 install --user -r ../../ci/docs_requirements cd ../../ diff --git a/docs/mxdoc.py b/docs/mxdoc.py index 1ca19868735c..33f64750e816 100644 --- a/docs/mxdoc.py +++ b/docs/mxdoc.py @@ -23,16 +23,16 @@ import sys from recommonmark import transform import pypandoc -# import StringIO from io for python3 compatibility -from io import StringIO import contextlib -from ConfigParser import SafeConfigParser +# Use six for Python 2 / 3 compatibility +from six import StringIO +from six.moves import configparser _BUILD_VER = os.getenv('BUILD_VER', 'default') print("Building version {}".format(_BUILD_VER)) _DOC_SET = 'document_sets_' + _BUILD_VER -parser = SafeConfigParser() +parser = configparser.SafeConfigParser() parser.read('settings.ini') if _DOC_SET not in parser.sections(): From 6890344449c73f28c4a8eaa59bf2942e8efd0e2b Mon Sep 17 00:00:00 2001 From: Aaron Markham Date: Wed, 15 Aug 2018 15:34:34 -0700 Subject: [PATCH 4/9] add dev Jenkinsfile to not spam alert email; path fix --- ci/docker/install/ubuntu_docs.sh | 4 +-- docs/Jenkinsfile-dev | 54 ++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 docs/Jenkinsfile-dev diff --git a/ci/docker/install/ubuntu_docs.sh b/ci/docker/install/ubuntu_docs.sh index ec8a285ba0b1..bc406fccc5bb 100755 --- a/ci/docker/install/ubuntu_docs.sh +++ b/ci/docker/install/ubuntu_docs.sh @@ -27,7 +27,7 @@ apt-get install -y \ doxygen \ pandoc -pip3 install -r ../../docs_requirements -pip2 install -r ../../docs_requirements +pip3 install -r ../docs_requirements +pip2 install -r ../docs_requirements echo 'Dependency installation complete.' diff --git a/docs/Jenkinsfile-dev b/docs/Jenkinsfile-dev new file mode 100644 index 000000000000..b93eba6489d0 --- /dev/null +++ b/docs/Jenkinsfile-dev @@ -0,0 +1,54 @@ +// -*- mode: groovy -*- + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// timeout in minutes +max_time = 60 + +node('mxnetlinux-cpu') { + // Loading the utilities requires a node context unfortunately + checkout scm + utils = load('ci/Jenkinsfile_utils.groovy') +} +utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu') + +utils.main_wrapper( +core_logic: { + stage('Build Docs') { + node(NODE_LINUX_CPU) { + ws('workspace/docs') { + utils.init_git() + timeout(time: max_time, unit: 'MINUTES') { + sh "ci/build.py -p ubuntu_cpu --docker-registry ${env.DOCKER_CACHE_REGISTRY} --docker-build-retries 3 /work/runtime_functions.sh build_docs ${params.tags_to_build} ${params.tag_list} ${params.fork} ${params.tag_default} ${params.domain}" + archiveArtifacts 'docs/build_version_doc/artifacts.tgz' + build 'test-website-publish' + } + } + } + } +} +, +failure_handler: { + if (currentBuild.result == "FAILURE") { + // Do nothing. + } +} +) From 719c09ac05f6374dbac8e51a1b9876bdadc71fd5 Mon Sep 17 00:00:00 2001 From: Aaron Markham Date: Wed, 15 Aug 2018 15:54:06 -0700 Subject: [PATCH 5/9] copy requirement to docker image --- ci/docker/Dockerfile.build.ubuntu_cpu | 1 + ci/docker/Dockerfile.build.ubuntu_gpu | 1 + ci/docker/Dockerfile.build.ubuntu_nightly_cpu | 1 + ci/docker/Dockerfile.build.ubuntu_nightly_gpu | 1 + ci/docker/install/ubuntu_docs.sh | 4 ++-- 5 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu index 76df3dfc1c1a..d93c161c4294 100644 --- a/ci/docker/Dockerfile.build.ubuntu_cpu +++ b/ci/docker/Dockerfile.build.ubuntu_cpu @@ -61,6 +61,7 @@ COPY install/ubuntu_onnx.sh /work/ RUN /work/ubuntu_onnx.sh COPY install/ubuntu_docs.sh /work/ +COPY docs_requirements /work/ RUN /work/ubuntu_docs.sh ARG USER_ID=0 diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu index 8b4cd1e5a3f1..b4ed120379e5 100644 --- a/ci/docker/Dockerfile.build.ubuntu_gpu +++ b/ci/docker/Dockerfile.build.ubuntu_gpu @@ -64,6 +64,7 @@ COPY install/ubuntu_onnx.sh /work/ RUN /work/ubuntu_onnx.sh COPY install/ubuntu_docs.sh /work/ +COPY docs_requirements /work/ RUN /work/ubuntu_docs.sh COPY install/ubuntu_tutorials.sh /work/ diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu b/ci/docker/Dockerfile.build.ubuntu_nightly_cpu index c803188263f2..8b945e7f2e2b 100644 --- a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu +++ b/ci/docker/Dockerfile.build.ubuntu_nightly_cpu @@ -58,6 +58,7 @@ COPY install/ubuntu_onnx.sh /work/ RUN /work/ubuntu_onnx.sh COPY install/ubuntu_docs.sh /work/ +COPY docs_requirements /work/ RUN /work/ubuntu_docs.sh COPY install/ubuntu_nightly_tests.sh /work/ diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu index c0e31e2c15ef..4f323b4f2eb4 100644 --- a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu +++ b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu @@ -64,6 +64,7 @@ COPY install/ubuntu_onnx.sh /work/ RUN /work/ubuntu_onnx.sh COPY install/ubuntu_docs.sh /work/ +COPY docs_requirements /work/ RUN /work/ubuntu_docs.sh COPY install/ubuntu_tutorials.sh /work/ diff --git a/ci/docker/install/ubuntu_docs.sh b/ci/docker/install/ubuntu_docs.sh index bc406fccc5bb..83616d0fe760 100755 --- a/ci/docker/install/ubuntu_docs.sh +++ b/ci/docker/install/ubuntu_docs.sh @@ -27,7 +27,7 @@ apt-get install -y \ doxygen \ pandoc -pip3 install -r ../docs_requirements -pip2 install -r ../docs_requirements +pip3 install -r docs_requirements +pip2 install -r docs_requirements echo 'Dependency installation complete.' From 661aa7e831090b186c96df132b59f22882a7f19a Mon Sep 17 00:00:00 2001 From: Aaron Markham Date: Wed, 15 Aug 2018 16:21:39 -0700 Subject: [PATCH 6/9] making docs_requirements available to docker --- ci/docker/Dockerfile.build.ubuntu_cpu | 2 +- ci/docker/Dockerfile.build.ubuntu_gpu | 2 +- ci/docker/Dockerfile.build.ubuntu_nightly_cpu | 2 +- ci/docker/Dockerfile.build.ubuntu_nightly_gpu | 2 +- ci/{ => docker/install}/docs_requirements | 0 5 files changed, 4 insertions(+), 4 deletions(-) rename ci/{ => docker/install}/docs_requirements (100%) diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu index d93c161c4294..08fb04df03e2 100644 --- a/ci/docker/Dockerfile.build.ubuntu_cpu +++ b/ci/docker/Dockerfile.build.ubuntu_cpu @@ -61,7 +61,7 @@ COPY install/ubuntu_onnx.sh /work/ RUN /work/ubuntu_onnx.sh COPY install/ubuntu_docs.sh /work/ -COPY docs_requirements /work/ +COPY install/docs_requirements /work/ RUN /work/ubuntu_docs.sh ARG USER_ID=0 diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu index b4ed120379e5..d99dafb0bd0c 100644 --- a/ci/docker/Dockerfile.build.ubuntu_gpu +++ b/ci/docker/Dockerfile.build.ubuntu_gpu @@ -64,7 +64,7 @@ COPY install/ubuntu_onnx.sh /work/ RUN /work/ubuntu_onnx.sh COPY install/ubuntu_docs.sh /work/ -COPY docs_requirements /work/ +COPY install/docs_requirements /work/ RUN /work/ubuntu_docs.sh COPY install/ubuntu_tutorials.sh /work/ diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu b/ci/docker/Dockerfile.build.ubuntu_nightly_cpu index 8b945e7f2e2b..834710c4ceb6 100644 --- a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu +++ b/ci/docker/Dockerfile.build.ubuntu_nightly_cpu @@ -58,7 +58,7 @@ COPY install/ubuntu_onnx.sh /work/ RUN /work/ubuntu_onnx.sh COPY install/ubuntu_docs.sh /work/ -COPY docs_requirements /work/ +COPY install/docs_requirements /work/ RUN /work/ubuntu_docs.sh COPY install/ubuntu_nightly_tests.sh /work/ diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu index 4f323b4f2eb4..fb34307063a0 100644 --- a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu +++ b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu @@ -64,7 +64,7 @@ COPY install/ubuntu_onnx.sh /work/ RUN /work/ubuntu_onnx.sh COPY install/ubuntu_docs.sh /work/ -COPY docs_requirements /work/ +COPY install/docs_requirements /work/ RUN /work/ubuntu_docs.sh COPY install/ubuntu_tutorials.sh /work/ diff --git a/ci/docs_requirements b/ci/docker/install/docs_requirements similarity index 100% rename from ci/docs_requirements rename to ci/docker/install/docs_requirements From 6fa7d940049eadc8f8d26552dfbb461a5704bcd0 Mon Sep 17 00:00:00 2001 From: Aaron Markham Date: Wed, 15 Aug 2018 16:32:00 -0700 Subject: [PATCH 7/9] full path needed --- ci/docker/install/ubuntu_docs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/docker/install/ubuntu_docs.sh b/ci/docker/install/ubuntu_docs.sh index 83616d0fe760..a709b3de7843 100755 --- a/ci/docker/install/ubuntu_docs.sh +++ b/ci/docker/install/ubuntu_docs.sh @@ -27,7 +27,7 @@ apt-get install -y \ doxygen \ pandoc -pip3 install -r docs_requirements -pip2 install -r docs_requirements +pip3 install -r /work/docs_requirements +pip2 install -r /work/docs_requirements echo 'Dependency installation complete.' From c25d73b3cd29679c5c640ef32cb5865b0c07688c Mon Sep 17 00:00:00 2001 From: Aaron Markham Date: Wed, 15 Aug 2018 17:24:44 -0700 Subject: [PATCH 8/9] reverting to production settings --- docs/Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/Jenkinsfile b/docs/Jenkinsfile index 41e73653079e..9c7b857615b5 100644 --- a/docs/Jenkinsfile +++ b/docs/Jenkinsfile @@ -23,12 +23,12 @@ // timeout in minutes max_time = 60 -node('mxnetlinux-cpu') { +node('restricted-mxnetlinux-cpu') { // Loading the utilities requires a node context unfortunately checkout scm utils = load('ci/Jenkinsfile_utils.groovy') } -utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu') +utils.assign_node_labels(linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu') utils.main_wrapper( core_logic: { @@ -39,7 +39,7 @@ core_logic: { timeout(time: max_time, unit: 'MINUTES') { sh "ci/build.py -p ubuntu_cpu --docker-registry ${env.DOCKER_CACHE_REGISTRY} --docker-build-retries 3 /work/runtime_functions.sh build_docs ${params.tags_to_build} ${params.tag_list} ${params.fork} ${params.tag_default} ${params.domain}" archiveArtifacts 'docs/build_version_doc/artifacts.tgz' - build 'test-website-publish' + build 'restricted-website-publish' } } } From 52854768d6424a81b3de497c276722672f1bc534 Mon Sep 17 00:00:00 2001 From: Aaron Markham Date: Wed, 15 Aug 2018 17:31:19 -0700 Subject: [PATCH 9/9] path fix to requirements --- docs/build_version_doc/setup_docs_ubuntu.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/build_version_doc/setup_docs_ubuntu.sh b/docs/build_version_doc/setup_docs_ubuntu.sh index 7105ee561759..22b2fe2c2f71 100755 --- a/docs/build_version_doc/setup_docs_ubuntu.sh +++ b/docs/build_version_doc/setup_docs_ubuntu.sh @@ -110,10 +110,10 @@ sudo apt-get update && sudo apt-get install -y \ wget -nv https://bootstrap.pypa.io/get-pip.py echo "Installing for Python 3..." sudo python3 get-pip.py -pip3 install --user -r ../../ci/docs_requirements +pip3 install --user -r ../../ci/docker/install/docs_requirements echo "Installing for Python 2..." sudo python2 get-pip.py -pip2 install --user -r ../../ci/docs_requirements +pip2 install --user -r ../../ci/docker/install/docs_requirements cd ../../