Merge branch 'master' of https://github.com/amoldavsky/seldon-core in…

…to assaf-java-s2i-update
SeldonIO · Jul 21, 2020 · 74099a4 · 74099a4
2 parents 3b829dc + 6e9780f
commit 74099a4
Show file tree

Hide file tree

Showing 21 changed files with 195 additions and 50 deletions.
diff --git a/ci/add-pr-build-comment b/ci/add-pr-build-comment
@@ -18,11 +18,12 @@ BUILD_NUMBER=${BUILD_NUMBER}
 PIPELINE_CONTEXT=${PIPELINE_CONTEXT}
 
 COMMENT_MSG=$(cat <<EOF
-$(date)
-The logs for [${PIPELINE_CONTEXT}] [${BUILD_NUMBER}] will show after the pipeline context has finished.  
+## STARTING TEST [ ${PIPELINE_CONTEXT} # ${BUILD_NUMBER} ]
+
+Logs will be available when test completes in the following link:
 https://github.com/${REPO_OWNER}/${REPO_NAME}/blob/gh-pages/jenkins-x/logs/${REPO_OWNER}/${REPO_NAME}/PR-${PULL_NUMBER}/${BUILD_NUMBER}.log
 
-impatient try  
+Impatient try:
 jx get build logs ${REPO_OWNER}/${REPO_NAME}/PR-${PULL_NUMBER} --build=${BUILD_NUMBER}
 EOF
 )

diff --git a/ci/add-pr-build-comment-success b/ci/add-pr-build-comment-success
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+#
+# add-pr-build-comment
+#
+set -o nounset
+set -o errexit
+set -o pipefail
+set -o noclobber
+set -o noglob
+set -o xtrace
+
+STARTUP_DIR="$( cd "$( dirname "$0" )" && pwd )"
+
+REPO_OWNER=${REPO_OWNER}
+REPO_NAME=${REPO_NAME}
+PULL_NUMBER=${PULL_NUMBER}
+BUILD_NUMBER=${BUILD_NUMBER}
+PIPELINE_CONTEXT=${PIPELINE_CONTEXT}
+
+COMMENT_MSG=$(cat <<EOF
+## SUCCESS [${PIPELINE_CONTEXT} # ${BUILD_NUMBER}] TEST PASSED ✅
+
+Well done! 😎
+EOF
+)
+
+jx step pr comment --owner=${REPO_OWNER} --repository=${REPO_NAME} --pull-request=${PULL_NUMBER} --comment="${COMMENT_MSG}"
+
diff --git a/core-builder/.gitignore b/core-builder/.gitignore
@@ -1 +1,3 @@
 dev_requirements.txt
+requirements.txt
+requirements-dev.txt
diff --git a/core-builder/Dockerfile b/core-builder/Dockerfile
@@ -64,12 +64,13 @@ RUN apt-get upgrade && \
         apt-get install zlib1g-dev -y && \
         apt-get install libssl-dev -y && \
 	apt-get install libbz2-dev -y && \
+	apt-get install libsqlite3-dev && \
 	apt-get install liblzma-dev -y
 
 RUN curl -SLO https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz \
     && tar xvf Python-${PYTHON_VERSION}.tgz \
     && cd Python-${PYTHON_VERSION} \
-    && ./configure --prefix=/usr/local --with-openssl \
+    && ./configure --prefix=/usr/local --with-openssl --enable-loadable-sqlite-extensions \
     && make \
     && make altinstall \
     && cd / \
@@ -88,12 +89,31 @@ RUN pip install --upgrade setuptools
 RUN pip install pyyaml
 RUN pip install grpcio
 RUN pip install grpcio-tools
+RUN pip install flatbuffers
+RUN pip install twine
+RUN pip install mypy-protobuf
+RUN pip install pysqlite3
+RUN pip install db-sqlite3
+
+# Install flatc
+RUN apt-get install cmake -y
+RUN INSTALL_DIR=/tmp/flatc-install && \
+    mkdir $INSTALL_DIR && \
+    cd $INSTALL_DIR && \
+    git clone https://github.com/google/flatbuffers.git && \
+    cd flatbuffers && \
+    cmake -G "Unix Makefiles" && \
+    make && \
+    cp ./flatc /usr/local/bin/ && \
+    rm -rf $INSTALL_DIR
+
 RUN apt-get remove -y --auto-remove && apt-get clean -y && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # INSTALL GO
+ENV GO_VERSION=1.14.6
 ENV PATH /usr/local/go/bin:$PATH
-RUN wget https://dl.google.com/go/go1.13.8.linux-amd64.tar.gz && \
-        tar -zxvf go1.13.8.linux-amd64.tar.gz && \
+RUN wget "https://dl.google.com/go/go$GO_VERSION.linux-amd64.tar.gz" && \
+        tar -zxvf "go$GO_VERSION.linux-amd64.tar.gz" && \
         mv go/ /usr/local/go
 RUN curl -sL https://go.kubebuilder.io/dl/2.3.0/linux/amd64 | tar -xz -C /tmp/ && \
         mv /tmp/kubebuilder_2.3.0_linux_amd64 /usr/local/kubebuilder/
@@ -115,18 +135,29 @@ RUN \
 # install jupyter
 RUN pip install jupyter
 
+# Ginkgo
+RUN go get github.com/onsi/ginkgo/ginkgo
+
 # Install grpcurl
-RUN go get github.com/fullstorydev/grpcurl && \
-    go install github.com/fullstorydev/grpcurl/cmd/grpcurl
+RUN wget https://github.com/fullstorydev/grpcurl/releases/download/v1.6.1/grpcurl_1.6.1_linux_x86_64.tar.gz && \
+        tar -zxvf grpcurl_1.6.1_linux_x86_64.tar.gz && \
+        chmod +x grpcurl && \
+        mv grpcurl /usr/local/bin/grpcurl
 
 ENV PATH="${PATH}:/root/go/bin"
 
 # Notebooks python requirements
-COPY dev_requirements.txt /tmp/requirements.txt
-RUN pip install -r /tmp/requirements.txt
-
-# Ginkgo
-RUN go get -u github.com/onsi/ginkgo/ginkgo
+COPY dev_requirements.txt /tmp/dev_requirements.txt
+RUN pip install -r /tmp/dev_requirements.txt
+# General python requirements
+COPY requirements.txt /tmp/requirements.txt
+COPY requirements-dev.txt /tmp/requirements-dev.txt
+RUN cd /tmp && pip install -r requirements-dev.txt
+
+# Install Jenkins X CLI
+RUN curl -L "https://github.com/jenkins-x/jx/releases/download/v2.1.95/jx-darwin-amd64.tar.gz" | tar xzv "jx" && \
+        chmod +x ./jx && \
+        mv ./jx /usr/bin/jx
 
 WORKDIR /work
 

diff --git a/core-builder/Makefile b/core-builder/Makefile
@@ -1,7 +1,9 @@
 DOCKER_IMAGE_NAME=seldonio/core-builder
-DOCKER_IMAGE_VERSION=0.15
+DOCKER_IMAGE_VERSION=0.16
 
 build_docker_image:
+	cp ../python/requirements.txt .
+	cp ../python/requirements-dev.txt .
 	cp ../testing/scripts/dev_requirements.txt .
 	docker build --force-rm=true -t $(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION) .
 push_to_registry:

diff --git a/doc/source/servers/mlflow.md b/doc/source/servers/mlflow.md
@@ -52,6 +52,32 @@ spec:
       replicas: 1
 ```
 
+## MLFlow xtype
+
+By default the server will call your loaded model's predict function with a `numpy.ndarray`. If you wish for it to call it with `pandas.DataFrame` instead, you can pass a parameter `xtype` and set it to `DataFrame`. For example:   
+
+```yaml
+apiVersion: machinelearning.seldon.io/v1alpha2
+kind: SeldonDeployment
+metadata:
+  name: mlflow
+spec:
+  name: wines
+  predictors:
+    - graph:
+        children: []
+        implementation: MLFLOW_SERVER
+        modelUri: gs://seldon-models/mlflow/elasticnet_wine
+        name: classifier
+      parameters:
+        - name: xtype
+          type: STRING
+          value: DataFrame
+      name: default
+      replicas: 1
+```
+
+```
 You can also try out a [worked
 notebook](../examples/server_examples.html#Serve-MLflow-Elasticnet-Wines-Model)
 or check our [talk at the Spark + AI Summit

diff --git a/engine/pom.xml b/engine/pom.xml
@@ -305,7 +305,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-lang3</artifactId>
-      <version>3.10</version>
+      <version>3.11</version>
     </dependency>
     <dependency>
       <groupId>org.springframework.boot</groupId>

diff --git a/executor/api/rest/server.go b/executor/api/rest/server.go
@@ -22,6 +22,7 @@ import (
 	"github.com/seldonio/seldon-core/executor/predictor"
 	v1 "github.com/seldonio/seldon-core/operator/apis/machinelearning.seldon.io/v1"
 	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
+	"time"
 )
 
 type SeldonRestApi struct {
@@ -67,8 +68,9 @@ func (r *SeldonRestApi) CreateHttpServer(port int) *http.Server {
 	// will apply no timeout at all. Instead, we control that through the
 	// http.Client instance making requests to the underlying node graph servers.
 	return &http.Server{
-		Handler: r.Router,
-		Addr:    address,
+		Handler:     r.Router,
+		Addr:        address,
+		IdleTimeout: 65 * time.Second,
 	}
 }
 

diff --git a/jenkins-x-engine.yml b/jenkins-x-engine.yml
@@ -4,7 +4,7 @@ pipelineConfig:
     pullRequest:
       pipeline:
         agent:
-          image: seldonio/core-builder:0.15
+          image: seldonio/core-builder:0.16
         stages:
           - name: pr-build-comment
             steps:
@@ -64,3 +64,9 @@ pipelineConfig:
                     type: Directory
                 - name: dind-storage
                   emptyDir: {}
+          - name: pr-build-comment-success
+            steps:
+              - agent:
+                  image: gcr.io/jenkinsxio/builder-go:2.0.916-264
+                dir: ci
+                sh: "./add-pr-build-comment-success"
diff --git a/jenkins-x-integration.yml b/jenkins-x-integration.yml
@@ -4,7 +4,7 @@ pipelineConfig:
     pullRequest:
       pipeline:
         agent:
-          image: seldonio/core-builder:0.15
+          image: seldonio/core-builder:0.16
         stages:
           - name: pr-build-comment
             steps:
@@ -54,3 +54,9 @@ pipelineConfig:
                     type: Directory
                 - name: dind-storage
                   emptyDir: {}
+          - name: pr-build-comment-success
+            steps:
+              - agent:
+                  image: gcr.io/jenkinsxio/builder-go:2.0.916-264
+                dir: ci
+                sh: "./add-pr-build-comment-success"
diff --git a/jenkins-x-lint.yml b/jenkins-x-lint.yml
@@ -4,7 +4,7 @@ pipelineConfig:
     pullRequest:
       pipeline:
         agent:
-          image: seldonio/core-builder:0.15
+          image: seldonio/core-builder:0.16
         stages:
         - name: pr-build-comment
           steps:
@@ -15,8 +15,6 @@ pipelineConfig:
         - name: lint-checks
           parallel:
           - name: lint-python
-            agent:
-              image: seldonio/python-builder:0.2
             steps:
               - name: lint-python
                 command: make
@@ -25,20 +23,22 @@ pipelineConfig:
                 - install_dev
                 - lint 
           - name: lint-operator
-            agent:
-              image: seldonio/core-builder:0.15
             steps:
               - name: lint-operator
                 command: make
                 args:
                   - -C operator
                   - lint
           - name: lint-executor
-            agent:
-              image: seldonio/core-builder:0.15
             steps:
               - name: lint-executor
                 command: make
                 args:
                   - -C executor
                   - lint
+        - name: pr-build-comment-success
+          steps:
+          - agent:
+              image: gcr.io/jenkinsxio/builder-go:2.0.916-264
+            dir: ci
+            sh: "./add-pr-build-comment-success"
diff --git a/jenkins-x-notebooks.yml b/jenkins-x-notebooks.yml
@@ -4,7 +4,7 @@ pipelineConfig:
     pullRequest:
       pipeline:
         agent:
-          image: seldonio/core-builder:0.15
+          image: seldonio/core-builder:0.16
         stages:
           - name: pr-build-comment
             steps:
@@ -54,3 +54,9 @@ pipelineConfig:
                     type: Directory
                 - name: dind-storage
                   emptyDir: {}
+          - name: pr-build-comment-success
+            steps:
+              - agent:
+                  image: gcr.io/jenkinsxio/builder-go:2.0.916-264
+                dir: ci
+                sh: "./add-pr-build-comment-success"
diff --git a/jenkins-x.yml b/jenkins-x.yml
@@ -9,8 +9,11 @@ pipelineConfig:
         command: echo "skipping promote"
     pullRequest:
       pipeline:
+        options:
+          containerOptions:
+            imagePullPolicy: Always
         agent:
-          image: seldonio/core-builder:0.15
+          image: seldonio/core-builder:0.16
         stages:
         - name: pr-build-comment
           steps:
@@ -20,17 +23,36 @@ pipelineConfig:
             sh: "./add-pr-build-comment"
         - name: build-and-test
           parallel:
-          - name: test-python
-            agent:
-              image: seldonio/python-builder:0.2
+          - name: seldon-python-all
+            steps:
+            - name: test-python-all
+              command: make
+              args:
+              - -C python
+              - update_package
+              - install_dev
+              - test
+              - TOX_TEST=all
+          - name: seldon-python-gcs
             steps:
-            - name: test-python
+            - name: test-python-gcs
               command: make
               args:
               - -C python
               - update_package
               - install_dev
               - test
+              - TOX_TEST=gcs
+          - name: seldon-python-tensorflow
+            steps:
+            - name: test-python-tensorflow
+              command: make
+              args:
+              - -C python
+              - update_package
+              - install_dev
+              - test
+              - TOX_TEST=tensorflow
           - name: seldon-operator
             steps:
             - name: test-operator
@@ -45,6 +67,12 @@ pipelineConfig:
               args:
               - -C executor
               - test
+        - name: pr-build-comment-success
+          steps:
+          - agent:
+              image: gcr.io/jenkinsxio/builder-go:2.0.916-264
+            dir: ci
+            sh: "./add-pr-build-comment-success"
     release:
       setVersion:
         steps:
@@ -103,7 +131,6 @@ pipelineConfig:
                     memory: 2000Mi
                 securityContext:
                   privileged: true
-                imagePullPolicy: Always
               volumes:
                 - name: modules
                   hostPath: