diff --git a/k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml b/k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml index 877ea3a2e4..cb01c1dca7 100644 --- a/k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml +++ b/k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml @@ -1363,7 +1363,7 @@ spec: 8 }} terminationGracePeriodSeconds: 5 replicas: 1 - - name: dataflow-engine + - name: seldon-dataflow-engine podSpec: containers: - env: diff --git a/k8s/kustomize/helm-components-sc/patch_dataflow.yaml b/k8s/kustomize/helm-components-sc/patch_dataflow.yaml index b8c65e33ce..64b47daa3b 100644 --- a/k8s/kustomize/helm-components-sc/patch_dataflow.yaml +++ b/k8s/kustomize/helm-components-sc/patch_dataflow.yaml @@ -4,7 +4,7 @@ metadata: name: default spec: components: - - name: dataflow-engine + - name: seldon-dataflow-engine replicas: 1 podSpec: imagePullSecrets: [] diff --git a/k8s/yaml/components.yaml b/k8s/yaml/components.yaml index eb64d0870c..13ee378ec5 100644 --- a/k8s/yaml/components.yaml +++ b/k8s/yaml/components.yaml @@ -985,7 +985,7 @@ spec: runAsUser: 1000 terminationGracePeriodSeconds: 5 replicas: 1 - - name: dataflow-engine + - name: seldon-dataflow-engine podSpec: containers: - env: diff --git a/operator/config/seldonconfigs/default.yaml b/operator/config/seldonconfigs/default.yaml index c5f6a072ca..43c5693b29 100644 --- a/operator/config/seldonconfigs/default.yaml +++ b/operator/config/seldonconfigs/default.yaml @@ -4,7 +4,7 @@ metadata: name: default spec: components: - - name: dataflow-engine + - name: seldon-dataflow-engine replicas: 1 podSpec: containers: diff --git a/operator/controllers/reconcilers/seldon/runtime_reconciler.go b/operator/controllers/reconcilers/seldon/runtime_reconciler.go index 140e67db7e..3a57cf57cf 100644 --- a/operator/controllers/reconcilers/seldon/runtime_reconciler.go +++ b/operator/controllers/reconcilers/seldon/runtime_reconciler.go @@ -55,7 +55,9 @@ func NewSeldonRuntimeReconciler( var componentReconcilers []common.Reconciler for _, c := range seldonConfig.Spec.Components { override := overrides[c.Name] + commonConfig.Logger.Info("Creating reconciler", "name", c.Name, "has override", override != nil) if override == nil || !override.Disable { + commonConfig.Logger.Info("Creating component", "name", c.Name) if len(c.VolumeClaimTemplates) > 0 { componentReconcilers = append(componentReconcilers, NewComponentStatefulSetReconciler( diff --git a/samples/k8s-clusterwide-kafka-mtls-demo.md b/samples/k8s-clusterwide-kafka-mtls-demo.md index e76553f4df..7390af7134 100644 --- a/samples/k8s-clusterwide-kafka-mtls-demo.md +++ b/samples/k8s-clusterwide-kafka-mtls-demo.md @@ -17,12 +17,12 @@ helm upgrade --install seldon-core-v2-crds ../k8s/helm-charts/seldon-core-v2-cr ``` ``` -Release "seldon-core-v2-crds" does not exist. Installing it now. +Release "seldon-core-v2-crds" has been upgraded. Happy Helming! NAME: seldon-core-v2-crds -LAST DEPLOYED: Sun May 14 16:08:17 2023 +LAST DEPLOYED: Thu Jun 22 15:22:04 2023 NAMESPACE: seldon-mesh STATUS: deployed -REVISION: 1 +REVISION: 5 TEST SUITE: None ``` @@ -74,7 +74,7 @@ helm install seldon-v2 ../k8s/helm-charts/seldon-core-v2-setup/ -n seldon-mesh - ```yaml NAME: seldon-v2 -LAST DEPLOYED: Sun May 14 16:13:24 2023 +LAST DEPLOYED: Thu Jun 22 15:22:11 2023 NAMESPACE: seldon-mesh STATUS: deployed REVISION: 1 @@ -87,16 +87,12 @@ TEST SUITE: None Strimzi doesn't allow a single Kafka cluster to be used in TLS easily from multiple namespaces without copying the user Secrets created by the KafkaUser to those namespaces. ```bash -kubectl create -f ../k8s/samples/strimzi-example-tls-user.yaml -n ns1 -``` - +kubectl get secret seldon -n seldon-mesh -o json | jq 'del(.metadata.ownerReferences) | del(.metadata.namespace)' | kubectl create -f - -n ns1 ``` -kafkauser.kafka.strimzi.io/seldon created ``` +secret/seldon created -```bash -kubectl get secret seldon -n seldon-mesh -o json | jq 'del(.metadata.ownerReferences) | del(.metadata.namespace)' | kubectl create -f - -n ns1 ``` ```bash @@ -129,12 +125,12 @@ secret/seldon-cluster-ca-cert created ## Create SeldonRuntimes ```bash -helm install seldon-v2-runtime ../k8s/helm-charts/seldon-core-v2-runtime -n ns1 +helm install seldon-v2-runtime ../k8s/helm-charts/seldon-core-v2-runtime -n ns1 --wait ``` ```yaml NAME: seldon-v2-runtime -LAST DEPLOYED: Sun May 14 16:13:43 2023 +LAST DEPLOYED: Thu Jun 22 15:22:19 2023 NAMESPACE: ns1 STATUS: deployed REVISION: 1 @@ -143,12 +139,40 @@ TEST SUITE: None ``` ```bash -helm install seldon-v2-runtime ../k8s/helm-charts/seldon-core-v2-runtime -n ns2 +helm install seldon-v2-runtime ../k8s/helm-charts/seldon-core-v2-runtime -n ns2 --wait ``` ```yaml NAME: seldon-v2-runtime -LAST DEPLOYED: Sun May 14 16:16:05 2023 +LAST DEPLOYED: Thu Jun 22 15:22:20 2023 +NAMESPACE: ns2 +STATUS: deployed +REVISION: 1 +TEST SUITE: None + +``` + +```bash +helm install seldon-v2-servers ../k8s/helm-charts/seldon-core-v2-servers -n ns1 --wait +``` + +```yaml +NAME: seldon-v2-servers +LAST DEPLOYED: Thu Jun 22 15:22:23 2023 +NAMESPACE: ns1 +STATUS: deployed +REVISION: 1 +TEST SUITE: None + +``` + +```bash +helm install seldon-v2-servers ../k8s/helm-charts/seldon-core-v2-servers -n ns2 --wait +``` + +```yaml +NAME: seldon-v2-servers +LAST DEPLOYED: Thu Jun 22 15:22:24 2023 NAMESPACE: ns2 STATUS: deployed REVISION: 1 @@ -156,6 +180,26 @@ TEST SUITE: None ``` +```bash +kubectl wait --for condition=ready --timeout=300s server --all -n ns1 +``` + +``` +server.mlops.seldon.io/mlserver condition met +server.mlops.seldon.io/triton condition met + +``` + +```bash +kubectl wait --for condition=ready --timeout=300s server --all -n ns2 +``` + +``` +server.mlops.seldon.io/mlserver condition met +server.mlops.seldon.io/triton condition met + +``` + ### Get Inference Endpoints ```python @@ -426,7 +470,16 @@ model.mlops.seldon.io "tfsimple2" deleted ``` ```bash -helm delete seldon-v2-runtime -n ns1 +helm delete seldon-v2-servers -n ns1 --wait +``` + +``` +release "seldon-v2-servers" uninstalled + +``` + +```bash +helm delete seldon-v2-runtime -n ns1 --wait ``` ``` @@ -435,7 +488,16 @@ release "seldon-v2-runtime" uninstalled ``` ```bash -helm delete seldon-v2-runtime -n ns2 +helm delete seldon-v2-servers -n ns2 --wait +``` + +``` +release "seldon-v2-servers" uninstalled + +``` + +```bash +helm delete seldon-v2-runtime -n ns2 --wait ``` ``` @@ -452,6 +514,17 @@ release "seldon-v2" uninstalled ``` +```bash +kubectl delete namespace ns1 +kubectl delete namespace ns2 +``` + +``` +namespace "ns1" deleted +namespace "ns2" deleted + +``` + ```python ``` diff --git a/samples/k8s-clusterwide-ssl-demo.md b/samples/k8s-clusterwide-ssl-demo.md index 37a1682acc..6ea652b0cd 100644 --- a/samples/k8s-clusterwide-ssl-demo.md +++ b/samples/k8s-clusterwide-ssl-demo.md @@ -17,12 +17,12 @@ helm upgrade --install seldon-core-v2-crds ../k8s/helm-charts/seldon-core-v2-cr ``` ``` -Release "seldon-core-v2-crds" does not exist. Installing it now. +Release "seldon-core-v2-crds" has been upgraded. Happy Helming! NAME: seldon-core-v2-crds -LAST DEPLOYED: Fri May 19 19:10:21 2023 +LAST DEPLOYED: Thu Jun 22 09:11:26 2023 NAMESPACE: seldon-mesh STATUS: deployed -REVISION: 1 +REVISION: 2 TEST SUITE: None ``` @@ -51,7 +51,7 @@ helm install seldon-v2-certs ../k8s/helm-charts/seldon-core-v2-certs/ -n ns1 ```yaml NAME: seldon-v2-certs -LAST DEPLOYED: Fri May 19 19:10:25 2023 +LAST DEPLOYED: Thu Jun 22 10:00:32 2023 NAMESPACE: ns1 STATUS: deployed REVISION: 1 @@ -65,7 +65,7 @@ helm install seldon-v2-certs ../k8s/helm-charts/seldon-core-v2-certs/ -n ns2 ```yaml NAME: seldon-v2-certs -LAST DEPLOYED: Fri May 19 19:10:28 2023 +LAST DEPLOYED: Thu Jun 22 09:11:36 2023 NAMESPACE: ns2 STATUS: deployed REVISION: 1 @@ -112,7 +112,7 @@ helm install seldon-v2 ../k8s/helm-charts/seldon-core-v2-setup/ -n seldon-mesh - ```yaml NAME: seldon-v2 -LAST DEPLOYED: Fri May 19 19:10:30 2023 +LAST DEPLOYED: Thu Jun 22 09:11:40 2023 NAMESPACE: seldon-mesh STATUS: deployed REVISION: 1 @@ -126,7 +126,21 @@ helm install seldon-v2-runtime ../k8s/helm-charts/seldon-core-v2-runtime -n ns1 ```yaml NAME: seldon-v2-runtime -LAST DEPLOYED: Fri May 19 19:10:32 2023 +LAST DEPLOYED: Thu Jun 22 10:00:36 2023 +NAMESPACE: ns1 +STATUS: deployed +REVISION: 1 +TEST SUITE: None + +``` + +```bash +helm install seldon-v2-servers ../k8s/helm-charts/seldon-core-v2-servers -n ns1 +``` + +```yaml +NAME: seldon-v2-servers +LAST DEPLOYED: Thu Jun 22 10:00:36 2023 NAMESPACE: ns1 STATUS: deployed REVISION: 1 @@ -140,7 +154,21 @@ helm install seldon-v2-runtime ../k8s/helm-charts/seldon-core-v2-runtime -n ns2 ```yaml NAME: seldon-v2-runtime -LAST DEPLOYED: Fri May 19 19:10:35 2023 +LAST DEPLOYED: Thu Jun 22 09:11:49 2023 +NAMESPACE: ns2 +STATUS: deployed +REVISION: 1 +TEST SUITE: None + +``` + +```bash +helm install seldon-v2-servers ../k8s/helm-charts/seldon-core-v2-servers -n ns2 +``` + +```yaml +NAME: seldon-v2-servers +LAST DEPLOYED: Thu Jun 22 09:12:16 2023 NAMESPACE: ns2 STATUS: deployed REVISION: 1 @@ -179,7 +207,7 @@ MESH_IP_NS1 ``` ``` -'172.21.255.4' +'172.21.255.2' ``` @@ -192,7 +220,7 @@ MESH_IP_NS2 ``` ``` -'172.21.255.2' +'172.21.255.4' ``` @@ -233,7 +261,7 @@ seldon model infer iris --inference-host ${MESH_IP_NS1}:80 \ { "model_name": "iris_1", "model_version": "1", - "id": "c584e29e-66fd-479c-9e9b-afe044256594", + "id": "77f21360-b6cf-4c50-a000-1f219f5e9e29", "parameters": {}, "outputs": [ { @@ -243,6 +271,9 @@ seldon model infer iris --inference-host ${MESH_IP_NS1}:80 \ 1 ], "datatype": "INT64", + "parameters": { + "content_type": "np" + }, "data": [ 2 ] @@ -281,7 +312,7 @@ seldon model infer iris --inference-host ${MESH_IP_NS2}:80 \ { "model_name": "iris_1", "model_version": "1", - "id": "ef72541f-2445-4fa9-b0d2-b9d0706cf4bf", + "id": "902b8483-0ede-4136-97be-dbf266f05a0c", "parameters": {}, "outputs": [ { @@ -291,6 +322,9 @@ seldon model infer iris --inference-host ${MESH_IP_NS2}:80 \ 1 ], "datatype": "INT64", + "parameters": { + "content_type": "np" + }, "data": [ 2 ] @@ -321,11 +355,13 @@ model.mlops.seldon.io "iris" deleted ``` ```bash +helm delete seldon-v2-servers -n ns1 helm delete seldon-v2-runtime -n ns1 helm delete seldon-v2-certs -n ns1 ``` ``` +release "seldon-v2-servers" uninstalled release "seldon-v2-runtime" uninstalled release "seldon-v2-certs" uninstalled @@ -333,11 +369,13 @@ release "seldon-v2-certs" uninstalled ```bash helm delete seldon-v2-runtime -n ns2 +helm delete seldon-v2-runtime -n ns2 helm delete seldon-v2-certs -n ns2 ``` ``` release "seldon-v2-runtime" uninstalled +Error: uninstall: Release not loaded: seldon-v2-runtime: release: not found release "seldon-v2-certs" uninstalled ``` diff --git a/samples/k8s-clusterwide.ipynb b/samples/k8s-clusterwide.ipynb index 18b3e2b39b..2751a27430 100644 --- a/samples/k8s-clusterwide.ipynb +++ b/samples/k8s-clusterwide.ipynb @@ -11,7 +11,55 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 41, + "id": "e09cb983", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Release \"seldon-core-v2-crds\" does not exist. Installing it now.\n", + "NAME: seldon-core-v2-crds\n", + "LAST DEPLOYED: Fri Jun 23 14:41:22 2023\n", + "NAMESPACE: seldon-mesh\n", + "STATUS: deployed\n", + "REVISION: 1\n", + "TEST SUITE: None\n" + ] + } + ], + "source": [ + "!helm upgrade --install seldon-core-v2-crds ../k8s/helm-charts/seldon-core-v2-crds -n seldon-mesh" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "id": "a4debc76", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Release \"seldon-v2\" does not exist. Installing it now.\n", + "NAME: seldon-v2\n", + "LAST DEPLOYED: Fri Jun 23 14:41:26 2023\n", + "NAMESPACE: seldon-mesh\n", + "STATUS: deployed\n", + "REVISION: 1\n", + "TEST SUITE: None\n" + ] + } + ], + "source": [ + "!helm upgrade --install seldon-v2 ../k8s/helm-charts/seldon-core-v2-setup/ -n seldon-mesh --set controller.clusterwide=true" + ] + }, + { + "cell_type": "code", + "execution_count": 43, "id": "29353104", "metadata": {}, "outputs": [ @@ -31,16 +79,39 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 10, "id": "56cdf114", "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME: seldon-v2-runtime\r\n", + "LAST DEPLOYED: Fri Jun 23 14:10:23 2023\r\n", + "NAMESPACE: ns1\r\n", + "STATUS: deployed\r\n", + "REVISION: 1\r\n", + "TEST SUITE: None\r\n" + ] + } + ], + "source": [ + "!helm install seldon-v2-runtime ../k8s/helm-charts/seldon-core-v2-runtime -n ns1 --wait" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "4b815db6", + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "NAME: seldon-v2-servers\r\n", - "LAST DEPLOYED: Thu May 11 10:15:20 2023\r\n", + "LAST DEPLOYED: Fri Jun 23 14:10:38 2023\r\n", "NAMESPACE: ns1\r\n", "STATUS: deployed\r\n", "REVISION: 1\r\n", @@ -49,21 +120,44 @@ } ], "source": [ - "!helm install seldon-v2-servers ../k8s/helm-charts/seldon-core-v2-runtime -n ns1 --wait" + "!helm install seldon-v2-servers ../k8s/helm-charts/seldon-core-v2-servers -n ns1 --wait" ] }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 12, "id": "3ce2d189", "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME: seldon-v2-runtime\r\n", + "LAST DEPLOYED: Fri Jun 23 14:10:42 2023\r\n", + "NAMESPACE: ns2\r\n", + "STATUS: deployed\r\n", + "REVISION: 1\r\n", + "TEST SUITE: None\r\n" + ] + } + ], + "source": [ + "!helm install seldon-v2-runtime ../k8s/helm-charts/seldon-core-v2-runtime -n ns2 --wait" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "4a214689", + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "NAME: seldon-v2-servers\r\n", - "LAST DEPLOYED: Thu May 11 10:15:25 2023\r\n", + "LAST DEPLOYED: Fri Jun 23 14:10:44 2023\r\n", "NAMESPACE: ns2\r\n", "STATUS: deployed\r\n", "REVISION: 1\r\n", @@ -72,12 +166,12 @@ } ], "source": [ - "!helm install seldon-v2-servers ../k8s/helm-charts/seldon-core-v2-runtime -n ns2 --wait" + "!helm install seldon-v2-servers ../k8s/helm-charts/seldon-core-v2-servers -n ns2 --wait" ] }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 14, "id": "sunrise-commercial", "metadata": {}, "outputs": [ @@ -87,7 +181,7 @@ "'172.21.255.2'" ] }, - "execution_count": 33, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -102,7 +196,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 15, "id": "012886cc", "metadata": {}, "outputs": [ @@ -112,7 +206,7 @@ "'172.21.255.4'" ] }, - "execution_count": 34, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -135,7 +229,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 16, "id": "operating-console", "metadata": {}, "outputs": [ @@ -161,7 +255,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 17, "id": "exempt-bumper", "metadata": { "scrolled": true @@ -181,7 +275,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 18, "id": "beneficial-logan", "metadata": {}, "outputs": [ @@ -199,7 +293,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 19, "id": "67900afd", "metadata": {}, "outputs": [ @@ -210,7 +304,7 @@ "{\r\n", "\t\"model_name\": \"iris_1\",\r\n", "\t\"model_version\": \"1\",\r\n", - "\t\"id\": \"83372e34-5f0c-4df6-8bde-cc4f4f2c3ffe\",\r\n", + "\t\"id\": \"276de7e7-9f2f-4329-9179-08d4d54bb0b5\",\r\n", "\t\"parameters\": {},\r\n", "\t\"outputs\": [\r\n", "\t\t{\r\n", @@ -220,6 +314,9 @@ "\t\t\t\t1\r\n", "\t\t\t],\r\n", "\t\t\t\"datatype\": \"INT64\",\r\n", + "\t\t\t\"parameters\": {\r\n", + "\t\t\t\t\"content_type\": \"np\"\r\n", + "\t\t\t},\r\n", "\t\t\t\"data\": [\r\n", "\t\t\t\t2\r\n", "\t\t\t]\r\n", @@ -236,7 +333,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 20, "id": "a1848c7d", "metadata": {}, "outputs": [ @@ -255,6 +352,11 @@ " \"1\",\r\n", " \"1\"\r\n", " ],\r\n", + " \"parameters\": {\r\n", + " \"content_type\": {\r\n", + " \"stringParam\": \"np\"\r\n", + " }\r\n", + " },\r\n", " \"contents\": {\r\n", " \"int64Contents\": [\r\n", " \"2\"\r\n", @@ -273,7 +375,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 21, "id": "0843b13e", "metadata": { "scrolled": true @@ -293,7 +395,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 22, "id": "99e270be", "metadata": {}, "outputs": [ @@ -311,7 +413,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 23, "id": "ae262b3d", "metadata": {}, "outputs": [ @@ -322,7 +424,7 @@ "{\r\n", "\t\"model_name\": \"iris_1\",\r\n", "\t\"model_version\": \"1\",\r\n", - "\t\"id\": \"4da38370-5790-4739-8ec7-6389f86843e5\",\r\n", + "\t\"id\": \"7c0bb004-be2c-4483-97a9-6fd2e0a834ad\",\r\n", "\t\"parameters\": {},\r\n", "\t\"outputs\": [\r\n", "\t\t{\r\n", @@ -332,6 +434,9 @@ "\t\t\t\t1\r\n", "\t\t\t],\r\n", "\t\t\t\"datatype\": \"INT64\",\r\n", + "\t\t\t\"parameters\": {\r\n", + "\t\t\t\t\"content_type\": \"np\"\r\n", + "\t\t\t},\r\n", "\t\t\t\"data\": [\r\n", "\t\t\t\t2\r\n", "\t\t\t]\r\n", @@ -348,7 +453,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 24, "id": "30b3342f", "metadata": {}, "outputs": [ @@ -367,6 +472,11 @@ " \"1\",\r\n", " \"1\"\r\n", " ],\r\n", + " \"parameters\": {\r\n", + " \"content_type\": {\r\n", + " \"stringParam\": \"np\"\r\n", + " }\r\n", + " },\r\n", " \"contents\": {\r\n", " \"int64Contents\": [\r\n", " \"2\"\r\n", @@ -385,7 +495,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 25, "id": "precious-development", "metadata": {}, "outputs": [ @@ -413,7 +523,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 26, "id": "e694edee", "metadata": {}, "outputs": [ @@ -427,15 +537,91 @@ } ], "source": [ - "!helm delete seldon-v2-servers -n ns1\n", - "!helm delete seldon-v2-servers -n ns2" + "!helm delete seldon-v2-servers -n ns1 --wait\n", + "!helm delete seldon-v2-servers -n ns2 --wait" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 35, "id": "8b00bd3e", "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "release \"seldon-v2-runtime\" uninstalled\n", + "Error: uninstall: Release not loaded: seldon-v2-runtime: release: not found\n" + ] + } + ], + "source": [ + "!helm delete seldon-v2-runtime -n ns1 --wait\n", + "!helm delete seldon-v2-runtime -n ns2 --wait" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "4c671429", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "release \"seldon-v2\" uninstalled\r\n" + ] + } + ], + "source": [ + "!helm delete seldon-v2 -n seldon-mesh --wait" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "72b24840", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "release \"seldon-core-v2-crds\" uninstalled\r\n" + ] + } + ], + "source": [ + "!helm delete seldon-core-v2-crds -n seldon-mesh" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "e7773481", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "namespace \"ns1\" deleted\n", + "namespace \"ns2\" deleted\n" + ] + } + ], + "source": [ + "!kubectl delete namespace ns1\n", + "!kubectl delete namespace ns2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bf5ce6cd", + "metadata": {}, "outputs": [], "source": [] } diff --git a/samples/k8s-clusterwide.md b/samples/k8s-clusterwide.md index f56fade74f..82dea9fa48 100644 --- a/samples/k8s-clusterwide.md +++ b/samples/k8s-clusterwide.md @@ -1,5 +1,35 @@ ## Seldon V2 Multi-Namespace Kubernetes Example +```bash +helm upgrade --install seldon-core-v2-crds ../k8s/helm-charts/seldon-core-v2-crds -n seldon-mesh +``` + +``` +Release "seldon-core-v2-crds" does not exist. Installing it now. +NAME: seldon-core-v2-crds +LAST DEPLOYED: Fri Jun 23 14:41:22 2023 +NAMESPACE: seldon-mesh +STATUS: deployed +REVISION: 1 +TEST SUITE: None + +``` + +```bash +helm upgrade --install seldon-v2 ../k8s/helm-charts/seldon-core-v2-setup/ -n seldon-mesh --set controller.clusterwide=true +``` + +``` +Release "seldon-v2" does not exist. Installing it now. +NAME: seldon-v2 +LAST DEPLOYED: Fri Jun 23 14:41:26 2023 +NAMESPACE: seldon-mesh +STATUS: deployed +REVISION: 1 +TEST SUITE: None + +``` + ```bash kubectl create namespace ns1 kubectl create namespace ns2 @@ -11,13 +41,27 @@ namespace/ns2 created ``` +```bash +helm install seldon-v2-runtime ../k8s/helm-charts/seldon-core-v2-runtime -n ns1 --wait +``` + +```yaml +NAME: seldon-v2-runtime +LAST DEPLOYED: Fri Jun 23 14:10:23 2023 +NAMESPACE: ns1 +STATUS: deployed +REVISION: 1 +TEST SUITE: None + +``` + ```bash helm install seldon-v2-servers ../k8s/helm-charts/seldon-core-v2-servers -n ns1 --wait ``` ```yaml NAME: seldon-v2-servers -LAST DEPLOYED: Thu May 11 10:15:20 2023 +LAST DEPLOYED: Fri Jun 23 14:10:38 2023 NAMESPACE: ns1 STATUS: deployed REVISION: 1 @@ -25,13 +69,27 @@ TEST SUITE: None ``` +```bash +helm install seldon-v2-runtime ../k8s/helm-charts/seldon-core-v2-runtime -n ns2 --wait +``` + +```yaml +NAME: seldon-v2-runtime +LAST DEPLOYED: Fri Jun 23 14:10:42 2023 +NAMESPACE: ns2 +STATUS: deployed +REVISION: 1 +TEST SUITE: None + +``` + ```bash helm install seldon-v2-servers ../k8s/helm-charts/seldon-core-v2-servers -n ns2 --wait ``` ```yaml NAME: seldon-v2-servers -LAST DEPLOYED: Thu May 11 10:15:25 2023 +LAST DEPLOYED: Fri Jun 23 14:10:44 2023 NAMESPACE: ns2 STATUS: deployed REVISION: 1 @@ -111,7 +169,7 @@ seldon model infer iris --inference-host ${MESH_IP_NS1}:80 \ { "model_name": "iris_1", "model_version": "1", - "id": "83372e34-5f0c-4df6-8bde-cc4f4f2c3ffe", + "id": "276de7e7-9f2f-4329-9179-08d4d54bb0b5", "parameters": {}, "outputs": [ { @@ -121,6 +179,9 @@ seldon model infer iris --inference-host ${MESH_IP_NS1}:80 \ 1 ], "datatype": "INT64", + "parameters": { + "content_type": "np" + }, "data": [ 2 ] @@ -147,6 +208,11 @@ seldon model infer iris --inference-mode grpc --inference-host ${MESH_IP_NS1}:80 "1", "1" ], + "parameters": { + "content_type": { + "stringParam": "np" + } + }, "contents": { "int64Contents": [ "2" @@ -185,7 +251,7 @@ seldon model infer iris --inference-host ${MESH_IP_NS2}:80 \ { "model_name": "iris_1", "model_version": "1", - "id": "4da38370-5790-4739-8ec7-6389f86843e5", + "id": "7c0bb004-be2c-4483-97a9-6fd2e0a834ad", "parameters": {}, "outputs": [ { @@ -195,6 +261,9 @@ seldon model infer iris --inference-host ${MESH_IP_NS2}:80 \ 1 ], "datatype": "INT64", + "parameters": { + "content_type": "np" + }, "data": [ 2 ] @@ -221,6 +290,11 @@ seldon model infer iris --inference-mode grpc --inference-host ${MESH_IP_NS2}:80 "1", "1" ], + "parameters": { + "content_type": { + "stringParam": "np" + } + }, "contents": { "int64Contents": [ "2" @@ -246,8 +320,8 @@ model.mlops.seldon.io "iris" deleted ## TearDown ```bash -helm delete seldon-v2-servers -n ns1 -helm delete seldon-v2-servers -n ns2 +helm delete seldon-v2-servers -n ns1 --wait +helm delete seldon-v2-servers -n ns2 --wait ``` ``` @@ -256,6 +330,46 @@ release "seldon-v2-servers" uninstalled ``` +```bash +helm delete seldon-v2-runtime -n ns1 --wait +helm delete seldon-v2-runtime -n ns2 --wait +``` + +``` +release "seldon-v2-runtime" uninstalled +Error: uninstall: Release not loaded: seldon-v2-runtime: release: not found + +``` + +```bash +helm delete seldon-v2 -n seldon-mesh --wait +``` + +``` +release "seldon-v2" uninstalled + +``` + +```bash +helm delete seldon-core-v2-crds -n seldon-mesh +``` + +``` +release "seldon-core-v2-crds" uninstalled + +``` + +```bash +kubectl delete namespace ns1 +kubectl delete namespace ns2 +``` + +``` +namespace "ns1" deleted +namespace "ns2" deleted + +``` + ```python ``` diff --git a/samples/k8s-examples.md b/samples/k8s-examples.md index e9ad0171ce..44205bd6a9 100644 --- a/samples/k8s-examples.md +++ b/samples/k8s-examples.md @@ -14,7 +14,7 @@ MESH_IP ``` ``` -'172.19.255.1' +'172.21.255.2' ``` @@ -63,12 +63,12 @@ kubectl get model iris -n ${NAMESPACE} -o jsonpath='{.status}' | jq -M . { "conditions": [ { - "lastTransitionTime": "2023-03-10T10:51:22Z", + "lastTransitionTime": "2023-05-09T10:16:01Z", "status": "True", "type": "ModelReady" }, { - "lastTransitionTime": "2023-03-10T10:51:22Z", + "lastTransitionTime": "2023-05-09T10:16:01Z", "status": "True", "type": "Ready" } @@ -87,7 +87,7 @@ seldon model infer iris --inference-host ${MESH_IP}:80 \ { "model_name": "iris_1", "model_version": "1", - "id": "ce0d6b9e-3752-4caa-8019-d0a664a8aa83", + "id": "24922888-94a9-4c12-b1a8-db7e9c31ec66", "parameters": {}, "outputs": [ { @@ -142,12 +142,12 @@ kubectl get server mlserver -n ${NAMESPACE} -o jsonpath='{.status}' | jq -M . { "conditions": [ { - "lastTransitionTime": "2023-03-10T10:50:49Z", + "lastTransitionTime": "2023-05-09T10:14:16Z", "status": "True", "type": "Ready" }, { - "lastTransitionTime": "2023-03-10T10:50:49Z", + "lastTransitionTime": "2023-05-09T10:14:16Z", "reason": "StatefulSet replicas matches desired replicas", "status": "True", "type": "StatefulSetReady" @@ -267,7 +267,7 @@ seldon model infer --inference-host ${MESH_IP}:80 -i 50 iris \ ``` ``` -Success: map[:iris2_1::24 :iris_1::26] +Success: map[:iris2_1::27 :iris_1::23] ``` @@ -707,12 +707,12 @@ kubectl get model income -n ${NAMESPACE} -o jsonpath='{.status}' | jq -M . { "conditions": [ { - "lastTransitionTime": "2023-03-10T10:52:24Z", + "lastTransitionTime": "2023-05-09T10:16:52Z", "status": "True", "type": "ModelReady" }, { - "lastTransitionTime": "2023-03-10T10:52:24Z", + "lastTransitionTime": "2023-05-09T10:16:52Z", "status": "True", "type": "Ready" } @@ -731,7 +731,7 @@ seldon model infer income --inference-host ${MESH_IP}:80 \ { "model_name": "income_1", "model_version": "1", - "id": "97c219fc-a978-4b0b-82ba-b4af7cb00bce", + "id": "ccea37e6-5b71-40ba-ab0f-48f358738c57", "parameters": {}, "outputs": [ { @@ -794,12 +794,12 @@ kubectl get model income-explainer -n ${NAMESPACE} -o jsonpath='{.status}' | jq { "conditions": [ { - "lastTransitionTime": "2023-03-10T10:52:39Z", + "lastTransitionTime": "2023-05-09T10:17:39Z", "status": "True", "type": "ModelReady" }, { - "lastTransitionTime": "2023-03-10T10:52:39Z", + "lastTransitionTime": "2023-05-09T10:17:39Z", "status": "True", "type": "Ready" } @@ -818,7 +818,7 @@ seldon model infer income-explainer --inference-host ${MESH_IP}:80 \ { "model_name": "income-explainer_1", "model_version": "1", - "id": "851a6c39-4241-4991-89bb-e4bd68e8ab5d", + "id": "4a94be87-b0ed-4abc-83fa-e83db84fd330", "parameters": {}, "outputs": [ { @@ -832,7 +832,7 @@ seldon model infer income-explainer --inference-host ${MESH_IP}:80 \ "content_type": "str" }, "data": [ - "{\"meta\": {\"name\": \"AnchorTabular\", \"type\": [\"blackbox\"], \"explanations\": [\"local\"], \"params\": {\"seed\": 1, \"disc_perc\": [25, 50, 75], \"threshold\": 0.95, \"delta\": 0.1, \"tau\": 0.15, \"batch_size\": 100, \"coverage_samples\": 10000, \"beam_size\": 1, \"stop_on_first\": false, \"max_anchor_size\": null, \"min_samples_start\": 100, \"n_covered_ex\": 10, \"binary_cache_size\": 10000, \"cache_margin\": 1000, \"verbose\": false, \"verbose_every\": 1, \"kwargs\": {}}, \"version\": \"0.9.0\"}, \"data\": {\"anchor\": [\"Marital Status = Never-Married\", \"Relationship = Own-child\", \"Capital Gain <= 0.00\", \"Capital Loss <= 0.00\"], \"precision\": 0.9947368421052631, \"coverage\": 0.06720071206052515, \"raw\": {\"feature\": [3, 5, 8, 9], \"mean\": [0.8121546961325967, 0.9134078212290503, 0.9927007299270073, 0.9947368421052631], \"precision\": [0.8121546961325967, 0.9134078212290503, 0.9927007299270073, 0.9947368421052631], \"coverage\": [0.3037383177570093, 0.07165109034267912, 0.06853582554517133, 0.06720071206052515], \"examples\": [{\"covered_true\": [[47, 4, 5, 1, 5, 0, 1, 1, 0, 0, 45, 1], [32, 6, 1, 1, 5, 3, 2, 0, 2174, 0, 60, 0], [45, 5, 1, 1, 2, 0, 4, 1, 4386, 0, 35, 9], [85, 4, 1, 1, 8, 0, 4, 1, 0, 0, 3, 4], [29, 4, 1, 1, 5, 0, 4, 1, 0, 0, 40, 9], [32, 7, 5, 1, 1, 1, 4, 0, 0, 0, 38, 9], [30, 4, 1, 1, 4, 1, 4, 1, 0, 0, 60, 4], [32, 7, 5, 1, 5, 1, 4, 1, 0, 0, 10, 9], [36, 4, 1, 1, 5, 1, 4, 0, 0, 0, 45, 9], [44, 4, 1, 1, 4, 1, 4, 1, 0, 0, 35, 9]], \"covered_false\": [[45, 4, 1, 1, 6, 1, 4, 1, 8614, 0, 48, 9], [59, 4, 1, 1, 8, 0, 4, 1, 0, 0, 60, 9], [51, 6, 1, 1, 6, 0, 4, 1, 15024, 0, 50, 9], [45, 2, 5, 1, 8, 0, 4, 1, 7688, 0, 50, 9], [40, 6, 1, 1, 5, 0, 4, 1, 7298, 0, 50, 9], [43, 4, 5, 1, 5, 0, 4, 1, 0, 0, 50, 9], [41, 1, 5, 1, 8, 0, 4, 1, 0, 0, 60, 9], [40, 4, 5, 1, 5, 5, 4, 0, 15024, 0, 24, 9], [67, 5, 1, 1, 8, 0, 4, 1, 0, 2392, 75, 9], [36, 4, 5, 1, 6, 0, 4, 1, 15024, 0, 45, 9]], \"uncovered_true\": [], \"uncovered_false\": []}, {\"covered_true\": [[55, 6, 1, 1, 8, 3, 4, 1, 0, 0, 40, 9], [39, 6, 1, 1, 6, 3, 4, 1, 0, 0, 60, 9], [53, 4, 1, 1, 6, 3, 4, 1, 0, 0, 50, 9], [50, 1, 5, 1, 4, 3, 4, 1, 0, 0, 40, 9], [46, 7, 1, 1, 5, 3, 4, 1, 0, 0, 37, 9], [42, 4, 1, 1, 5, 3, 4, 1, 0, 653, 50, 9], [31, 4, 1, 1, 1, 3, 4, 1, 0, 0, 40, 9], [35, 4, 1, 1, 4, 3, 4, 1, 0, 0, 40, 9], [47, 7, 1, 1, 7, 3, 4, 1, 0, 0, 40, 9], [26, 4, 1, 1, 8, 3, 4, 0, 0, 0, 45, 9]], \"covered_false\": [[38, 1, 1, 1, 6, 3, 4, 1, 7688, 0, 42, 9], [44, 4, 1, 1, 5, 3, 4, 1, 15024, 0, 50, 3], [52, 4, 1, 1, 8, 3, 4, 1, 15024, 0, 55, 9], [35, 4, 1, 1, 6, 3, 4, 0, 8614, 0, 45, 9], [50, 5, 1, 1, 8, 3, 4, 1, 15024, 0, 60, 9], [60, 5, 1, 1, 8, 3, 4, 1, 7298, 0, 60, 9], [70, 6, 1, 1, 5, 3, 4, 1, 20051, 0, 35, 9], [43, 5, 1, 1, 6, 3, 4, 1, 15024, 0, 60, 9], [42, 4, 1, 1, 5, 3, 4, 1, 7688, 0, 40, 9], [46, 7, 1, 1, 8, 3, 4, 1, 7688, 0, 40, 9]], \"uncovered_true\": [], \"uncovered_false\": []}, {\"covered_true\": [[44, 4, 1, 1, 2, 3, 4, 1, 0, 0, 55, 9], [69, 5, 1, 1, 8, 3, 4, 1, 0, 0, 40, 9], [28, 4, 1, 1, 5, 3, 4, 0, 0, 0, 40, 9], [43, 4, 5, 1, 5, 3, 3, 1, 0, 0, 50, 9], [45, 5, 1, 1, 8, 3, 4, 1, 0, 0, 51, 9], [26, 4, 1, 1, 6, 3, 4, 1, 0, 0, 55, 9], [39, 6, 1, 1, 1, 3, 4, 1, 0, 1762, 40, 9], [39, 6, 1, 1, 2, 3, 2, 1, 0, 1669, 60, 0], [42, 4, 1, 1, 8, 3, 4, 1, 0, 0, 40, 9], [63, 4, 1, 1, 5, 3, 4, 1, 0, 0, 45, 9]], \"covered_false\": [], \"uncovered_true\": [], \"uncovered_false\": []}, {\"covered_true\": [[26, 7, 1, 1, 5, 3, 4, 1, 0, 0, 40, 9], [32, 4, 1, 1, 1, 3, 4, 0, 0, 0, 40, 9], [29, 4, 1, 1, 4, 3, 4, 1, 0, 0, 40, 9], [48, 7, 2, 1, 5, 3, 4, 1, 0, 0, 65, 9], [37, 4, 1, 1, 2, 3, 4, 1, 0, 0, 40, 9], [23, 4, 1, 1, 7, 3, 4, 0, 0, 0, 40, 9], [35, 4, 2, 1, 5, 3, 1, 1, 0, 0, 40, 2], [38, 4, 1, 1, 6, 3, 4, 1, 0, 0, 40, 9], [27, 4, 1, 1, 1, 3, 2, 0, 0, 0, 40, 9], [27, 4, 5, 1, 5, 3, 1, 1, 0, 0, 40, 2]], \"covered_false\": [], \"uncovered_true\": [], \"uncovered_false\": []}], \"all_precision\": 0, \"num_preds\": 1000000, \"success\": true, \"names\": [\"Marital Status = Never-Married\", \"Relationship = Own-child\", \"Capital Gain <= 0.00\", \"Capital Loss <= 0.00\"], \"prediction\": [0], \"instance\": [47.0, 4.0, 1.0, 1.0, 1.0, 3.0, 4.0, 1.0, 0.0, 0.0, 40.0, 9.0], \"instances\": [[47.0, 4.0, 1.0, 1.0, 1.0, 3.0, 4.0, 1.0, 0.0, 0.0, 40.0, 9.0]]}}}" + "{\"meta\": {\"name\": \"AnchorTabular\", \"type\": [\"blackbox\"], \"explanations\": [\"local\"], \"params\": {\"seed\": 1, \"disc_perc\": [25, 50, 75], \"threshold\": 0.95, \"delta\": 0.1, \"tau\": 0.15, \"batch_size\": 100, \"coverage_samples\": 10000, \"beam_size\": 1, \"stop_on_first\": false, \"max_anchor_size\": null, \"min_samples_start\": 100, \"n_covered_ex\": 10, \"binary_cache_size\": 10000, \"cache_margin\": 1000, \"verbose\": false, \"verbose_every\": 1, \"kwargs\": {}}, \"version\": \"0.9.1\"}, \"data\": {\"anchor\": [\"Marital Status = Never-Married\", \"Capital Gain <= 0.00\", \"Relationship = Own-child\"], \"precision\": 0.9869281045751634, \"coverage\": 0.06853582554517133, \"raw\": {\"feature\": [3, 8, 5], \"mean\": [0.8080110497237569, 0.896, 0.9869281045751634], \"precision\": [0.8080110497237569, 0.896, 0.9869281045751634], \"coverage\": [0.3037383177570093, 0.2834890965732087, 0.06853582554517133], \"examples\": [{\"covered_true\": [[49, 5, 1, 1, 5, 4, 4, 1, 6497, 0, 45, 9], [41, 6, 5, 1, 2, 0, 4, 1, 0, 0, 65, 9], [28, 4, 1, 1, 1, 2, 4, 0, 0, 0, 15, 9], [66, 0, 1, 1, 0, 0, 4, 1, 6767, 0, 20, 9], [43, 2, 1, 1, 5, 3, 4, 0, 0, 0, 39, 9], [35, 4, 5, 1, 8, 1, 4, 0, 0, 0, 55, 9], [58, 2, 5, 1, 5, 0, 4, 1, 0, 0, 40, 9], [41, 0, 1, 1, 0, 5, 4, 0, 0, 0, 30, 9], [44, 4, 1, 1, 1, 4, 4, 0, 0, 0, 40, 9], [39, 4, 1, 1, 8, 1, 4, 1, 0, 0, 55, 9]], \"covered_false\": [[46, 4, 1, 1, 8, 0, 4, 1, 0, 0, 50, 9], [32, 4, 1, 1, 5, 0, 4, 1, 99999, 0, 50, 9], [39, 4, 2, 1, 5, 0, 4, 1, 99999, 0, 55, 9], [32, 4, 5, 1, 8, 1, 4, 1, 14084, 0, 40, 9], [55, 4, 1, 1, 8, 0, 4, 1, 15024, 0, 42, 9], [45, 4, 1, 1, 8, 0, 4, 1, 0, 0, 50, 9], [38, 0, 5, 1, 0, 5, 2, 0, 15024, 0, 2, 9], [43, 4, 5, 1, 5, 4, 4, 0, 0, 2547, 40, 9], [30, 4, 1, 1, 4, 1, 4, 0, 4787, 0, 45, 9], [69, 6, 1, 1, 5, 0, 4, 1, 20051, 0, 45, 9]], \"uncovered_true\": [], \"uncovered_false\": []}, {\"covered_true\": [[40, 2, 1, 1, 5, 0, 4, 1, 0, 0, 48, 9], [38, 4, 1, 1, 6, 0, 4, 1, 0, 0, 40, 9], [43, 6, 5, 1, 5, 1, 4, 0, 0, 0, 20, 9], [38, 6, 1, 1, 5, 0, 4, 1, 0, 0, 70, 9], [60, 4, 1, 1, 5, 0, 4, 1, 0, 0, 50, 9], [36, 4, 1, 1, 5, 0, 4, 1, 0, 0, 40, 9], [35, 4, 1, 1, 2, 0, 1, 1, 0, 0, 40, 7], [67, 0, 1, 1, 0, 0, 4, 1, 0, 0, 60, 9], [28, 2, 1, 1, 1, 1, 2, 0, 0, 0, 60, 9], [47, 4, 1, 1, 1, 0, 4, 1, 0, 0, 40, 9]], \"covered_false\": [[49, 4, 1, 1, 8, 1, 4, 0, 0, 0, 56, 9], [35, 4, 5, 1, 5, 0, 4, 1, 0, 1977, 45, 9], [49, 4, 5, 1, 8, 0, 4, 1, 0, 1977, 40, 9], [51, 4, 1, 1, 8, 1, 4, 1, 0, 0, 50, 9], [32, 4, 1, 1, 8, 0, 4, 1, 0, 0, 60, 9], [42, 1, 1, 1, 8, 0, 4, 1, 0, 0, 52, 9], [36, 4, 5, 1, 8, 0, 4, 1, 0, 0, 50, 9], [55, 4, 5, 1, 8, 0, 4, 1, 0, 0, 55, 9], [37, 4, 5, 1, 8, 0, 4, 1, 0, 0, 45, 9], [48, 4, 1, 1, 8, 0, 4, 1, 0, 0, 60, 9]], \"uncovered_true\": [], \"uncovered_false\": []}, {\"covered_true\": [[30, 4, 1, 1, 8, 3, 1, 1, 0, 0, 40, 2], [40, 4, 2, 1, 5, 3, 4, 1, 0, 0, 40, 9], [30, 6, 5, 1, 5, 3, 1, 1, 0, 0, 50, 1], [30, 4, 1, 1, 8, 3, 1, 1, 0, 0, 40, 2], [27, 4, 1, 1, 8, 3, 4, 1, 0, 0, 45, 9], [65, 4, 1, 1, 7, 3, 4, 1, 0, 0, 20, 9], [31, 4, 1, 1, 1, 3, 4, 0, 0, 0, 45, 9], [57, 6, 1, 1, 6, 3, 4, 1, 0, 0, 10, 9], [28, 4, 1, 1, 7, 3, 4, 0, 0, 0, 50, 9], [26, 2, 1, 1, 5, 3, 4, 0, 0, 0, 40, 9]], \"covered_false\": [[71, 5, 1, 1, 8, 3, 4, 1, 0, 2392, 60, 9]], \"uncovered_true\": [], \"uncovered_false\": []}], \"all_precision\": 0, \"num_preds\": 1000000, \"success\": true, \"names\": [\"Marital Status = Never-Married\", \"Capital Gain <= 0.00\", \"Relationship = Own-child\"], \"prediction\": [0], \"instance\": [47.0, 4.0, 1.0, 1.0, 1.0, 3.0, 4.0, 1.0, 0.0, 0.0, 40.0, 9.0], \"instances\": [[47.0, 4.0, 1.0, 1.0, 1.0, 3.0, 4.0, 1.0, 0.0, 0.0, 40.0, 9.0]]}}}" ] } ]