From c63a1c813103577e9acd68773a8a0712c8a08e15 Mon Sep 17 00:00:00 2001 From: dhruvsgarg Date: Tue, 12 Jul 2022 16:00:35 -0400 Subject: [PATCH] Initial deployer registration commit with older endpoints and components misc: add missing license (#173) Several files that are recently added don't have license statements. license statements are added in those files. Making docker driver default. (#171) Updated endpoints and components, removed registration-db controller code --- api/computes_api.partials.yml | 12 +-- api/computes_components.partials.yml | 25 ++--- cmd/controller/app/controller.go | 1 + cmd/controller/app/database/db_interfaces.go | 6 ++ .../app/database/mongodb/compute.go | 91 +++++++++++++++++++ .../app/database/mongodb/helpers.go | 1 + cmd/controller/app/database/mongodb/job.go | 2 +- cmd/deployer/app/appserver.go | 16 ++++ cmd/deployer/app/register.go | 80 ++++++++++++++++ cmd/deployer/app/service.go | 16 ++++ cmd/deployer/cmd/root.go | 90 ++++++++++++++++++ cmd/deployer/cmd/root_test.go | 43 +++++++++ cmd/deployer/main.go | 35 +++++-- docs/03-fiab.md | 12 ++- .../templates/apiserver-ingress.yaml | 16 ++++ .../templates/controller-job-configmap.yaml | 16 ++++ .../templates/deployer-deployment.yaml | 21 ++++- fiab/helm-chart/templates/mlflow-ingress.yaml | 16 ++++ .../templates/notifier-ingress.yaml | 16 ++++ pkg/openapi/api.go | 2 +- pkg/openapi/api_computes.go | 7 +- pkg/openapi/apiserver/api_computes_service.go | 49 ++++++---- .../controller/api_computes_service.go | 20 ++-- pkg/openapi/model_compute_spec.go | 11 --- pkg/openapi/model_compute_status.go | 19 ++-- pkg/restapi/restapi.go | 14 +++ 26 files changed, 539 insertions(+), 98 deletions(-) create mode 100644 cmd/controller/app/database/mongodb/compute.go create mode 100644 cmd/deployer/app/register.go create mode 100644 cmd/deployer/cmd/root_test.go diff --git a/api/computes_api.partials.yml b/api/computes_api.partials.yml index 70cf78821..d6d773976 100644 --- a/api/computes_api.partials.yml +++ b/api/computes_api.partials.yml @@ -14,7 +14,7 @@ # # SPDX-License-Identifier: Apache-2.0 -/computes/{compute}: +/computes: ######################################## # Register a new compute cluster ######################################## @@ -23,15 +23,6 @@ summary: Register a new compute cluster tags: - computes - parameters: - - name: compute - description: deployer id of compute cluster - in: path - schema: - type: string - style: simple - explode: false - required: true requestBody: content: application/json: @@ -49,6 +40,7 @@ $ref: '#/components/schemas/Error' description: unexpected error +/computes/{compute}: ######################################## # Get the status of a compute cluster ######################################## diff --git a/api/computes_components.partials.yml b/api/computes_components.partials.yml index b06dc2eff..b26211822 100644 --- a/api/computes_components.partials.yml +++ b/api/computes_components.partials.yml @@ -21,8 +21,6 @@ ComputeSpec: description: Compute specification type: object properties: - deployerId: - type: string adminId: type: string region: @@ -31,14 +29,11 @@ ComputeSpec: type: string apiKey: type: string - required: - - deployerId example: - deployerId: "60d0d66716af13b787d9ef0a" adminId: "admin1" region: "us-east" - computeId: "1" apiKey: "892nsfd2ds82" + computeId: "62cdd50e5e28e1473c3768fc" ######################################## # Compute status @@ -47,27 +42,19 @@ ComputeStatus: description: Cluster compute status type: object properties: - deployerId: - type: string - state: - $ref: '#/components/schemas/ComputeState' registeredAt: type: string format: date-time + state: + $ref: '#/components/schemas/ComputeState' updatedAt: type: string format: date-time - region: - type: string - computeId: - type: string - computeSpec: - type: string + spec: + $ref: '#/components/schemas/ComputeSpec' required: - - deployerId - state - - region - - computeId + - spec ######################################## # Compute state diff --git a/cmd/controller/app/controller.go b/cmd/controller/app/controller.go index 3740ada89..b67eaf7c1 100644 --- a/cmd/controller/app/controller.go +++ b/cmd/controller/app/controller.go @@ -102,6 +102,7 @@ func (c *Controller) serveRestApi() { openapi.NewDesignCodesApiController(controller.NewDesignCodesApiService(c.dbService)), openapi.NewDesignSchemasApiController(controller.NewDesignSchemasApiService(c.dbService)), openapi.NewJobsApiController(controller.NewJobsApiService(c.dbService, c.jobEventQ, c.jobBuilder)), + openapi.NewComputesApiController(controller.NewComputesApiService(c.dbService)), } router := openapi.NewRouter(apiRouters...) diff --git a/cmd/controller/app/database/db_interfaces.go b/cmd/controller/app/database/db_interfaces.go index b657734c7..80a155fbe 100644 --- a/cmd/controller/app/database/db_interfaces.go +++ b/cmd/controller/app/database/db_interfaces.go @@ -30,6 +30,7 @@ type DBService interface { DesignService JobService TaskService + ComputeService } // DatasetService is an interface that defines a collection of APIs related to dataset @@ -80,3 +81,8 @@ type TaskService interface { UpdateTaskStateByFilter(string, openapi.JobState, map[string]interface{}) error UpdateTaskStatus(string, string, openapi.TaskStatus) error } + +// ComputeService is an interface that defines a collection of APIs related to computes +type ComputeService interface { + RegisterCompute(openapi.ComputeSpec) (openapi.ComputeStatus, error) +} diff --git a/cmd/controller/app/database/mongodb/compute.go b/cmd/controller/app/database/mongodb/compute.go new file mode 100644 index 000000000..22b8a3940 --- /dev/null +++ b/cmd/controller/app/database/mongodb/compute.go @@ -0,0 +1,91 @@ +// Copyright 2022 Cisco Systems, Inc. and its affiliates +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package mongodb + +import ( + "context" + "fmt" + "time" + + "github.com/cisco-open/flame/pkg/openapi" + "github.com/cisco-open/flame/pkg/util" + "go.mongodb.org/mongo-driver/bson" + "go.uber.org/zap" +) + +// RegisterCompute creates a new cluster compute specification and returns ComputeStatus +func (db *MongoService) RegisterCompute(computeSpec openapi.ComputeSpec) (openapi.ComputeStatus, error) { + // override userId in jobSpec to prevent an incorrect record in the db + result, err := db.jobCollection.InsertOne(context.TODO(), computeSpec) + if err != nil { + zap.S().Warnf("Failed to create a new job: %v", err) + + return openapi.ComputeStatus{}, ErrorCheck(err) + } + + updatedSpec := computeSpec + updatedSpec.ComputeId = GetStringID(result.InsertedID) + + computeStatus := openapi.ComputeStatus{ + State: openapi.UP, + Spec: updatedSpec, + RegisteredAt: time.Now(), + } + + err = db.UpdateComputeStatus(computeStatus.Spec.ComputeId, computeStatus) + if err != nil { + zap.S().Warnf("Failed to update compute status: %v", err) + + return openapi.ComputeStatus{}, ErrorCheck(err) + } + + zap.S().Infof("Successfully registered a new compute cluster with computeID %s", computeStatus.Spec.ComputeId) + + return computeStatus, err +} + +// UpdateJobStatus update Job's status +func (db *MongoService) UpdateComputeStatus(computeId string, computeStatus openapi.ComputeStatus) error { + dateKey := "" + switch computeStatus.State { + case openapi.UP: + fallthrough + case openapi.DOWN: + fallthrough + case openapi.MAINTENANCE: + dateKey = updatedAt + + default: + return fmt.Errorf("unknown state: %s", computeStatus.State) + } + + setElements := bson.M{util.DBFieldId: computeId, "state": computeStatus.State} + if dateKey != "" { + setElements[dateKey] = time.Now() + } + + filter := bson.M{util.DBFieldMongoID: ConvertToObjectID(computeId)} + update := bson.M{"$set": setElements} + + updatedDoc := openapi.ComputeStatus{} + err := db.jobCollection.FindOneAndUpdate(context.TODO(), filter, update).Decode(&updatedDoc) + if err != nil { + return ErrorCheck(err) + } + + return nil +} diff --git a/cmd/controller/app/database/mongodb/helpers.go b/cmd/controller/app/database/mongodb/helpers.go index 4ad425c14..2472d7722 100644 --- a/cmd/controller/app/database/mongodb/helpers.go +++ b/cmd/controller/app/database/mongodb/helpers.go @@ -29,6 +29,7 @@ import ( const ( latestVersion = "latest" unknownVersion = "unknown" + updatedAt = "updatedat" ) /* diff --git a/cmd/controller/app/database/mongodb/job.go b/cmd/controller/app/database/mongodb/job.go index 938ec7536..9c25723b0 100644 --- a/cmd/controller/app/database/mongodb/job.go +++ b/cmd/controller/app/database/mongodb/job.go @@ -206,7 +206,7 @@ func (db *MongoService) UpdateJobStatus(userId string, jobId string, jobStatus o dateKey = "startedat" case openapi.APPLYING: - dateKey = "updatedat" + dateKey = updatedAt case openapi.FAILED: fallthrough diff --git a/cmd/deployer/app/appserver.go b/cmd/deployer/app/appserver.go index 4879f7a48..61c413e32 100644 --- a/cmd/deployer/app/appserver.go +++ b/cmd/deployer/app/appserver.go @@ -1 +1,17 @@ +// Copyright 2022 Cisco Systems, Inc. and its affiliates +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + package app diff --git a/cmd/deployer/app/register.go b/cmd/deployer/app/register.go new file mode 100644 index 000000000..6915f7862 --- /dev/null +++ b/cmd/deployer/app/register.go @@ -0,0 +1,80 @@ +// Copyright 2022 Cisco Systems, Inc. and its affiliates +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package app + +import ( + "crypto/tls" + "net/http" + "os" + + "github.com/cisco-open/flame/pkg/openapi" + "github.com/cisco-open/flame/pkg/restapi" + "go.uber.org/zap" +) + +type ComputeResource struct { + apiserverEp string + name string + spec openapi.ComputeSpec + registered bool +} + +func NewCompute(apiserverEp string, computeSpec openapi.ComputeSpec, bInsecure bool, bPlain bool) (*ComputeResource, error) { + name, err := os.Hostname() + if err != nil { + return nil, err + } + + if bPlain { + } else { + tlsCfg := &tls.Config{} + if bInsecure { + zap.S().Warn("Warning: allow insecure connection\n") + + tlsCfg.InsecureSkipVerify = true + http.DefaultTransport.(*http.Transport).TLSClientConfig = tlsCfg + } + } + + compute := &ComputeResource{ + apiserverEp: apiserverEp, + name: name, + spec: computeSpec, + registered: false, + } + + return compute, nil +} + +func (compute *ComputeResource) RegisterNewCompute() error { + // construct URL + uriMap := map[string]string{} + url := restapi.CreateURL(compute.apiserverEp, restapi.RegisterComputeEndpoint, uriMap) + + code, resp, err := restapi.HTTPPost(url, compute.spec, "application/json") + if err != nil || restapi.CheckStatusCode(code) != nil { + zap.S().Errorf("Failed to register compute, sent computeSpec: %v, resp: %v, code: %d, err: %v", + compute.spec, string(resp), code, err) + return err + } + + zap.S().Infof("Success in registering new compute, sent obj: %v, resp: %v, code: %d", + compute.spec, string(resp), code) + compute.registered = true + + return nil +} diff --git a/cmd/deployer/app/service.go b/cmd/deployer/app/service.go index 4879f7a48..61c413e32 100644 --- a/cmd/deployer/app/service.go +++ b/cmd/deployer/app/service.go @@ -1 +1,17 @@ +// Copyright 2022 Cisco Systems, Inc. and its affiliates +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + package app diff --git a/cmd/deployer/cmd/root.go b/cmd/deployer/cmd/root.go index 1d619dd05..e5f2461a9 100644 --- a/cmd/deployer/cmd/root.go +++ b/cmd/deployer/cmd/root.go @@ -1 +1,91 @@ +// Copyright 2022 Cisco Systems, Inc. and its affiliates +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + package cmd + +import ( + "fmt" + "strings" + + "github.com/cisco-open/flame/cmd/deployer/app" + "github.com/cisco-open/flame/pkg/openapi" + "github.com/cisco-open/flame/pkg/util" + "github.com/spf13/cobra" +) + +const ( + argApiserver = "apiserver" + + optionInsecure = "insecure" + optionPlain = "plain" +) + +var rootCmd = &cobra.Command{ + Use: util.Deployer, + Short: util.ProjectName + " Deployer", + RunE: func(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + apiserver, err := flags.GetString(argApiserver) + if err != nil { + return err + } + if len(strings.Split(apiserver, ":")) != util.NumTokensInRestEndpoint { + return fmt.Errorf("incorrect format for apiserver endpoint: %s", apiserver) + } + + bInsecure, _ := flags.GetBool(optionInsecure) + bPlain, _ := flags.GetBool(optionPlain) + + if bInsecure && bPlain { + err = fmt.Errorf("options --%s and --%s are incompatible; enable one of them", optionInsecure, optionPlain) + return err + } + + // Pass more values to NewCompute using + computeSpec := openapi.ComputeSpec{ + AdminId: "admin1", + Region: "us-east", + ComputeId: "1", + ApiKey: "temp", + } + + compute, err := app.NewCompute(apiserver, computeSpec, bInsecure, bPlain) + if err != nil { + return err + } + + if err := compute.RegisterNewCompute(); err != nil { + err = fmt.Errorf("error from RegisterNewCompute: %s", err) + return err + } + + select {} + }, +} + +func init() { + defaultApiServerEp := fmt.Sprintf("http://0.0.0.0:%d", util.ApiServerRestApiPort) + rootCmd.Flags().StringP(argApiserver, "a", defaultApiServerEp, "API server endpoint") + rootCmd.MarkFlagRequired(argApiserver) + + rootCmd.PersistentFlags().Bool(optionInsecure, false, "Allow insecure connection") + rootCmd.PersistentFlags().Bool(optionPlain, false, "Allow unencrypted connection") +} + +func Execute() error { + return rootCmd.Execute() +} diff --git a/cmd/deployer/cmd/root_test.go b/cmd/deployer/cmd/root_test.go new file mode 100644 index 000000000..51f29ba3c --- /dev/null +++ b/cmd/deployer/cmd/root_test.go @@ -0,0 +1,43 @@ +// Copyright 2022 Cisco Systems, Inc. and its affiliates +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "testing" + + "github.com/cisco-open/flame/cmd/deployer/app" + "github.com/cisco-open/flame/pkg/openapi" + "github.com/stretchr/testify/assert" +) + +var ( + testComputeSpec = openapi.ComputeSpec{ + AdminId: "admin1", + Region: "us-east", + ApiKey: "temp", + } + + apiargApiserver = "apiserver.flame.test" + bInsecure = true + bPlain = false +) + +func TestNewCompute(t *testing.T) { + compute, err := app.NewCompute(apiargApiserver, testComputeSpec, bInsecure, bPlain) + assert.NotNil(t, compute) + assert.Nil(t, err) +} diff --git a/cmd/deployer/main.go b/cmd/deployer/main.go index 5f0a1872f..e5175a1a5 100644 --- a/cmd/deployer/main.go +++ b/cmd/deployer/main.go @@ -1,14 +1,35 @@ +// Copyright 2022 Cisco Systems, Inc. and its affiliates +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + package main -import "time" +import ( + "os" + + "github.com/cisco-open/flame/cmd/deployer/cmd" + "github.com/cisco-open/flame/pkg/util" + "go.uber.org/zap" +) func main() { - go forever() - select {} // block forever -} + loggerMgr := util.InitZapLog(util.Deployer) + zap.ReplaceGlobals(loggerMgr) + defer loggerMgr.Sync() -func forever() { - for { - time.Sleep(time.Second) + if err := cmd.Execute(); err != nil { + os.Exit(1) } } diff --git a/docs/03-fiab.md b/docs/03-fiab.md index cb37a39a5..d48239b8f 100644 --- a/docs/03-fiab.md +++ b/docs/03-fiab.md @@ -65,7 +65,7 @@ The fiab env was tested under Archlinux in a x86 machine. #### step 1: Installing minikube We install the latest minikube stable release on x86-64 Linux using binary downloaded from [here](https://minikube.sigs.k8s.io/docs/start/). -#### step 2: Installing kubectl, helm and jq. +#### step 2: Installing kubectl, helm and jq To install [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/#install-using-other-package-management). ```bash sudo snap install kubectl --classic @@ -80,7 +80,7 @@ To install jq. sudo apt update sudo apt install -y jq ``` -#### step3: Install docker. +#### step3: Install docker 1. Update the apt package index and install packages to allow apt to use a repository over HTTPS. ```bash sudo apt-get update @@ -90,18 +90,18 @@ sudo apt-get install \ gnupg \ lsb-release ``` -2. Add Docker’s official GPG key: +2. Add Docker’s official GPG key. ```bash sudo mkdir -p /etc/apt/keyrings curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg ``` -3. Use the following command to set up the repository: +3. Use the following command to set up the repository. ```bash echo \ "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null ``` -4. Update the apt package index, and install the latest version of Docker Engine +4. Update the apt package index, and install the latest version of Docker Engine. ```bash sudo apt-get update sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin @@ -121,6 +121,8 @@ For example, ```bash minikube start --cpus 4 --memory 4096m --disk-size 100gb ``` +When `docker` driver is in use, run ```minikube config set driver docker``` to make docker driver default. These changes will take effect upon a minikube delete and then a minikube start. + We recommend a disk space of 100GB to allow sufficient disk space to store the flame container images and other images in the minikube VM. Next, `ingress` and `ingress` addons need to be installed with the following command: diff --git a/fiab/helm-chart/templates/apiserver-ingress.yaml b/fiab/helm-chart/templates/apiserver-ingress.yaml index c3f028160..7d97522bf 100644 --- a/fiab/helm-chart/templates/apiserver-ingress.yaml +++ b/fiab/helm-chart/templates/apiserver-ingress.yaml @@ -1,3 +1,19 @@ +# Copyright 2022 Cisco Systems, Inc. and its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + apiVersion: {{ .Values.ingress.apiVersion }} kind: Ingress metadata: diff --git a/fiab/helm-chart/templates/controller-job-configmap.yaml b/fiab/helm-chart/templates/controller-job-configmap.yaml index 9f93af6d8..1f6636a4b 100644 --- a/fiab/helm-chart/templates/controller-job-configmap.yaml +++ b/fiab/helm-chart/templates/controller-job-configmap.yaml @@ -1,3 +1,19 @@ +# Copyright 2022 Cisco Systems, Inc. and its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + --- apiVersion: v1 kind: ConfigMap diff --git a/fiab/helm-chart/templates/deployer-deployment.yaml b/fiab/helm-chart/templates/deployer-deployment.yaml index fdf3ff6e7..592b40d23 100644 --- a/fiab/helm-chart/templates/deployer-deployment.yaml +++ b/fiab/helm-chart/templates/deployer-deployment.yaml @@ -1,3 +1,19 @@ +# Copyright 2022 Cisco Systems, Inc. and its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + apiVersion: apps/v1 kind: Deployment metadata: @@ -15,7 +31,10 @@ spec: app: {{ .Release.Name }}-deployer spec: containers: - - args: [] + - args: + - --apiserver + - "https://{{ .Values.frontDoorUrl.apiserver }}:443" + - --insecure command: ["/usr/bin/deployer"] image: {{ .Values.imageName }}:{{ .Values.imageTag }} imagePullPolicy: IfNotPresent diff --git a/fiab/helm-chart/templates/mlflow-ingress.yaml b/fiab/helm-chart/templates/mlflow-ingress.yaml index d43fec46a..9aee08119 100644 --- a/fiab/helm-chart/templates/mlflow-ingress.yaml +++ b/fiab/helm-chart/templates/mlflow-ingress.yaml @@ -1,3 +1,19 @@ +# Copyright 2022 Cisco Systems, Inc. and its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + apiVersion: {{ .Values.ingress.apiVersion }} kind: Ingress metadata: diff --git a/fiab/helm-chart/templates/notifier-ingress.yaml b/fiab/helm-chart/templates/notifier-ingress.yaml index db543f76d..20fdb7e51 100644 --- a/fiab/helm-chart/templates/notifier-ingress.yaml +++ b/fiab/helm-chart/templates/notifier-ingress.yaml @@ -1,3 +1,19 @@ +# Copyright 2022 Cisco Systems, Inc. and its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + apiVersion: {{ .Values.ingress.apiVersion }} kind: Ingress metadata: diff --git a/pkg/openapi/api.go b/pkg/openapi/api.go index 8eede363b..9b8031a01 100644 --- a/pkg/openapi/api.go +++ b/pkg/openapi/api.go @@ -106,7 +106,7 @@ type ComputesApiServicer interface { DeleteCompute(context.Context, string) (ImplResponse, error) GetComputeConfig(context.Context, string) (ImplResponse, error) GetComputeStatus(context.Context, string) (ImplResponse, error) - RegisterCompute(context.Context, string, ComputeSpec) (ImplResponse, error) + RegisterCompute(context.Context, ComputeSpec) (ImplResponse, error) UpdateCompute(context.Context, string, ComputeSpec) (ImplResponse, error) } diff --git a/pkg/openapi/api_computes.go b/pkg/openapi/api_computes.go index fb793f810..d400b8adb 100644 --- a/pkg/openapi/api_computes.go +++ b/pkg/openapi/api_computes.go @@ -87,7 +87,7 @@ func (c *ComputesApiController) Routes() Routes { { "RegisterCompute", strings.ToUpper("Post"), - "/computes/{compute}", + "/computes", c.RegisterCompute, }, { @@ -146,9 +146,6 @@ func (c *ComputesApiController) GetComputeStatus(w http.ResponseWriter, r *http. // RegisterCompute - Register a new compute cluster func (c *ComputesApiController) RegisterCompute(w http.ResponseWriter, r *http.Request) { - params := mux.Vars(r) - computeParam := params["compute"] - computeSpecParam := ComputeSpec{} d := json.NewDecoder(r.Body) d.DisallowUnknownFields() @@ -160,7 +157,7 @@ func (c *ComputesApiController) RegisterCompute(w http.ResponseWriter, r *http.R c.errorHandler(w, r, err, nil) return } - result, err := c.service.RegisterCompute(r.Context(), computeParam, computeSpecParam) + result, err := c.service.RegisterCompute(r.Context(), computeSpecParam) // If an error occurred, encode the error with the status code if err != nil { c.errorHandler(w, r, err, &result) diff --git a/pkg/openapi/apiserver/api_computes_service.go b/pkg/openapi/apiserver/api_computes_service.go index 9a140c71e..3bfc317c7 100644 --- a/pkg/openapi/apiserver/api_computes_service.go +++ b/pkg/openapi/apiserver/api_computes_service.go @@ -28,9 +28,13 @@ package apiserver import ( "context" "errors" + "fmt" "net/http" "github.com/cisco-open/flame/pkg/openapi" + "github.com/cisco-open/flame/pkg/restapi" + "github.com/cisco-open/flame/pkg/util" + "go.uber.org/zap" ) // ComputesApiService is a service that implements the logic for the ComputesApiServicer @@ -96,22 +100,35 @@ func (s *ComputesApiService) GetComputeStatus(ctx context.Context, compute strin } // RegisterCompute - Register a new compute cluster -func (s *ComputesApiService) RegisterCompute(ctx context.Context, compute string, - computeSpec openapi.ComputeSpec) (openapi.ImplResponse, error) { - // TODO - update RegisterCompute with the required logic for this service method. - // Add api_computes_service.go to the .openapi-generator-ignore to avoid overwriting - // this service implementation when updating open api generation. - - //TODO: Uncomment the next line to return response Response(200, {}) or use other options such as http.Ok ... - //return Response(200, nil),nil - - //TODO: Uncomment the next line to return response Response(401, {}) or use other options such as http.Ok ... - //return Response(401, nil),nil - - //TODO: Uncomment the next line to return response Response(0, Error{}) or use other options such as http.Ok ... - //return Response(0, Error{}), nil - - return openapi.Response(http.StatusNotImplemented, nil), errors.New("RegisterCompute method not implemented") +func (s *ComputesApiService) RegisterCompute(ctx context.Context, computeSpec openapi.ComputeSpec) (openapi.ImplResponse, error) { + zap.S().Debugf("Registering new compute cluster with computeSpec: %v", computeSpec) + + // create controller request + uriMap := map[string]string{} + url := restapi.CreateURL(HostEndpoint, restapi.RegisterComputeEndpoint, uriMap) + + // send post request + code, resp, err := restapi.HTTPPost(url, computeSpec, "application/json") + if err != nil { + errMsg := fmt.Sprintf("failed to send post to controller: %v", err) + zap.S().Errorf(errMsg) + return openapi.Response(http.StatusInternalServerError, nil), err + } + + if err = restapi.CheckStatusCode(code); err != nil { + return openapi.Response(code, nil), fmt.Errorf("%s", string(resp)) + } + + // TODO everything went well and response is a unique compute cluster id + computeStatus := openapi.ComputeStatus{} + err = util.ByteToStruct(resp, &computeStatus) + if err != nil { + errMsg := fmt.Sprintf("failed to construct response message: %v", err) + zap.S().Errorf(errMsg) + return openapi.Response(http.StatusInternalServerError, nil), fmt.Errorf(errMsg) + } + + return openapi.Response(http.StatusCreated, computeStatus), nil } // UpdateCompute - Update a compute cluster's specification diff --git a/pkg/openapi/controller/api_computes_service.go b/pkg/openapi/controller/api_computes_service.go index edc7f35d4..b2ad56103 100644 --- a/pkg/openapi/controller/api_computes_service.go +++ b/pkg/openapi/controller/api_computes_service.go @@ -30,6 +30,7 @@ import ( "errors" "net/http" + "github.com/cisco-open/flame/cmd/controller/app/database" "github.com/cisco-open/flame/pkg/openapi" ) @@ -37,11 +38,14 @@ import ( // This service should implement the business logic for every endpoint for the ComputesApi API. // Include any external packages or services that will be required by this service. type ComputesApiService struct { + dbService database.DBService } // NewComputesApiService creates a default api service -func NewComputesApiService() openapi.ComputesApiServicer { - return &ComputesApiService{} +func NewComputesApiService(dbService database.DBService) openapi.ComputesApiServicer { + return &ComputesApiService{ + dbService: dbService, + } } // DeleteCompute - Delete compute cluster specification @@ -96,17 +100,13 @@ func (s *ComputesApiService) GetComputeStatus(ctx context.Context, compute strin } // RegisterCompute - Register a new compute cluster -func (s *ComputesApiService) RegisterCompute(ctx context.Context, compute string, - computeSpec openapi.ComputeSpec) (openapi.ImplResponse, error) { - // TODO - update RegisterCompute with the required logic for this service method. +func (s *ComputesApiService) RegisterCompute(ctx context.Context, computeSpec openapi.ComputeSpec) (openapi.ImplResponse, error) { + // TODO - update GetComputeStatus with the required logic for this service method. // Add api_computes_service.go to the .openapi-generator-ignore to avoid overwriting // this service implementation when updating open api generation. - //TODO: Uncomment the next line to return response Response(200, {}) or use other options such as http.Ok ... - //return Response(200, nil),nil - - //TODO: Uncomment the next line to return response Response(401, {}) or use other options such as http.Ok ... - //return Response(401, nil),nil + //TODO: Uncomment the next line to return response Response(200, ComputeStatus{}) or use other options such as http.Ok ... + //return Response(200, ComputeStatus{}), nil //TODO: Uncomment the next line to return response Response(0, Error{}) or use other options such as http.Ok ... //return Response(0, Error{}), nil diff --git a/pkg/openapi/model_compute_spec.go b/pkg/openapi/model_compute_spec.go index 5f222e70c..d548679bf 100644 --- a/pkg/openapi/model_compute_spec.go +++ b/pkg/openapi/model_compute_spec.go @@ -27,8 +27,6 @@ package openapi // ComputeSpec - Compute specification type ComputeSpec struct { - DeployerId string `json:"deployerId"` - AdminId string `json:"adminId,omitempty"` Region string `json:"region,omitempty"` @@ -40,15 +38,6 @@ type ComputeSpec struct { // AssertComputeSpecRequired checks if the required fields are not zero-ed func AssertComputeSpecRequired(obj ComputeSpec) error { - elements := map[string]interface{}{ - "deployerId": obj.DeployerId, - } - for name, el := range elements { - if isZero := IsZeroValue(el); isZero { - return &RequiredError{Field: name} - } - } - return nil } diff --git a/pkg/openapi/model_compute_status.go b/pkg/openapi/model_compute_status.go index 12643789c..59b99935c 100644 --- a/pkg/openapi/model_compute_status.go +++ b/pkg/openapi/model_compute_status.go @@ -31,28 +31,20 @@ import ( // ComputeStatus - Cluster compute status type ComputeStatus struct { - DeployerId string `json:"deployerId"` + RegisteredAt time.Time `json:"registeredAt,omitempty"` State ComputeState `json:"state"` - RegisteredAt time.Time `json:"registeredAt,omitempty"` - UpdatedAt time.Time `json:"updatedAt,omitempty"` - Region string `json:"region"` - - ComputeId string `json:"computeId"` - - ComputeSpec string `json:"computeSpec,omitempty"` + Spec ComputeSpec `json:"spec"` } // AssertComputeStatusRequired checks if the required fields are not zero-ed func AssertComputeStatusRequired(obj ComputeStatus) error { elements := map[string]interface{}{ - "deployerId": obj.DeployerId, - "state": obj.State, - "region": obj.Region, - "computeId": obj.ComputeId, + "state": obj.State, + "spec": obj.Spec, } for name, el := range elements { if isZero := IsZeroValue(el); isZero { @@ -60,6 +52,9 @@ func AssertComputeStatusRequired(obj ComputeStatus) error { } } + if err := AssertComputeSpecRequired(obj.Spec); err != nil { + return err + } return nil } diff --git a/pkg/restapi/restapi.go b/pkg/restapi/restapi.go index dbd0c602c..da3f12107 100644 --- a/pkg/restapi/restapi.go +++ b/pkg/restapi/restapi.go @@ -71,6 +71,13 @@ const ( // Keys for task GetTaskEndpoint = "GET_TASK" UpdateTaskStatusEndPoint = "UPDATE_TASK_STATUS" + + // Keys for computes endpoints + RegisterComputeEndpoint = "REGISTER_COMPUTE" + GetComputeStatusEndpoint = "GET_COMPUTE_STATUS" + UpdateComputeEndpoint = "UPDATE_COMPUTE" + DeleteComputeEndpoint = "DELETE_COMPUTE" + GetComputeConfigEndpoint = "GET_COMPUTE_CONFIG" ) var URI = map[string]string{ @@ -111,6 +118,13 @@ var URI = map[string]string{ // Task GetTaskEndpoint: "/jobs/{{.jobId}}/{{.taskId}}/task/?key={{.key}}", UpdateTaskStatusEndPoint: "/jobs/{{.jobId}}/{{.taskId}}/task/status", + + // Computes + RegisterComputeEndpoint: "/computes", + GetComputeStatusEndpoint: "/computes/{{.compute}}", + UpdateComputeEndpoint: "/computes/{{.compute}}", + DeleteComputeEndpoint: "/computes/{{.compute}}", + GetComputeConfigEndpoint: "/computes/{{.compute}}/config", } func FromTemplate(skeleton string, inputMap map[string]string) (string, error) {