epic: Handle role group associations into control plane and sdk (#366)

Based on the new schema requirements we introduce, we added the groupAssociation information which specifies what channel and group the role is associated with. Based on these we create tasks which generates a config file that meats the new format that is processed by the SDK.

In this PR we also moved the dataSpec field data to dataSpec.json and modelSpec field data to modelSpec.json files to avoid polluting the job.json file

.github/workflows/presubmits.yml

@@ -43,6 +43,9 @@ jobs:
       with:
         go-version: 1.17.x
 
+    - name: Start MongoDB
+      uses: supercharge/mongodb-github-action@1.8.0
+
     - name: Test
       run: go test -v -race -covermode=atomic -coverprofile=coverage.out ./...
     - name: Upload codecoverage

api/job_components.partials.yml

@@ -19,6 +19,17 @@
 ########################################
 JobSpec:
   description: Job specification
+  allOf:
+    - $ref: '#/components/schemas/BasicJobInfo'
+    - type: object
+      properties:
+        dataSpec:
+          $ref: '#/components/schemas/DataSpec'
+        modelSpec:
+          $ref: '#/components/schemas/ModelSpec'
+
+BasicJobInfo:
+  description: Basic Job specification
   type: object
   properties:
     id:
@@ -31,12 +42,6 @@ JobSpec:
       type: string
     codeVersion:
       type: string
-    dataSpec:
-      $ref: '#/components/schemas/DataSpec'
-    optimizer:
-      $ref: '#/components/schemas/Optimizer'
-    selector:
-      $ref: '#/components/schemas/Selector'
     priority:
       $ref: '#/components/schemas/JobPriority'
     backend:
@@ -45,14 +50,6 @@ JobSpec:
       type: integer
       format: int32
       default: 600 # 600s = 10min
-    baseModel:
-      $ref: '#/components/schemas/BaseModel'
-    hyperparameters:
-      type: object
-    dependencies:
-      type: array
-      items:
-        type: string
   required:
     - designId
     - schemaVersion
@@ -62,21 +59,9 @@ JobSpec:
     designId: "60d0d66716af12b787d9ef0a"
     schemaVersion: "1"
     codeVersion: "1"
-    selector:
-      sort: random
-      kwargs:
-        k: 10
-    prority: "low"
+    priority: "low"
     backend: "mqtt"
     maxRunTime: 600
-    baseModel:
-        name: mnist-624d9fc43395791
-        version: 2
-    hyperparameters:
-      - rounds: 5
-      - runningRate: 0.01
-    dependencies:
-      - numpy >= 1.2.0
 
 JobPriority:
   enum:
@@ -90,43 +75,85 @@ CommBackend:
     - p2p
 
 ########################################
-# Base Model
+# Data Spec
 ########################################
-BaseModel:
+DataSpec:
+  description: Dataset specification
   type: object
   properties:
-    name:
-      type: string
-    version:
-      type: integer
-      format: int32
+    fromUser:
+      $ref: '#/components/schemas/FromUser'
+    fromSystem:
+      $ref: '#/components/schemas/FromSystem'
   example:
-    name: mnist-624d9fc43395791
-    version: 2
+    fromUser:
+      default: 1 # there is one user-fed dataset under default realm
+      uk: 2 # there are two user-fed datasets (hence two compute nodes) under uk realm
+      us/foo: 3 # there are three user-fed datasets under us/foo realm
+    fromSystem:
+      trainer:
+        "default/eu":
+          - 61609290fa724deafdb2a4fa # id of dataset registered in the flame system
+          - 61609290fa724deafdb2a4fb # id of dataset registered in the flame system
+        "default/us":
+          - 61609290fa724deafdb2a4fc # id of dataset registered in the flame system
 
 ########################################
-# Data Spec
+# FromUser
 ########################################
-DataSpec:
+FromUser:
+  # an object is a key-value pair of realm and count
+  type: object
+  additionalProperties:
+    type: integer
+    format: int32
+
+########################################
+# FromSystem
+########################################
+FromSystem:
+  type: object
+  additionalProperties:
+    type: object
+    additionalProperties:
+      type: array
+      items:
+        type: string
+
+########################################
+# Model Spec
+########################################
+ModelSpec:
+  description: Model specification
   type: object
   properties:
-    fromUser:
-      # an object is a key-value pair of realm and count
+    baseModel:
+      $ref: '#/components/schemas/BaseModel'
+    optimizer:
+      $ref: '#/components/schemas/Optimizer'
+    selector:
+      $ref: '#/components/schemas/Selector'
+    hyperparameters:
       type: object
-      additionalProperties:
-        type: integer
-        format: int32
-    fromSystem:
+    dependencies:
       type: array
       items:
         type: string
+
+########################################
+# Base Model
+########################################
+BaseModel:
+  type: object
+  properties:
+    name:
+      type: string
+    version:
+      type: integer
+      format: int32
   example:
-    fromUser:
-      default: 1 # there is one user-fed dataset under default realm
-      uk: 2 # there are two user-fed datasets (hence two compute nodes) under uk realm
-      us/foo: 3 # there are three user-fed datasets under us/foo realm
-    fromSystem:
-      - 61609290fa724deafdb2a4fa # id of dataset registered in the flame system
+    name: mnist-624d9fc43395791
+    version: 2
 
 ########################################
 # Optimizer
@@ -274,4 +301,4 @@ TaskStatus:
       type: string
     timestamp:
       type: string
-      format: date-time
+      format: date-time

cmd/controller/app/database/mongodb/service.go

@@ -93,6 +93,10 @@ func NewMongoService(uri string) (*MongoService, error) {
 	}
 	zap.S().Infof("Successfully connected to database and pinged")
 
+	return NewMongoServiceWithClient(ctx, client)
+}
+
+func NewMongoServiceWithClient(ctx context.Context, client *mongo.Client) (*MongoService, error) {
 	db := client.Database(DatabaseName)
 	mongoDB := &MongoService{
 		client:               client,