Merge pull request #1 from elqurio/development

Make sdk config backwards compatible. (cisco-open#355)

lib/python/flame/config.py

@@ -16,8 +16,8 @@
 """Config parser."""
 
 from enum import Enum
-from pydantic import Field
 import typing as t
+from pydantic import Field
 from pydantic import BaseModel as pydBaseModel
 import json
 
@@ -82,6 +82,8 @@ class Registry(FlameSchema):
 class Selector(FlameSchema):
     sort: SelectorType = Field(default=SelectorType.DEFAULT)
     kwargs: dict = Field(default={})
+
+
 class Optimizer(FlameSchema):
     sort: OptimizerType = Field(default=OptimizerType.DEFAULT)
     kwargs: dict = Field(default={})
@@ -97,6 +99,7 @@ class Hyperparameters(FlameSchema):
     learning_rate: t.Optional[float] = Field(alias="learningRate")
     rounds: int
     epochs: int
+    aggregation_goal: t.Optional[int] = Field(alias="aggGoal", default=None)
 
 
 class Groups(FlameSchema):
@@ -148,6 +151,12 @@ class ChannelConfigs(FlameSchema):
 
 
 class Config(FlameSchema):
+    def __init__(self, config_path: str):
+        raw_config = read_config(config_path)
+        transformed_config = transform_config(raw_config)
+
+        super().__init__(**transformed_config)
+
     role: str
     realm: str
     task: t.Optional[str] = Field(default="local")
@@ -174,8 +183,7 @@ def read_config(filename: str) -> dict:
         return json.loads(f.read())
 
 
-def load_config(filename: str) -> Config:
-    raw_config = read_config(filename)
+def transform_config(raw_config: dict) -> dict:
     config_data = {
         "role": raw_config["role"],
         "realm": raw_config["realm"],

lib/python/flame/examples/adult/aggregator/main.py

@@ -19,7 +19,7 @@
 
 import torch
 import torch.nn as nn
-from flame.config import Config, load_config
+from flame.config import Config
 from flame.mode.horizontal.top_aggregator import TopAggregator
 
 logger = logging.getLogger(__name__)
@@ -88,7 +88,7 @@ def evaluate(self) -> None:
 
     args = parser.parse_args()
 
-    config = load_config(args.config)
+    config = Config(args.config)
 
     t = PyTorchAdultAggregator(config)
     t.compose()

lib/python/flame/examples/adult/trainer/main.py

@@ -24,7 +24,7 @@
 import torch.optim as optim
 from flame.common.constants import DATA_FOLDER_PATH
 from flame.common.util import install_packages
-from flame.config import Config, load_config
+from flame.config import Config
 from flame.mode.horizontal.trainer import Trainer
 
 install_packages(['scikit-learn'])
@@ -149,7 +149,7 @@ def evaluate(self) -> None:
 
     args = parser.parse_args()
 
-    config = load_config(args.config)
+    config = Config(args.config)
 
     t = PyTorchAdultTrainer(config)
     t.compose()

lib/python/flame/examples/dist_mnist/trainer/keras/main.py

@@ -21,7 +21,7 @@
 from statistics import mean
 
 import numpy as np
-from flame.config import Config, load_config
+from flame.config import Config
 from flame.mode.distributed.trainer import Trainer
 from tensorflow import keras
 from tensorflow.keras import layers
@@ -132,7 +132,7 @@ def evaluate(self) -> None:
 
     args = parser.parse_args()
 
-    config = load_config(args.config)
+    config = Config(args.config)
 
     t = KerasMnistTrainer(config)
     t.compose()

lib/python/flame/examples/dist_mnist/trainer/pytorch/main.py

@@ -26,7 +26,7 @@
 import torch.nn.functional as F
 import torch.optim as optim
 import torch.utils.data as data_utils
-from flame.config import Config, load_config
+from flame.config import Config
 from flame.mode.distributed.trainer import Trainer
 from torchvision import datasets, transforms
 
@@ -145,7 +145,7 @@ def evaluate(self) -> None:
 
     args = parser.parse_args()
 
-    config = load_config(args.config)
+    config = Config(args.config)
 
     t = PyTorchMnistTrainer(config)
     t.compose()

lib/python/flame/examples/hier_mnist/middle_aggregator/main.py

@@ -17,7 +17,7 @@
 
 import logging
 
-from flame.config import Config, load_config
+from flame.config import Config
 from flame.mode.horizontal.middle_aggregator import MiddleAggregator
 # the following needs to be imported to let the flame know
 # this aggregator works on tensorflow model
@@ -58,7 +58,7 @@ def evaluate(self) -> None:
 
     args = parser.parse_args()
 
-    config = load_config(args.config)
+    config = Config(args.config)
 
     a = KerasMnistMiddleAggregator(config)
     a.compose()

lib/python/flame/examples/hier_mnist/top_aggregator/main.py

@@ -17,7 +17,7 @@
 
 import logging
 
-from flame.config import Config, load_config
+from flame.config import Config
 from flame.dataset import Dataset
 from flame.mode.horizontal.top_aggregator import TopAggregator
 from tensorflow import keras
@@ -82,7 +82,7 @@ def evaluate(self) -> None:
 
     args = parser.parse_args()
 
-    config = load_config(args.config)
+    config = Config(args.config)
 
     a = KerasMnistTopAggregator(config)
     a.compose()

lib/python/flame/examples/hier_mnist/trainer/main.py

@@ -20,7 +20,7 @@
 from statistics import mean
 
 import numpy as np
-from flame.config import Config, load_config
+from flame.config import Config
 from flame.mode.horizontal.trainer import Trainer
 from tensorflow import keras
 from tensorflow.keras import layers
@@ -131,7 +131,7 @@ def evaluate(self) -> None:
 
     args = parser.parse_args()
 
-    config = load_config(args.config)
+    config = Config(args.config)
 
     t = KerasMnistTrainer(config)
     t.compose()

lib/python/flame/examples/hybrid/aggregator/main.py

@@ -18,7 +18,7 @@
 
 import logging
 
-from flame.config import Config, load_config
+from flame.config import Config
 from flame.dataset import Dataset
 from flame.mode.horizontal.top_aggregator import TopAggregator
 from tensorflow import keras
@@ -81,7 +81,7 @@ def evaluate(self) -> None:
 
     args = parser.parse_args()
 
-    config = load_config(args.config)
+    config = Config(args.config)
 
     a = KerasMnistAggregator(config)
     a.compose()

lib/python/flame/examples/hybrid/trainer/main.py

@@ -20,7 +20,7 @@
 from statistics import mean
 
 import numpy as np
-from flame.config import Config, load_config
+from flame.config import Config
 from flame.mode.hybrid.trainer import Trainer
 from tensorflow import keras
 from tensorflow.keras import layers
@@ -131,7 +131,7 @@ def evaluate(self) -> None:
 
     args = parser.parse_args()
 
-    config = load_config(args.config)
+    config = Config(args.config)
 
     t = KerasMnistTrainer(config)
     t.compose()

lib/python/flame/examples/medmnist/aggregator/pytorch/main.py

@@ -18,7 +18,7 @@
 
 import logging
 
-from flame.config import Config, load_config
+from flame.config import Config
 from flame.dataset import Dataset # Not sure why we need this.
 from flame.mode.horizontal.top_aggregator import TopAggregator
 import torch
@@ -85,7 +85,7 @@ def evaluate(self) -> None:
 
     args = parser.parse_args()
 
-    config = load_config(args.config)
+    config = Config(args.config)
 
     a = PyTorchMedMNistAggregator(config)
     a.compose()

lib/python/flame/examples/medmnist/trainer/pytorch/main.py

@@ -20,7 +20,7 @@
 from flame.common.util import install_packages
 install_packages(['scikit-learn'])
 
-from flame.config import Config, load_config
+from flame.config import Config
 from flame.mode.horizontal.trainer import Trainer
 import torch
 import torchvision
@@ -212,7 +212,7 @@ def evaluate(self) -> None:
 
     args = parser.parse_args()
 
-    config = load_config(args.config)
+    config = Config(args.config)
 
     t = PyTorchMedMNistTrainer(config)
     t.compose()

lib/python/flame/examples/mnist/aggregator/keras/main.py

@@ -17,7 +17,7 @@
 
 import logging
 
-from flame.config import Config, load_config
+from flame.config import Config
 from flame.dataset import Dataset
 from flame.mode.horizontal.top_aggregator import TopAggregator
 from tensorflow import keras
@@ -82,7 +82,7 @@ def evaluate(self) -> None:
 
     args = parser.parse_args()
 
-    config = load_config(args.config)
+    config = Config(args.config)
 
     a = KerasMnistAggregator(config)
     a.compose()

lib/python/flame/examples/mnist/aggregator/pytorch/main.py

@@ -24,7 +24,7 @@
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
-from flame.config import Config, load_config
+from flame.config import Config
 from flame.dataset import Dataset
 from flame.mode.horizontal.top_aggregator import TopAggregator
 from torchvision import datasets, transforms
@@ -143,7 +143,7 @@ def evaluate(self) -> None:
 
     args = parser.parse_args()
 
-    config = load_config(args.config)
+    config = Config(args.config)
 
     a = PyTorchMnistAggregator(config)
     a.compose()

lib/python/flame/examples/mnist/trainer/keras/main.py

@@ -20,7 +20,7 @@
 from statistics import mean
 
 import numpy as np
-from flame.config import Config, load_config
+from flame.config import Config
 from flame.mode.horizontal.trainer import Trainer
 from tensorflow import keras
 from tensorflow.keras import layers
@@ -131,7 +131,7 @@ def evaluate(self) -> None:
 
     args = parser.parse_args()
 
-    config = load_config(args.config)
+    config = Config(args.config)
 
     t = KerasMnistTrainer(config)
     t.compose()

lib/python/flame/examples/mnist/trainer/pytorch/main.py

@@ -26,7 +26,7 @@
 import torch.nn.functional as F
 import torch.optim as optim
 import torch.utils.data as data_utils
-from flame.config import Config, load_config
+from flame.config import Config
 from flame.mode.horizontal.trainer import Trainer
 
 
@@ -146,7 +146,7 @@ def evaluate(self) -> None:
     parser.add_argument('config', nargs='?', default="./config.json")
 
     args = parser.parse_args()
-    config = load_config(args.config)
+    config = Config(args.config)
 
     t = PyTorchMnistTrainer(config)
     t.compose()