Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(framework) Add deployment engine executor #3629

Merged
merged 28 commits into from
Jun 29, 2024
Merged
Changes from 3 commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
b908908
feat(framework) Add deployment engine executor
charlesbvll Jun 17, 2024
93f4fc5
Merge branch 'main' into add-deployment-engine-executor
charlesbvll Jun 17, 2024
fd70046
Merge branch 'main' into add-deployment-engine-executor
danieljanes Jun 19, 2024
1216a70
Merge branch 'main' into add-deployment-engine-executor
charlesbvll Jun 21, 2024
3eed1b3
Add copyright notice
charlesbvll Jun 21, 2024
988d63f
Apply suggestions
charlesbvll Jun 21, 2024
7eb92d0
Add simple error handling
charlesbvll Jun 21, 2024
d33ced6
Disable pylint error
charlesbvll Jun 21, 2024
986cbd6
fix imports
charlesbvll Jun 21, 2024
25434ef
Merge branch 'main' into add-deployment-engine-executor
charlesbvll Jun 21, 2024
21831e3
Merge branch 'main' into add-deployment-engine-executor
charlesbvll Jun 21, 2024
d69b3e7
Remove debug line
charlesbvll Jun 21, 2024
035abb0
Merge branch 'main' into add-deployment-engine-executor
charlesbvll Jun 21, 2024
45e1af1
Add app to flwr run and fix order
charlesbvll Jun 21, 2024
e8e4015
Merge branch 'main' into add-deployment-engine-executor
charlesbvll Jun 21, 2024
2968e61
Merge branch 'main' into add-deployment-engine-executor
charlesbvll Jun 23, 2024
8f11125
Merge branch 'main' into add-deployment-engine-executor
charlesbvll Jun 23, 2024
d0a746c
Merge branch 'main' into add-deployment-engine-executor
charlesbvll Jun 24, 2024
dbcdac2
Merge branch 'main' into add-deployment-engine-executor
danieljanes Jun 28, 2024
05e1afc
Log run start
danieljanes Jun 29, 2024
e24790d
Create run after FAB installation
danieljanes Jun 29, 2024
62708ef
Rename deployment_plugin to deployment
danieljanes Jun 29, 2024
4f736d4
Format deployment engine exec
danieljanes Jun 29, 2024
2672b95
Log run_id
danieljanes Jun 29, 2024
60fa3c7
Remove mentions of 'plugin'
danieljanes Jun 29, 2024
c4a1039
Align argument naming
danieljanes Jun 29, 2024
da3182f
Fix lint issues
danieljanes Jun 29, 2024
70e6777
Merge branch 'main' into add-deployment-engine-executor
danieljanes Jun 29, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 84 additions & 0 deletions src/py/flwr/superexec/deployment_plugin.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
"""Deployment engine executor plugin."""
charlesbvll marked this conversation as resolved.
Show resolved Hide resolved

import subprocess
import sys
from pathlib import Path
from typing import Optional

from flwr.cli.config_utils import get_fab_metadata
from flwr.cli.install import install_from_fab
from flwr.common.grpc import create_channel
from flwr.proto.driver_pb2 import CreateRunRequest # pylint: disable=E0611
from flwr.proto.driver_pb2_grpc import DriverStub
from flwr.server.driver.grpc_driver import DEFAULT_SERVER_ADDRESS_DRIVER

from .executor import Executor, RunTracker


class DeploymentEngine(Executor):
"""Deployment engine executor plugin."""

def __init__(
self,
address: str = DEFAULT_SERVER_ADDRESS_DRIVER,
root_certificates: Optional[bytes] = None,
) -> None:
self.address = address
self.root_certificates = root_certificates
self.stub: Optional[DriverStub] = None

def _connect(self) -> None:
if self.stub is None:
channel = create_channel(
server_address=self.address,
insecure=(self.root_certificates is None),
root_certificates=self.root_certificates,
)
self.stub = DriverStub(channel)

def _create_run(self, fab_id: str, fab_version: str) -> int:
if self.stub is None:
self._connect()

assert self.stub is not None

req = CreateRunRequest(fab_id=fab_id, fab_version=fab_version)
res = self.stub.CreateRun(request=req)
return int(res.run_id)

def _install_fab(self, fab_file: bytes) -> Path:
return install_from_fab(fab_file, None, True)
charlesbvll marked this conversation as resolved.
Show resolved Hide resolved

def start_run(self, fab_file: bytes, ttl: Optional[float] = None) -> RunTracker:
charlesbvll marked this conversation as resolved.
Show resolved Hide resolved
"""Start run using the Flower Deployment Engine."""
_ = ttl
fab_id, fab_version = get_fab_metadata(fab_file)

run_id = self._create_run(fab_id, fab_version)

fab_path = self._install_fab(fab_file)

subprocess.check_call(
[sys.executable, "-m", "pip", "install", str(fab_path)],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)

return RunTracker(
run_id=run_id,
proc=subprocess.Popen(
[
"flower-server-app",
"build_demo.server:app",
"--run-id",
str(run_id),
"--insecure",
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
),
)


deployment_plugin = DeploymentEngine()