Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Search for proper run_name and not state name #3215

Merged
merged 38 commits into from
Apr 30, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
10b30a7
search properly
KuuCi Apr 26, 2024
4a3f537
fix
KuuCi Apr 26, 2024
f4f29ad
patch
KuuCi Apr 26, 2024
d1f9503
patch
KuuCi Apr 26, 2024
3fcfbb1
patch
KuuCi Apr 26, 2024
09ae50a
reorder
KuuCi Apr 26, 2024
d1d1fae
Merge branch 'dev' into mlflow-logger-resumption-path
dakinggg Apr 26, 2024
3367827
add mlflow url
KuuCi Apr 26, 2024
df160dd
Merge branch 'mlflow-logger-resumption-path' of https://github.com/mo…
KuuCi Apr 26, 2024
292527b
pyright
KuuCi Apr 26, 2024
bf88fdf
isort
KuuCi Apr 26, 2024
08e8ba5
have tags detected after run_name is set
KuuCi Apr 26, 2024
fcf4f4a
fix url
KuuCi Apr 26, 2024
3e27783
rm func to get run
KuuCi Apr 26, 2024
13a963f
Merge branch 'dev' into mlflow-logger-resumption-path
dakinggg Apr 27, 2024
fb8bcd8
test
KuuCi Apr 27, 2024
f540d89
Merge branch 'mlflow-logger-resumption-path' of https://github.com/mo…
KuuCi Apr 27, 2024
65feb4f
test
KuuCi Apr 27, 2024
4f771ce
enable
KuuCi Apr 27, 2024
76c0e46
direct insertion
KuuCi Apr 27, 2024
7afad4a
set tracking_uri
KuuCi Apr 27, 2024
fdb690d
revert
KuuCi Apr 27, 2024
43652b8
fix link
KuuCi Apr 27, 2024
3925ba0
rm slash
KuuCi Apr 27, 2024
be15e61
debug statements
KuuCi Apr 27, 2024
e5628fc
debug
KuuCi Apr 27, 2024
741e2de
more debug
KuuCi Apr 27, 2024
685213b
tracking client
KuuCi Apr 27, 2024
8a337a8
rm debug statements
KuuCi Apr 27, 2024
c43658f
naive test
KuuCi Apr 29, 2024
35d8ff2
import
KuuCi Apr 29, 2024
eed24b2
debug log
KuuCi Apr 29, 2024
d7fa0f9
move to after_load
KuuCi Apr 29, 2024
7cd0df4
precommit
KuuCi Apr 29, 2024
7ebc4c5
get
KuuCi Apr 29, 2024
f095430
precommmit
KuuCi Apr 29, 2024
078f5d8
Merge branch 'dev' into mlflow-logger-resumption-path
dakinggg Apr 29, 2024
55d9fc0
Merge branch 'dev' into mlflow-logger-resumption-path
dakinggg Apr 30, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion composer/loggers/mlflow_logger.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import logging
import os
import pathlib
import posixpath
import textwrap
import time
import warnings
Expand Down Expand Up @@ -110,6 +111,7 @@ def __init__(

self._experiment_id: Optional[str] = None
self._run_id = None
self.run_url = None

if self._enabled:
self.tracking_uri = str(tracking_uri or mlflow.get_tracking_uri())
Expand Down Expand Up @@ -165,9 +167,10 @@ def init(self, state: State, logger: Logger) -> None:
else:
# Search for an existing run tagged with this Composer run.
assert self._experiment_id is not None
run_name = self.tags['run_name']
existing_runs = mlflow.search_runs(
experiment_ids=[self._experiment_id],
filter_string=f'tags.run_name = "{state.run_name}"',
filter_string=f'tags.run_name = "{run_name}"',
output_format='list',
)

Expand All @@ -194,6 +197,14 @@ def init(self, state: State, logger: Logger) -> None:

def after_load(self, state: State, logger: Logger) -> None:
logger.log_hyperparameters({'mlflow_experiment_id': self._experiment_id, 'mlflow_run_id': self._run_id})
self.run_url = posixpath.join(
os.environ.get('DATABRICKS_HOST', ''),
'ml',
'experiments',
str(self._experiment_id),
'runs',
str(self._run_id),
)

def log_table(
self,
Expand Down
4 changes: 4 additions & 0 deletions composer/loggers/mosaicml_logger.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from composer.core.time import TimeUnit
from composer.loggers import Logger
from composer.loggers.logger_destination import LoggerDestination
from composer.loggers.mlflow_logger import MLFlowLogger
from composer.loggers.wandb_logger import WandBLogger
from composer.utils import dist

Expand Down Expand Up @@ -113,6 +114,9 @@ def after_load(self, state: State, logger: Logger) -> None:
log.debug(f'Logging WandB run URL to metadata: {run_url}')
else:
log.debug('WandB run URL not found, not logging to metadata')
if isinstance(callback, MLFlowLogger) and callback._enabled:
self._log_metadata({'mlflow/run_url': callback.run_url})
log.debug(f'Logging MLFlow run URL to metadata: {callback.run_url}')
self._flush_metadata(force_flush=True)

def batch_start(self, state: State, logger: Logger) -> None:
Expand Down
Loading