Skip to content

Commit

Permalink
remove auto installs and switch to pypi forks for transformers and yo…
Browse files Browse the repository at this point in the history 
…lov5 (#1518)

* remove auto installs and switch to pypi forks for transformers and yolov5

* quality

* quality
  • Loading branch information
@bfineran
bfineran committed May 10, 2023
1 parent aee7a0c commit 4825c21
Show file tree
Hide file tree
Showing 4 changed files with 26 additions and 176 deletions.
14 changes: 5 additions & 9 deletions docker/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,23 +95,19 @@ ARG MODE
RUN \
if [ -n "$BRANCH" ] ; then \
echo Installing from BRANCH && \
$VENV/bin/pip install --no-cache-dir "./sparseml[onnxruntime,torchvision,ultralytics]"; \
$VENV/bin/pip install --no-cache-dir "./sparseml[onnxruntime,torchvision,transformers,yolov5,ultralytics]"; \
elif [ "$MODE" = "nightly" ] ; then \
if [ -z $VERSION] ; then \
$VENV/bin/pip install --no-cache-dir "sparseml-nightly[onnxruntime,torchvision,ultralytics]"; \
$VENV/bin/pip install --no-cache-dir "sparseml-nightly[onnxruntime,torchvision,transformers,yolov5,ultralytics]"; \
else \
$VENV/bin/pip install --no-cache-dir "sparseml-nightly[onnxruntime,torchvision,ultralytics]==$VERSION"; \
$VENV/bin/pip install --no-cache-dir "sparseml-nightly[onnxruntime,torchvision,transformers,yolov5,ultralytics]==$VERSION"; \
fi; \
elif [ -z $VERSION] ; then \
$VENV/bin/pip install --no-cache-dir "sparseml[onnxruntime,torchvision,ultralytics]"; \
$VENV/bin/pip install --no-cache-dir "sparseml[onnxruntime,torchvision,transformers,yolov5,ultralytics]"; \
else \
$VENV/bin/pip install --no-cache-dir "sparseml[onnxruntime,torchvision,ultralytics]==$VERSION"; \
$VENV/bin/pip install --no-cache-dir "sparseml[onnxruntime,torchvision,transformers,yolov5,ultralytics]==$VERSION"; \
fi;

RUN sparseml.transformers.question_answering --help \
&& sparseml.yolov5.train --help \
&& sparseml.ultralytics.train --help


FROM cuda_builder AS container_branch_dev
ARG VENV
Expand Down
12 changes: 12 additions & 0 deletions setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,16 @@
"torchvision>=0.3.0,<=0.14",
]
_pytorch_vision_deps = _pytorch_deps + ["torchvision>=0.3.0,<=0.14"]
_transformers_deps = _pytorch_deps + [
f"{'nm-transformers' if is_release else 'nm-transformers-nightly'}"
f"~={version_nm_deps}",
"datasets<=1.18.4",
"scikit-learn",
"seqeval",
]
_yolov5_deps = _pytorch_vision_deps + [
f"{'nm-yolov5' if is_release else 'nm-yolov5-nightly'}~={version_nm_deps}"
]
_tensorflow_v1_deps = ["tensorflow<2.0.0", "tensorboard<2.0.0", "tf2onnx>=1.0.0,<1.6"]
_tensorflow_v1_gpu_deps = [
"tensorflow-gpu<2.0.0",
Expand Down Expand Up @@ -132,10 +142,12 @@ def _setup_extras() -> Dict:
"torch": _pytorch_deps,
"torch_all": _pytorch_all_deps,
"torchvision": _pytorch_vision_deps,
"transformers": _transformers_deps,
"tf_v1": _tensorflow_v1_deps,
"tf_v1_gpu": _tensorflow_v1_gpu_deps,
"tf_keras": _keras_deps,
"ultralytics": _ultralytics_deps,
"yolov5": _yolov5_deps,
}


Expand Down
97 changes: 3 additions & 94 deletions src/sparseml/transformers/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,111 +20,20 @@

import logging as _logging

import pkg_resources
from sparseml.analytics import sparseml_analytics as _analytics


_analytics.send_event("python__transformers__init")

_EXPECTED_VERSION = "4.23.1"


_LOGGER = _logging.getLogger(__name__)
_NM_TRANSFORMERS_TAR_TEMPLATE = (
"https://github.com/neuralmagic/transformers/releases/download/"
"{version}/transformers-4.23.1-py3-none-any.whl"
)
_NM_TRANSFORMERS_NIGHTLY = _NM_TRANSFORMERS_TAR_TEMPLATE.format(version="nightly")


def _install_transformers_and_deps():

import subprocess as _subprocess
import sys as _sys

import sparseml as _sparseml

nm_transformers_release = (
"nightly" if not _sparseml.is_release else f"v{_sparseml.version_major_minor}"
)
transformers_requirement = _NM_TRANSFORMERS_TAR_TEMPLATE.format(
version=nm_transformers_release
)
try:
_subprocess.check_call(
[
_sys.executable,
"-m",
"pip",
"install",
transformers_requirement,
"datasets<=1.18.4",
"scikit-learn",
"seqeval",
]
)

import transformers as _transformers

_LOGGER.info("sparseml-transformers and dependencies successfully installed")
except Exception:
raise ValueError(
"Unable to install and import sparseml-transformers dependencies check "
"that transformers is installed, if not, install via "
f"`pip install {_NM_TRANSFORMERS_NIGHTLY}`"
)


def _check_transformers_install():
transformers_version = next(
(
pkg.version
for pkg in pkg_resources.working_set
if pkg.project_name.lower() == "transformers"
),
None,
)

# Either no transformers install is found or wrong version installed
if transformers_version != _EXPECTED_VERSION:
import os

if os.getenv("NM_NO_AUTOINSTALL_TRANSFORMERS", False):
_LOGGER.warning(
"Unable to import, skipping auto installation "
"due to NM_NO_AUTOINSTALL_TRANSFORMERS"
)
# skip any further checks
return
else:
_LOGGER.warning(
f"sparseml-transformers v{_EXPECTED_VERSION} installation not "
f"detected. Installing sparseml-transformers v{_EXPECTED_VERSION} "
"dependencies if transformers is already installed in the "
"environment, it will be overwritten. Set environment variable "
"NM_NO_AUTOINSTALL_TRANSFORMERS to disable"
)
_install_transformers_and_deps()

else:
import transformers as _transformers

# Edge case where user has expected version of transformers installed, but
# not the nm integrated one
if not _transformers.NM_INTEGRATED:
_install_transformers_and_deps()
raise RuntimeError(
"Installed transformers package has been overwritten with "
"sparseml-transformers. Stopping process as this is likely to cause "
"import issues. Please re-run command"
)

# re check import after potential install
try:
import transformers as _transformers
# check for NM integration in transformers version
import transformers as _transformers

assert _transformers.NM_INTEGRATED
except Exception:
if not _transformers.NM_INTEGRATED:
_LOGGER.warning(
"the neuralmagic fork of transformers may not be installed. it can be "
"installed via "
Expand Down
79 changes: 6 additions & 73 deletions src/sparseml/yolov5/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,90 +21,23 @@

from sparseml.analytics import sparseml_analytics as _analytics

from .helpers import *


_analytics.send_event("python__yolov5__init")

try:
import yolov5 as _yolov5

_yolov5_import_error = None
except Exception as _yolov5_import_err:
_yolov5_import_error = _yolov5_import_err

_LOGGER = _logging.getLogger(__name__)
_NM_YOLOV5_TAR_TEMPLATE = (
"https://github.com/neuralmagic/yolov5/releases/download/"
"{version}/yolov5-6.2.0-py3-none-any.whl"
)
_NM_YOLOV5_NIGHTLY = _NM_YOLOV5_TAR_TEMPLATE.format(version="nightly")


def _install_yolov5_and_deps():

import subprocess as _subprocess
import sys as _sys

import sparseml as _sparseml

nm_yolov5_release = (
"nightly" if not _sparseml.is_release else f"v{_sparseml.version_major_minor}"
)

yolov5_requirement = _NM_YOLOV5_TAR_TEMPLATE.format(version=nm_yolov5_release)

try:
_subprocess.check_call(
[
_sys.executable,
"-m",
"pip",
"install",
yolov5_requirement,
]
)

import yolov5 as _yolov5

_LOGGER.info("sparseml-yolov5 and dependencies successfully installed")
except Exception:
raise ValueError(
"Unable to install and import sparseml-yolov5 dependencies check "
"that yolov5 is installed, if not, install via "
f"`pip install {_NM_YOLOV5_NIGHTLY}`"
)


def _check_yolov5_install():
if _yolov5_import_error is not None:
import os

if os.getenv("NM_NO_AUTOINSTALL_YOLOV5", False):
_LOGGER.warning(
"Unable to import, skipping auto installation "
"due to NM_NO_AUTOINSTALL_YOLOV5"
)
# skip any further checks
return
else:
_LOGGER.warning(
"sparseml-yolov5 installation not detected. Installing "
"sparseml-yolov5 dependencies if yolov5 is already "
"installed in the environment, it will be overwritten. Set "
"environment variable NM_NO_AUTOINSTALL_YOLOV5 to disable"
)
_install_yolov5_and_deps()

# re check import after potential install
# check nm-yolov5 is installed
try:
import yolov5 as _yolov5
except Exception:
_LOGGER.warning(
"the neuralmagic fork of yolov5 may not be installed. it can be "
"installed via "
f"`pip install {_NM_YOLOV5_NIGHTLY}`"
raise ImportError(
"Unable to import the neuralmagic fork of yolov5 may not be installed. "
f"it can be installed via `pip install nn-yolov5`"
)


_check_yolov5_install()

from .helpers import *

0 comments on commit 4825c21

Please sign in to comment.