Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

remove auto installs and switch to pypi forks for transformers and yolov5 #1518

Merged
merged 5 commits into from
May 8, 2023
Merged

remove auto installs and switch to pypi forks for transformers and yolov5 #1518

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e978de3
remove auto installs and switch to pypi forks for transformers and yo…
bfineran Apr 12, 2023
3e475ca
Merge branch 'main' into remove-autoinstalls
bfineran May 1, 2023
69768f4
quality
bfineran May 1, 2023
28dee10
quality
bfineran May 8, 2023
5c1a843
Merge branch 'main' into remove-autoinstalls
bfineran May 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 5 additions & 9 deletions docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,23 +95,19 @@ ARG MODE
RUN \
if [ -n "$BRANCH" ] ; then \
echo Installing from BRANCH && \
$VENV/bin/pip install --no-cache-dir "./sparseml[onnxruntime,torchvision,ultralytics]"; \
$VENV/bin/pip install --no-cache-dir "./sparseml[onnxruntime,torchvision,transformers,yolov5,ultralytics]"; \
elif [ "$MODE" = "nightly" ] ; then \
if [ -z $VERSION] ; then \
$VENV/bin/pip install --no-cache-dir "sparseml-nightly[onnxruntime,torchvision,ultralytics]"; \
$VENV/bin/pip install --no-cache-dir "sparseml-nightly[onnxruntime,torchvision,transformers,yolov5,ultralytics]"; \
else \
$VENV/bin/pip install --no-cache-dir "sparseml-nightly[onnxruntime,torchvision,ultralytics]==$VERSION"; \
$VENV/bin/pip install --no-cache-dir "sparseml-nightly[onnxruntime,torchvision,transformers,yolov5,ultralytics]==$VERSION"; \
fi; \
elif [ -z $VERSION] ; then \
$VENV/bin/pip install --no-cache-dir "sparseml[onnxruntime,torchvision,ultralytics]"; \
$VENV/bin/pip install --no-cache-dir "sparseml[onnxruntime,torchvision,transformers,yolov5,ultralytics]"; \
else \
$VENV/bin/pip install --no-cache-dir "sparseml[onnxruntime,torchvision,ultralytics]==$VERSION"; \
$VENV/bin/pip install --no-cache-dir "sparseml[onnxruntime,torchvision,transformers,yolov5,ultralytics]==$VERSION"; \
fi;

RUN sparseml.transformers.question_answering --help \
&& sparseml.yolov5.train --help \
&& sparseml.ultralytics.train --help


FROM cuda_builder AS container_branch_dev
ARG VENV
Expand Down
12 changes: 12 additions & 0 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,16 @@
"torchvision>=0.3.0,<=0.14",
]
_pytorch_vision_deps = _pytorch_deps + ["torchvision>=0.3.0,<=0.14"]
_transformers_deps = _pytorch_deps + [
f"{'nm-transformers' if is_release else 'nm-transformers-nightly'}"
f"~={version_nm_deps}",
"datasets<=1.18.4",
"scikit-learn",
"seqeval",
]
_yolov5_deps = _pytorch_vision_deps + [
f"{'nm-yolov5' if is_release else 'nm-yolov5-nightly'}~={version_nm_deps}"
]
_tensorflow_v1_deps = ["tensorflow<2.0.0", "tensorboard<2.0.0", "tf2onnx>=1.0.0,<1.6"]
_tensorflow_v1_gpu_deps = [
"tensorflow-gpu<2.0.0",
Expand Down Expand Up @@ -132,10 +142,12 @@ def _setup_extras() -> Dict:
"torch": _pytorch_deps,
"torch_all": _pytorch_all_deps,
"torchvision": _pytorch_vision_deps,
"transformers": _transformers_deps,
"tf_v1": _tensorflow_v1_deps,
"tf_v1_gpu": _tensorflow_v1_gpu_deps,
"tf_keras": _keras_deps,
"ultralytics": _ultralytics_deps,
"yolov5": _yolov5_deps,
}


Expand Down
97 changes: 3 additions & 94 deletions src/sparseml/transformers/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,111 +20,20 @@

import logging as _logging

import pkg_resources
from sparseml.analytics import sparseml_analytics as _analytics


_analytics.send_event("python__transformers__init")

_EXPECTED_VERSION = "4.23.1"


_LOGGER = _logging.getLogger(__name__)
_NM_TRANSFORMERS_TAR_TEMPLATE = (
"https://github.com/neuralmagic/transformers/releases/download/"
"{version}/transformers-4.23.1-py3-none-any.whl"
)
_NM_TRANSFORMERS_NIGHTLY = _NM_TRANSFORMERS_TAR_TEMPLATE.format(version="nightly")


def _install_transformers_and_deps():

import subprocess as _subprocess
import sys as _sys

import sparseml as _sparseml

nm_transformers_release = (
"nightly" if not _sparseml.is_release else f"v{_sparseml.version_major_minor}"
)
transformers_requirement = _NM_TRANSFORMERS_TAR_TEMPLATE.format(
version=nm_transformers_release
)
try:
_subprocess.check_call(
[
_sys.executable,
"-m",
"pip",
"install",
transformers_requirement,
"datasets<=1.18.4",
"scikit-learn",
"seqeval",
]
)

import transformers as _transformers

_LOGGER.info("sparseml-transformers and dependencies successfully installed")
except Exception:
raise ValueError(
"Unable to install and import sparseml-transformers dependencies check "
"that transformers is installed, if not, install via "
f"`pip install {_NM_TRANSFORMERS_NIGHTLY}`"
)


def _check_transformers_install():
transformers_version = next(
(
pkg.version
for pkg in pkg_resources.working_set
if pkg.project_name.lower() == "transformers"
),
None,
)

# Either no transformers install is found or wrong version installed
if transformers_version != _EXPECTED_VERSION:
import os

if os.getenv("NM_NO_AUTOINSTALL_TRANSFORMERS", False):
_LOGGER.warning(
"Unable to import, skipping auto installation "
"due to NM_NO_AUTOINSTALL_TRANSFORMERS"
)
# skip any further checks
return
else:
_LOGGER.warning(
f"sparseml-transformers v{_EXPECTED_VERSION} installation not "
f"detected. Installing sparseml-transformers v{_EXPECTED_VERSION} "
"dependencies if transformers is already installed in the "
"environment, it will be overwritten. Set environment variable "
"NM_NO_AUTOINSTALL_TRANSFORMERS to disable"
)
_install_transformers_and_deps()

else:
import transformers as _transformers

# Edge case where user has expected version of transformers installed, but
# not the nm integrated one
if not _transformers.NM_INTEGRATED:
_install_transformers_and_deps()
raise RuntimeError(
"Installed transformers package has been overwritten with "
"sparseml-transformers. Stopping process as this is likely to cause "
"import issues. Please re-run command"
)

# re check import after potential install
try:
import transformers as _transformers
# check for NM integration in transformers version
import transformers as _transformers

assert _transformers.NM_INTEGRATED
except Exception:
if not _transformers.NM_INTEGRATED:
_LOGGER.warning(
"the neuralmagic fork of transformers may not be installed. it can be "
"installed via "
Expand Down
79 changes: 6 additions & 73 deletions src/sparseml/yolov5/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,90 +21,23 @@

from sparseml.analytics import sparseml_analytics as _analytics

from .helpers import *


_analytics.send_event("python__yolov5__init")

try:
import yolov5 as _yolov5

_yolov5_import_error = None
except Exception as _yolov5_import_err:
_yolov5_import_error = _yolov5_import_err

_LOGGER = _logging.getLogger(__name__)
_NM_YOLOV5_TAR_TEMPLATE = (
"https://github.com/neuralmagic/yolov5/releases/download/"
"{version}/yolov5-6.2.0-py3-none-any.whl"
)
_NM_YOLOV5_NIGHTLY = _NM_YOLOV5_TAR_TEMPLATE.format(version="nightly")


def _install_yolov5_and_deps():

import subprocess as _subprocess
import sys as _sys

import sparseml as _sparseml

nm_yolov5_release = (
"nightly" if not _sparseml.is_release else f"v{_sparseml.version_major_minor}"
)

yolov5_requirement = _NM_YOLOV5_TAR_TEMPLATE.format(version=nm_yolov5_release)

try:
_subprocess.check_call(
[
_sys.executable,
"-m",
"pip",
"install",
yolov5_requirement,
]
)

import yolov5 as _yolov5

_LOGGER.info("sparseml-yolov5 and dependencies successfully installed")
except Exception:
raise ValueError(
"Unable to install and import sparseml-yolov5 dependencies check "
"that yolov5 is installed, if not, install via "
f"`pip install {_NM_YOLOV5_NIGHTLY}`"
)


def _check_yolov5_install():
if _yolov5_import_error is not None:
import os

if os.getenv("NM_NO_AUTOINSTALL_YOLOV5", False):
_LOGGER.warning(
"Unable to import, skipping auto installation "
"due to NM_NO_AUTOINSTALL_YOLOV5"
)
# skip any further checks
return
else:
_LOGGER.warning(
"sparseml-yolov5 installation not detected. Installing "
"sparseml-yolov5 dependencies if yolov5 is already "
"installed in the environment, it will be overwritten. Set "
"environment variable NM_NO_AUTOINSTALL_YOLOV5 to disable"
)
_install_yolov5_and_deps()

# re check import after potential install
# check nm-yolov5 is installed
try:
import yolov5 as _yolov5
except Exception:
_LOGGER.warning(
"the neuralmagic fork of yolov5 may not be installed. it can be "
"installed via "
f"`pip install {_NM_YOLOV5_NIGHTLY}`"
raise ImportError(
"Unable to import the neuralmagic fork of yolov5 may not be installed. "
f"it can be installed via `pip install nn-yolov5`"
)


_check_yolov5_install()

from .helpers import *