Add --use-openvino

README.md

@@ -69,7 +69,7 @@ Additional models will be added as they become available and there is public int
 - Any GPU compatibile with *DirectX* on *Windows* using **DirectML** libraries.  
   This includes support for AMD GPUs that are not supported by native ROCm libraries
 - *Intel Arc* GPUs using *Intel OneAPI* **Ipex/XPU** libraries
-- *Intel* iGPUs using *Intel OneAPI* **OpenVINO** libraries
+- *Intel* GPUs using *Intel OneAPI* **OpenVINO** libraries
 - *Apple M1/M2* on *OSX* using built-in support in Torch with **MPS** optimizations
 
 ## Install & Run
@@ -96,6 +96,7 @@ Below is partial list of all available parameters, run `webui --help` for the fu
 
     Setup options:
       --use-directml                   Use DirectML if no compatible GPU is detected, default: False
+      --use-openvino                   Use Intel OpenVINO backend, default: False
       --use-ipex                       Force use Intel OneAPI XPU backend, default: False
       --use-cuda                       Force use nVidia CUDA backend, default: False
       --use-rocm                       Force use AMD ROCm backend, default: False

installer.py

@@ -319,12 +319,13 @@ def check_torch():
     if args.profile:
         pr = cProfile.Profile()
         pr.enable()
-    allow_cuda = not (args.use_rocm or args.use_directml or args.use_ipex)
-    allow_rocm = not (args.use_cuda or args.use_directml or args.use_ipex)
-    allow_ipex = not (args.use_cuda or args.use_rocm or args.use_directml)
-    allow_directml = not (args.use_cuda or args.use_rocm or args.use_ipex)
-    log.debug(f'Torch overrides: cuda={args.use_cuda} rocm={args.use_rocm} ipex={args.use_ipex} diml={args.use_directml}')
-    log.debug(f'Torch allowed: cuda={allow_cuda} rocm={allow_rocm} ipex={allow_ipex} diml={allow_directml}')
+    allow_cuda = not (args.use_rocm or args.use_directml or args.use_ipex or args.use_openvino)
+    allow_rocm = not (args.use_cuda or args.use_directml or args.use_ipex or args.use_openvino)
+    allow_ipex = not (args.use_cuda or args.use_rocm or args.use_directml or args.use_openvino)
+    allow_directml = not (args.use_cuda or args.use_rocm or args.use_ipex or args.use_openvino)
+    allow_openvino = not (args.use_cuda or args.use_rocm or args.use_ipex or args.use_directml)
+    log.debug(f'Torch overrides: cuda={args.use_cuda} rocm={args.use_rocm} ipex={args.use_ipex} diml={args.use_directml} openvino={args.use_openvino}')
+    log.debug(f'Torch allowed: cuda={allow_cuda} rocm={allow_rocm} ipex={allow_ipex} diml={allow_directml} openvino={allow_openvino}')
     torch_command = os.environ.get('TORCH_COMMAND', '')
     xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none')
     if torch_command != '':
@@ -392,6 +393,10 @@ def check_torch():
             os.environ.setdefault('TENSORFLOW_PACKAGE', 'tensorflow==2.13.0 intel-extension-for-tensorflow[gpu]')
         else:
             torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.0.0a0 intel_extension_for_pytorch==2.0.110+gitba7f6c1 -f https://developer.intel.com/ipex-whl-stable-xpu')
+    elif allow_openvino and args.use_openvino:
+        #Remove this after 2.1.0 releases
+        log.info('Using OpenVINO with Torch Nightly CPU')
+        torch_command = os.environ.get('TORCH_COMMAND', '--pre torch==2.1.0.dev20230713+cpu torchvision==0.16.0.dev20230713+cpu -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html')
     else:
         machine = platform.machine()
         if sys.platform == 'darwin':
@@ -456,7 +461,7 @@ def check_torch():
         log.debug(f'Cannot install xformers package: {e}')
     if opts.get('cuda_compile_backend', '') == 'hidet':
         install('hidet', 'hidet')
-    if opts.get('cuda_compile_backend', '') == 'openvino_fx':
+    if args.use_openvino or opts.get('cuda_compile_backend', '') == 'openvino_fx':
         install('openvino==2023.1.0.dev20230811', 'openvino')
         os.environ.setdefault('PYTORCH_TRACING_MODE', 'TORCHFX')
     if args.profile:
@@ -495,7 +500,6 @@ def install_packages():
     install(invisiblewatermark_package, 'invisible-watermark')
     install('onnxruntime==1.15.1', 'onnxruntime', ignore=True)
     install('pi-heif', 'pi_heif', ignore=True)
-    install('git+https://github.com/damian0815/compel', 'compel', ignore=True)
     tensorflow_package = os.environ.get('TENSORFLOW_PACKAGE', 'tensorflow==2.13.0')
     install(tensorflow_package, 'tensorflow', ignore=True)
     install('git+https://github.com/google-research/torchsde', 'torchsde', ignore=True)
@@ -831,6 +835,7 @@ def add_args(parser):
     group.add_argument('--requirements', default = False, action='store_true', help = "Force re-check of requirements, default: %(default)s")
     group.add_argument('--quick', default = False, action='store_true', help = "Run with startup sequence only, default: %(default)s")
     group.add_argument('--use-directml', default = False, action='store_true', help = "Use DirectML if no compatible GPU is detected, default: %(default)s")
+    group.add_argument("--use-openvino", default = False, action='store_true', help="Use Intel OpenVINO backend, default: %(default)s")
     group.add_argument("--use-ipex", default = False, action='store_true', help="Force use Intel OneAPI XPU backend, default: %(default)s")
     group.add_argument("--use-cuda", default=False, action='store_true', help="Force use nVidia CUDA backend, default: %(default)s")
     group.add_argument("--use-rocm", default=False, action='store_true', help="Force use AMD ROCm backend, default: %(default)s")

modules/cmd_args.py

@@ -41,6 +41,7 @@
 group.add_argument("--disable-queue", action='store_true', help="Disable queues, default: %(default)s")
 group.add_argument('--debug', default = False, action='store_true', help = "Run installer with debug logging, default: %(default)s")
 group.add_argument('--use-directml', default = False, action='store_true', help = "Use DirectML if no compatible GPU is detected, default: %(default)s")
+group.add_argument("--use-openvino", default = False, action='store_true', help="Use Intel OpenVINO backend, default: %(default)s")
 group.add_argument("--use-ipex", default = False, action='store_true', help="Force use Intel OneAPI XPU backend, default: %(default)s")
 group.add_argument("--use-cuda", default=False, action='store_true', help="Force use nVidia CUDA backend, default: %(default)s")
 group.add_argument("--use-rocm", default=False, action='store_true', help="Force use AMD ROCm backend, default: %(default)s")

modules/intel/openvino/__init__.py

@@ -8,7 +8,6 @@
 from torch.fx.experimental.proxy_tensor import make_fx
 from torch._inductor.compile_fx import compile_fx
 from hashlib import sha256
-import modules.shared
 
 class ModelState:
     def __init__(self):
@@ -113,3 +112,15 @@ def get_cached_file_name(*args, model_hash_str, device, cache_root):
             file_name = None
             model_hash_str = None
     return file_name
+
+def openvino_override_opts():
+    from modules import shared
+    if not shared.opts.cuda_compile:
+        shared.log.warn("OpenVINO: Enabling Torch Compile")
+        shared.opts.cuda_compile = True
+    if shared.opts.cuda_compile_backend != "openvino_fx":
+        shared.log.warn("OpenVINO: Setting Torch Compiler backend to OpenVINO FX")
+        shared.opts.cuda_compile_backend = "openvino_fx"
+    if shared.opts.sd_backend != "diffusers":
+        shared.log.warn("OpenVINO: Setting backend to Diffusers")
+        shared.opts.sd_backend = "diffusers"

modules/shared.py

@@ -184,8 +184,11 @@ def assign_current_image(self, image):
 
 state = State()
 state.server_start = time.time()
-
-backend = Backend.DIFFUSERS if (cmd_opts.backend is not None) and (cmd_opts.backend.lower() == 'diffusers') else Backend.ORIGINAL # initial since we don't have opts loaded yet
+if cmd_opts.use_openvino:
+    backend = Backend.DIFFUSERS
+    cmd_opts.backend = 'diffusers'
+else:
+    backend = Backend.DIFFUSERS if (cmd_opts.backend is not None) and (cmd_opts.backend.lower() == 'diffusers') else Backend.ORIGINAL # initial since we don't have opts loaded yet
 
 
 class OptionInfo:
@@ -342,7 +345,7 @@ def default(obj):
     cross_attention_optimization_default ="Scaled-Dot-Product"
 
 options_templates.update(options_section(('sd', "Stable Diffusion"), {
-    "sd_backend": OptionInfo("original", "Stable Diffusion backend", gr.Radio, lambda: {"choices": ["original", "diffusers"] }),
+    "sd_backend": OptionInfo("diffusers" if cmd_opts.use_openvino else "original", "Stable Diffusion backend", gr.Radio, lambda: {"choices": ["original", "diffusers"] }),
     "sd_checkpoint_autoload": OptionInfo(True, "Stable Diffusion checkpoint autoload on server start"),
     "sd_model_checkpoint": OptionInfo(default_checkpoint, "Stable Diffusion checkpoint", gr.Dropdown, lambda: {"choices": list_checkpoint_tiles()}, refresh=refresh_checkpoints),
     "sd_model_refiner": OptionInfo('None', "Stable Diffusion refiner", gr.Dropdown, lambda: {"choices": ['None'] + list_checkpoint_tiles()}, refresh=refresh_checkpoints),
@@ -373,7 +376,7 @@ def default(obj):
 options_templates.update(options_section(('cuda', "Compute Settings"), {
     "memmon_poll_rate": OptionInfo(2, "VRAM usage polls per second during generation", gr.Slider, {"minimum": 0, "maximum": 40, "step": 1}),
     "precision": OptionInfo("Autocast", "Precision type", gr.Radio, lambda: {"choices": ["Autocast", "Full"]}),
-    "cuda_dtype": OptionInfo("FP32" if sys.platform == "darwin" else "BF16" if devices.backend == "ipex" else "FP16", "Device precision type", gr.Radio, lambda: {"choices": ["FP32", "FP16", "BF16"]}),
+    "cuda_dtype": OptionInfo("FP32" if sys.platform == "darwin" or cmd_opts.use_openvino else "BF16" if devices.backend == "ipex" else "FP16", "Device precision type", gr.Radio, lambda: {"choices": ["FP32", "FP16", "BF16"]}),
     "no_half": OptionInfo(False, "Use full precision for model (--no-half)", None, None, None),
     "no_half_vae": OptionInfo(False, "Use full precision for VAE (--no-half-vae)"),
     "upcast_sampling": OptionInfo(True if sys.platform == "darwin" else False, "Enable upcast sampling"),
@@ -385,8 +388,8 @@ def default(obj):
     "cudnn_benchmark": OptionInfo(False, "Enable full-depth cuDNN benchmark feature"),
     # "cuda_allow_tf32": OptionInfo(True, "Allow TF32 math ops"),
     # "cuda_allow_tf16_reduced": OptionInfo(True, "Allow TF16 reduced precision math ops"),
-    "cuda_compile": OptionInfo(False, "Enable model compile (experimental)"),
-    "cuda_compile_backend": OptionInfo("none", "Model compile backend (experimental)", gr.Radio, lambda: {"choices": ['none', 'inductor', 'cudagraphs', 'aot_ts_nvfuser', 'hidet', 'ipex', 'openvino_fx']}),
+    "cuda_compile": OptionInfo(True if cmd_opts.use_openvino else False, "Enable model compile (experimental)"),
+    "cuda_compile_backend": OptionInfo("openvino_fx" if cmd_opts.use_openvino else "none", "Model compile backend (experimental)", gr.Radio, lambda: {"choices": ['none', 'inductor', 'cudagraphs', 'aot_ts_nvfuser', 'hidet', 'ipex', 'openvino_fx']}),
     "cuda_compile_mode": OptionInfo("default", "Model compile mode (experimental)", gr.Radio, lambda: {"choices": ['default', 'reduce-overhead', 'max-autotune']}),
     "cuda_compile_fullgraph": OptionInfo(False, "Model compile fullgraph"),
     "cuda_compile_verbose": OptionInfo(False, "Model compile verbose mode"),
@@ -801,9 +804,9 @@ def cast_value(self, key, value):
 opts.load(config_filename)
 cmd_opts = cmd_args.compatibility_args(opts, cmd_opts)
 if cmd_opts.backend is None:
-    backend = Backend.DIFFUSERS if opts.data.get('sd_backend', 'original') == 'diffusers' else Backend.ORIGINAL
+    backend = Backend.DIFFUSERS if cmd_opts.use_openvino or opts.data.get('sd_backend', 'original') == 'diffusers' else Backend.ORIGINAL
 else:
-    backend = Backend.DIFFUSERS if cmd_opts.backend.lower() == 'diffusers' else Backend.ORIGINAL
+    backend = Backend.DIFFUSERS if cmd_opts.use_openvino or cmd_opts.backend.lower() == 'diffusers' else Backend.ORIGINAL
 opts.data['sd_backend'] = 'diffusers' if backend == Backend.DIFFUSERS else 'original'
 opts.data['uni_pc_lower_order_final'] = opts.schedulers_use_loworder
 opts.data['uni_pc_order'] = opts.schedulers_solver_order
@@ -997,7 +1000,7 @@ def sd_refiner(self, value):
 
     @property
     def backend(self):
-        return Backend.ORIGINAL if opts.data['sd_backend'] == 'original' else Backend.DIFFUSERS
+        return Backend.ORIGINAL if not cmd_opts.use_openvino and opts.data['sd_backend'] == 'original' else Backend.DIFFUSERS
 
     @property
     def sd_model_type(self):

modules/ui.py

@@ -15,6 +15,7 @@
 from modules.paths import script_path, data_path
 from modules.shared import opts, cmd_opts, readfile
 from modules.dml import directml_override_opts
+from modules.intel.openvino import openvino_override_opts
 from modules import prompt_parser
 import modules.codeformer_model
 import modules.generation_parameters_copypaste as parameters_copypaste
@@ -966,6 +967,8 @@ def run_settings(*args):
                 changed.append(key)
         if cmd_opts.use_directml:
             directml_override_opts()
+        if cmd_opts.use_openvino:
+            openvino_override_opts()
         try:
             opts.save(modules.shared.config_filename)
             log.info(f'Settings changed: {len(changed)} {changed}')

requirements.txt

@@ -39,6 +39,7 @@ voluptuous
 yapf
 scikit-image
 basicsr
+compel==2.0.2
 fasteners
 typing-extensions==4.7.1
 antlr4-python3-runtime==4.9.3