diff --git a/.github/workflows/cli-foundation-models-azure_openai-openai_completions_finetune_pipeline.yml b/.github/workflows/cli-foundation-models-azure_openai-openai_completions_finetune_pipeline.yml deleted file mode 100644 index 11199cf08b..0000000000 --- a/.github/workflows/cli-foundation-models-azure_openai-openai_completions_finetune_pipeline.yml +++ /dev/null @@ -1,56 +0,0 @@ -# This code is autogenerated. -# Code is generated by running custom script: python3 readme.py -# Any manual changes to this file may cause incorrect behavior. -# Any manual changes will be overwritten if the code is regenerated. - -name: cli-foundation-models-azure_openai-openai_completions_finetune_pipeline -# This file is created by readme.py. -# Please do not edit directly. -on: - workflow_dispatch: - schedule: - - cron: "23 2/12 * * *" - pull_request: - branches: - - main - paths: - - cli/foundation-models/azure_openai/openai_completions_finetune/** - - .github/workflows/cli-foundation-models-azure_openai-openai_completions_finetune_pipeline.yml - - infra/bootstrapping/** - - cli/run-pipeline-jobs.sh - - cli/setup.sh -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: check out repo - uses: actions/checkout@v2 - - name: azure login - uses: azure/login@v1 - with: - creds: ${{secrets.AZUREML_CREDENTIALS}} - - name: bootstrap resources - run: | - echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; - bash bootstrap.sh - working-directory: infra/bootstrapping - continue-on-error: false - - name: setup-cli - run: | - source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; - source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - bash setup.sh - working-directory: cli - continue-on-error: true - - name: run job - run: | - source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; - source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; - [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; - bash -x ../dataset-create.sh - bash -x ../../../run-job.sh openai_completions_finetune_pipeline_spec.yaml - working-directory: cli/foundation-models/azure_openai/openai_completions_finetune_pipeline diff --git a/.github/workflows/sdk-foundation-models-azure_openai-openai_completions_finetune_pipeline.yml b/.github/workflows/sdk-foundation-models-azure_openai-openai_completions_finetune_pipeline.yml deleted file mode 100644 index aced460fb0..0000000000 --- a/.github/workflows/sdk-foundation-models-azure_openai-openai_completions_finetune_pipeline.yml +++ /dev/null @@ -1,75 +0,0 @@ -# This code is autogenerated. -# Code is generated by running custom script: python3 readme.py -# Any manual changes to this file may cause incorrect behavior. -# Any manual changes will be overwritten if the code is regenerated. - -name: sdk-foundation-models-azure_openai-openai_completions_finetune_pipeline -# This file is created by sdk/python/readme.py. -# Please do not edit directly. -on: - workflow_dispatch: - schedule: - - cron: "57 2/12 * * *" - pull_request: - branches: - - main - paths: - - sdk/python/foundation-models/azure_openai/** - - .github/workflows/sdk-foundation-models-azure_openai-openai_completions_finetune_pipeline.yml - - sdk/python/dev-requirements.txt - - infra/bootstrapping/** - - sdk/python/setup.sh -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: check out repo - uses: actions/checkout@v2 - - name: setup python - uses: actions/setup-python@v2 - with: - python-version: "3.8" - - name: pip install notebook reqs - run: pip install -r sdk/python/dev-requirements.txt - - name: azure login - uses: azure/login@v1 - with: - creds: ${{secrets.AZUREML_CREDENTIALS}} - - name: bootstrap resources - run: | - echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}'; - bash bootstrap.sh - working-directory: infra/bootstrapping - continue-on-error: false - - name: setup SDK - run: | - source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; - source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - bash setup.sh - working-directory: sdk/python - continue-on-error: true - - name: setup-cli - run: | - source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; - source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - bash setup.sh - working-directory: cli - continue-on-error: true - - name: run foundation-models/azure_openai/openai_completions_finetune_pipeline.ipynb - run: | - source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh"; - source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh"; - bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json"; - bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "openai_completions_finetune_pipeline.ipynb"; - [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; - papermill -k python openai_completions_finetune_pipeline.ipynb openai_completions_finetune_pipeline.output.ipynb - working-directory: sdk/python/foundation-models/azure_openai - - name: upload notebook's working folder as an artifact - if: ${{ always() }} - uses: actions/upload-artifact@v2 - with: - name: openai_completions_finetune_pipeline - path: sdk/python/foundation-models/azure_openai diff --git a/cli/README.md b/cli/README.md index 65196f6782..eaa5d729e6 100644 --- a/cli/README.md +++ b/cli/README.md @@ -210,8 +210,7 @@ path|status|description [jobs/parallel/1a_oj_sales_prediction/pipeline.yml](jobs/parallel/1a_oj_sales_prediction/pipeline.yml)|[![jobs/parallel/1a_oj_sales_prediction/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-parallel-1a_oj_sales_prediction-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-parallel-1a_oj_sales_prediction-pipeline.yml)|The hello world pipeline job with partition by key [jobs/parallel/2a_iris_batch_prediction/pipeline.yml](jobs/parallel/2a_iris_batch_prediction/pipeline.yml)|[![jobs/parallel/2a_iris_batch_prediction/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-parallel-2a_iris_batch_prediction-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-parallel-2a_iris_batch_prediction-pipeline.yml)|The hello world pipeline job with inline parallel job [jobs/parallel/3a_mnist_batch_identification/pipeline.yml](jobs/parallel/3a_mnist_batch_identification/pipeline.yml)|[![jobs/parallel/3a_mnist_batch_identification/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-parallel-3a_mnist_batch_identification-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-parallel-3a_mnist_batch_identification-pipeline.yml)|The hello world pipeline job with inline parallel job -[jobs/parallel/3a_mnist_batch_identification/pipeline.yml](jobs/parallel/3a_mnist_batch_identification/pipeline.yml)|[![jobs/parallel/3a_mnist_batch_identification/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-parallel-3a_mnist_batch_identification-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-parallel-3a_mnist_batch_identification-pipeline.yml)|The hello world pipeline job with inline parallel job -[jobs/parallel/3a_mnist_batch_identification/pipeline.yml](jobs/parallel/3a_mnist_batch_identification/pipeline.yml)|[![jobs/parallel/3a_mnist_batch_identification/pipeline](https://github.com/Azure/azureml-examples/workflows/cli-jobs-parallel-3a_mnist_batch_identification-pipeline/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-jobs-parallel-3a_mnist_batch_identification-pipeline.yml)|The hello world pipeline job with inline parallel job +[cli\foundation-models\azure_openai\openai_completions_finetune_pipeline\openai_completions_finetune_spec.yaml](cli\foundation-models\azure_openai\openai_completions_finetune_pipeline\openai_completions_finetune_spec.yaml)|[![cli\foundation-models\azure_openai\openai_completions_finetune_pipeline\openai_completions_finetune_spec.yaml](.github\workflows\cli-foundation-models-azure_openai-openai_completions_finetune/badge.svg?branch=main)](.github\workflows\cli-foundation-models-azure_openai-openai_completions_finetune.yml)|The hello world pipeline job with inline parallel job **Endpoints** ([endpoints](endpoints)) diff --git a/cli/foundation-models/azure_openai/openai_completions_finetune/images/pipeline_completed.png b/cli/foundation-models/azure_openai/openai_completions_finetune/images/pipeline_completed.png index d0d2f9cd0a..0a9c5f001e 100644 Binary files a/cli/foundation-models/azure_openai/openai_completions_finetune/images/pipeline_completed.png and b/cli/foundation-models/azure_openai/openai_completions_finetune/images/pipeline_completed.png differ diff --git a/cli/foundation-models/azure_openai/openai_completions_finetune/images/pipeline_running.png b/cli/foundation-models/azure_openai/openai_completions_finetune/images/pipeline_running.png index f13b1f38cf..f016933f02 100644 Binary files a/cli/foundation-models/azure_openai/openai_completions_finetune/images/pipeline_running.png and b/cli/foundation-models/azure_openai/openai_completions_finetune/images/pipeline_running.png differ diff --git a/cli/foundation-models/azure_openai/openai_completions_finetune/openai_completions_finetune.md b/cli/foundation-models/azure_openai/openai_completions_finetune/openai_completions_finetune.md index 8799999f84..11be7df901 100644 --- a/cli/foundation-models/azure_openai/openai_completions_finetune/openai_completions_finetune.md +++ b/cli/foundation-models/azure_openai/openai_completions_finetune/openai_completions_finetune.md @@ -1,4 +1,4 @@ -## How to finetune openai models via CLI mode using command components ? +## How to finetune openai models via CLI mode using command component ? 1. Login to az cli using `az login` diff --git a/cli/foundation-models/azure_openai/openai_completions_finetune/openai_completions_finetune_spec.yaml b/cli/foundation-models/azure_openai/openai_completions_finetune/openai_completions_finetune_spec.yaml index 6e0dd1671c..5012de6033 100644 --- a/cli/foundation-models/azure_openai/openai_completions_finetune/openai_completions_finetune_spec.yaml +++ b/cli/foundation-models/azure_openai/openai_completions_finetune/openai_completions_finetune_spec.yaml @@ -9,26 +9,18 @@ inputs: model_name: ada jobs: - model_selector: - type: command - component: azureml://registries/azure-openai-preview/components/openai_model_import/versions/0.2.0 - inputs: - model: ${{parent.inputs.model_name}} - outputs: - output_model: - type: uri_folder - mode: mount - - - finetune: - type: command - component: azureml://registries/azure-openai-preview/components/openai_completions_finetune/versions/0.2.1 + pipeline_finetune: + type: pipeline + component: azureml://registries/azure-openai-preview/components/openai_completions_finetune/versions/0.2.9 inputs: train_dataset: type: uri_folder path: azureml:oai_sample_training_data:1 mode: ro_mount - base_model: ${{parent.jobs.model_selector.outputs.output_model}} + validation_dataset: + type: uri_folder + path: azureml:oai_sample_training_data:1 + mode: ro_mount registered_model_name: ft_ada_model_from_cli_doc model: ${{parent.inputs.model_name}} outputs: diff --git a/cli/foundation-models/azure_openai/openai_completions_finetune_pipeline/images/pipeline_completed.png b/cli/foundation-models/azure_openai/openai_completions_finetune_pipeline/images/pipeline_completed.png deleted file mode 100644 index cb4691eee3..0000000000 Binary files a/cli/foundation-models/azure_openai/openai_completions_finetune_pipeline/images/pipeline_completed.png and /dev/null differ diff --git a/cli/foundation-models/azure_openai/openai_completions_finetune_pipeline/images/pipeline_running.png b/cli/foundation-models/azure_openai/openai_completions_finetune_pipeline/images/pipeline_running.png deleted file mode 100644 index 7549ff4e50..0000000000 Binary files a/cli/foundation-models/azure_openai/openai_completions_finetune_pipeline/images/pipeline_running.png and /dev/null differ diff --git a/cli/foundation-models/azure_openai/openai_completions_finetune_pipeline/images/registered_model.png b/cli/foundation-models/azure_openai/openai_completions_finetune_pipeline/images/registered_model.png deleted file mode 100644 index 01006fd2e7..0000000000 Binary files a/cli/foundation-models/azure_openai/openai_completions_finetune_pipeline/images/registered_model.png and /dev/null differ diff --git a/cli/foundation-models/azure_openai/openai_completions_finetune_pipeline/openai_completions_finetune_pipeline.md b/cli/foundation-models/azure_openai/openai_completions_finetune_pipeline/openai_completions_finetune_pipeline.md deleted file mode 100644 index eda757da2b..0000000000 --- a/cli/foundation-models/azure_openai/openai_completions_finetune_pipeline/openai_completions_finetune_pipeline.md +++ /dev/null @@ -1,19 +0,0 @@ -## How to finetune openai models via CLI mode using pipeline component ? - -1. Login to az cli using `az login` - -2. Navigate to `cli\foundation-models\azure_openai\openai_completions_finetune_pipeline` - -3. Run `az ml job create --subscription --resource-group --workspace-name --file "openai_completions_finetune_pipeline_spec.yaml"` - -4. The url to the run will be logged in the command line after step 3 is completed. - -![plot](./images/pipeline_running.png) - -5. After completion of the run, the status will be `Completed` of each component. - -![plot](./images/pipeline_completed.png) - -6. Now you can view the fine-tuned model in `Models` section. - -![plot](./images/registered_model.png) \ No newline at end of file diff --git a/cli/foundation-models/azure_openai/openai_completions_finetune_pipeline/openai_completions_finetune_pipeline_spec.yaml b/cli/foundation-models/azure_openai/openai_completions_finetune_pipeline/openai_completions_finetune_pipeline_spec.yaml deleted file mode 100644 index e69b611d87..0000000000 --- a/cli/foundation-models/azure_openai/openai_completions_finetune_pipeline/openai_completions_finetune_pipeline_spec.yaml +++ /dev/null @@ -1,21 +0,0 @@ -$schema: http://azureml/sdk-2-0/PipelineJob.json -type: pipeline -display_name: CLI OpenAI Completions Finetune Pipeline -description: CLI OpenAI Completions Finetune Pipeline -settings: - default_compute: azureml:serverless - force_rerun: true -inputs: - model_name: ada - -jobs: - pipeline_finetune: - type: pipeline - component: azureml://registries/azureml-preview/components/openai_completions_finetune/versions/0.0.7 - inputs: - train_dataset: - type: uri_folder - path: azureml:oai_sample_training_data:1 - mode: ro_mount - registered_model_name: ft_ada_model_from_cli_doc_pipeline - model: ${{parent.inputs.model_name}} diff --git a/sdk/python/README.md b/sdk/python/README.md index 02344fb672..00cd9de2eb 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -68,7 +68,6 @@ Test Status is for branch - **_main_** |endpoints|online|[online-endpoints-deploy-mlflow-model](endpoints/online/mlflow/online-endpoints-deploy-mlflow-model.ipynb)|Deploy an mlflow model to an online endpoint. This will be a no-code-deployment. It doesn't require scoring script and environment.|[![online-endpoints-deploy-mlflow-model](https://github.com/Azure/azureml-examples/actions/workflows/sdk-endpoints-online-mlflow-online-endpoints-deploy-mlflow-model.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-endpoints-online-mlflow-online-endpoints-deploy-mlflow-model.yml)| |endpoints|online|[online-endpoints-triton](endpoints/online/triton/single-model/online-endpoints-triton.ipynb)|Deploy a custom container as an online endpoint. Use web servers other than the default Python Flask server used by Azure ML without losing the benefits of Azure ML's built-in monitoring, scaling, alerting, and authentication.|[![online-endpoints-triton](https://github.com/Azure/azureml-examples/actions/workflows/sdk-endpoints-online-triton-single-model-online-endpoints-triton.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-endpoints-online-triton-single-model-online-endpoints-triton.yml)| |foundation-models|azure_openai|[openai_completions_finetune](foundation-models/azure_openai/openai_completions_finetune.ipynb)|*no description*|[![openai_completions_finetune](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-azure_openai-openai_completions_finetune.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-azure_openai-openai_completions_finetune.yml)| -|foundation-models|azure_openai|[openai_completions_finetune_pipeline](foundation-models/azure_openai/openai_completions_finetune_pipeline.ipynb)|*no description*|[![openai_completions_finetune_pipeline](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-azure_openai-openai_completions_finetune_pipeline.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-azure_openai-openai_completions_finetune_pipeline.yml)| |foundation-models|system|[import_model_into_registry](foundation-models/system/import/import_model_into_registry.ipynb)|*no description*|[![import_model_into_registry](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-import-import_model_into_registry.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-import-import_model_into_registry.yml)| |foundation-models|system|[asr-batch-endpoint](foundation-models/system/inference/automatic-speech-recognition/asr-batch-endpoint.ipynb)|*no description*|[![asr-batch-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-automatic-speech-recognition-asr-batch-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-automatic-speech-recognition-asr-batch-endpoint.yml)| |foundation-models|system|[summarization-batch-endpoint](foundation-models/system/inference/summarization/summarization-batch-endpoint.ipynb)|*no description*|[![summarization-batch-endpoint](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-summarization-summarization-batch-endpoint.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-foundation-models-system-inference-summarization-summarization-batch-endpoint.yml)| diff --git a/sdk/python/foundation-models/azure_openai/openai_completions_finetune.ipynb b/sdk/python/foundation-models/azure_openai/openai_completions_finetune.ipynb index 99e140ece8..4587e67854 100644 --- a/sdk/python/foundation-models/azure_openai/openai_completions_finetune.ipynb +++ b/sdk/python/foundation-models/azure_openai/openai_completions_finetune.ipynb @@ -3,6 +3,7 @@ { "attachments": {}, "cell_type": "markdown", + "id": "69484eca", "metadata": {}, "source": [ "# Build Pipeline with Azure OpenAI CommandComponents from registry\n", @@ -25,11 +26,12 @@ { "attachments": {}, "cell_type": "markdown", + "id": "0a46a0d8-d35a-437b-8fe6-25854ef6dd42", "metadata": {}, "source": [ "# 1. Connect to Azure Machine Learning Registry\n", "\n", - "The [workspace](https://docs.microsoft.com/en-us/azure/machine-learning/concept-workspace) is the top-level resource for Azure Machine Learning, providing a centralized place to work with all the artifacts you create when you use Azure Machine Learning. In this section we will connect to the workspace in which the job will be run.\n", + "A [registry](https://learn.microsoft.com/en-us/azure/machine-learning/concept-machine-learning-registries-mlops) is a container of shared assets that can be used across many workspaces.\n", "\n", "## 1.1 Import the required libraries" ] @@ -37,6 +39,7 @@ { "cell_type": "code", "execution_count": null, + "id": "f5d327ef-3466-40cf-a6a6-395529cfcf6e", "metadata": {}, "outputs": [], "source": [ @@ -51,6 +54,7 @@ { "attachments": {}, "cell_type": "markdown", + "id": "7bc31b17", "metadata": {}, "source": [ "## 1.2 Configure credential\n", @@ -64,6 +68,7 @@ { "cell_type": "code", "execution_count": null, + "id": "c4169370-43a9-4bd7-adca-ec3accf14400", "metadata": {}, "outputs": [], "source": [ @@ -79,6 +84,7 @@ { "attachments": {}, "cell_type": "markdown", + "id": "ac24d6f7", "metadata": {}, "source": [ "## 1.3 Get a handle to the registry\n", @@ -89,10 +95,11 @@ { "cell_type": "code", "execution_count": null, + "id": "e5b58d89-5ead-41fa-b295-972ad7b15acf", "metadata": {}, "outputs": [], "source": [ - "# Get a handle to registry\n", + "# Get a handle to workspace\n", "ml_client = MLClient(\n", " credential=credential,\n", " registry_name=\"azure-openai-preview\",\n", @@ -103,6 +110,7 @@ { "attachments": {}, "cell_type": "markdown", + "id": "b48af564", "metadata": {}, "source": [ "# 2. Define and create components into workspace\n", @@ -112,20 +120,19 @@ { "cell_type": "code", "execution_count": null, + "id": "9fe7872f-d02e-49f1-b3f6-4e3c54c57186", "metadata": {}, "outputs": [], "source": [ - "model_import_component = load_component(\n", - " client=ml_client, name=\"openai_model_import\", version=\"0.2.3\"\n", - ")\n", - "finetune_component = load_component(\n", - " client=ml_client, name=\"openai_completions_finetune\", version=\"0.2.4\"\n", + "openai_completions_finetune_component = load_component(\n", + " client=ml_client, name=\"openai_completions_finetune\", version=\"0.2.9\"\n", ")" ] }, { "attachments": {}, "cell_type": "markdown", + "id": "7a744987", "metadata": {}, "source": [ "## 2.2 Inspect loaded components" @@ -134,16 +141,18 @@ { "cell_type": "code", "execution_count": null, + "id": "17744c1b", "metadata": {}, "outputs": [], "source": [ - "print(model_import_component)\n", - "print(finetune_component)" + "print(\"Pipeline component loaded successfully. Component spec:\")\n", + "print(openai_completions_finetune_component)" ] }, { "attachments": {}, "cell_type": "markdown", + "id": "cb7335d1", "metadata": {}, "source": [ "# 3. Sample pipeline job\n", @@ -153,6 +162,7 @@ { "cell_type": "code", "execution_count": null, + "id": "0246232c-dc7b-4d9a-b9c4-de43782121ee", "metadata": {}, "outputs": [], "source": [ @@ -161,37 +171,35 @@ "def pipeline_with_registered_components(\n", " train_dataset, validation_dataset, training_max_epochs=20, model=\"ada\"\n", "):\n", - " \"\"\"E2E pipeline that selects and does finetuning on an OpenAI model using CommandComponents\"\"\"\n", - " # Call component obj as function: apply given inputs & parameters to create a node in pipeline\n", - " selected_model = model_import_component(model=\"ada\")\n", - " selected_model.outputs.output_model.mode = \"mount\"\n", - " finetune_results = finetune_component(\n", - " train_dataset=train_dataset,\n", - " validation_dataset=validation_dataset,\n", - " base_model=selected_model.outputs.output_model,\n", - " n_epochs=training_max_epochs,\n", - " model=model,\n", - " registered_model_name=\"mrpc_test_model\",\n", + " openai_completions_finetune_component_results = (\n", + " openai_completions_finetune_component(\n", + " train_dataset=train_dataset,\n", + " validation_dataset=validation_dataset,\n", + " n_epochs=training_max_epochs,\n", + " model=model,\n", + " registered_model_name=\"ft_ada_model_from_sdk\",\n", + " )\n", " )\n", - " finetune_results.outputs.output_model.mode = \"mount\"\n", "\n", - " return finetune_results\n", + " return openai_completions_finetune_component_results\n", "\n", "\n", "pipeline_job = pipeline_with_registered_components(\n", " train_dataset=Input(type=\"uri_folder\", path=\"data/\"),\n", " validation_dataset=Input(type=\"uri_folder\", path=\"data/\"),\n", " training_max_epochs=1,\n", - " model=\"ada\",\n", + " model=\"ada\", # Select any model from [\"ada\", \"babbage\", \"curie\", \"davinci\", \"text-davinci-fine-tune-002\"]\n", ")\n", "\n", "# set pipeline level compute\n", - "pipeline_job.settings.default_compute = \"serverless\"" + "pipeline_job.settings.default_compute = \"serverless\"\n", + "pipeline_job.settings.force_rerun = True" ] }, { "attachments": {}, "cell_type": "markdown", + "id": "811cddb4", "metadata": {}, "source": [ "## 3.2 Configure workspace details and get a handle to the workspace\n", @@ -202,6 +210,7 @@ { "cell_type": "code", "execution_count": null, + "id": "3e34a18f-0f70-4222-8812-78894f430e05", "metadata": {}, "outputs": [], "source": [ @@ -222,6 +231,7 @@ { "attachments": {}, "cell_type": "markdown", + "id": "3ab904e9", "metadata": {}, "source": [ "## 3.3 Submit pipeline job" @@ -230,6 +240,7 @@ { "cell_type": "code", "execution_count": null, + "id": "109caadf-ac2e-4a61-831b-13a9f4c78e14", "metadata": {}, "outputs": [], "source": [ @@ -243,6 +254,7 @@ { "cell_type": "code", "execution_count": null, + "id": "daf966d8", "metadata": {}, "outputs": [], "source": [ @@ -253,14 +265,23 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10 - SDK V2", + "display_name": "sdkv2", "language": "python", - "name": "python310-sdkv2" + "name": "python3" }, "language_info": { - "name": "python" + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 5 } diff --git a/sdk/python/foundation-models/azure_openai/openai_completions_finetune_pipeline.ipynb b/sdk/python/foundation-models/azure_openai/openai_completions_finetune_pipeline.ipynb deleted file mode 100644 index d93c61cbbc..0000000000 --- a/sdk/python/foundation-models/azure_openai/openai_completions_finetune_pipeline.ipynb +++ /dev/null @@ -1,285 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "id": "69484eca", - "metadata": {}, - "source": [ - "# Build Pipeline with Azure OpenAI PipelineComponents from registry\n", - "\n", - "**Requirements** - In order to benefit from this tutorial, you will need:\n", - "- A basic understanding of Machine Learning\n", - "- An Azure account with an active subscription - [Create an account for free](https://azure.microsoft.com/free/?WT.mc_id=A261C142F)\n", - "- An Azure ML workspace with computer cluster - [Configure workspace](../../configuration.ipynb)\n", - "- A python environment\n", - "- Installed Azure Machine Learning Python SDK v2 - [install instructions](../../../README.md) - check the getting started section\n", - "\n", - "**Learning Objectives** - By the end of this tutorial, you should be able to:\n", - "- Connect to your AML workspace from the Python SDKv2\n", - "- Define and load Azure OpenAI `PipelineComponent` from the registry\n", - "- Create `Pipeline` using loaded component.\n", - "\n", - "**Motivations** - This notebook covers the scenario where a user can load OpenAI components from the registry to create a pipeline and submit the job using sdkv2 " - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "0a46a0d8-d35a-437b-8fe6-25854ef6dd42", - "metadata": {}, - "source": [ - "# 1. Connect to Azure Machine Learning Workspace\n", - "\n", - "The [workspace](https://docs.microsoft.com/en-us/azure/machine-learning/concept-workspace) is the top-level resource for Azure Machine Learning, providing a centralized place to work with all the artifacts you create when you use Azure Machine Learning. In this section we will connect to the workspace in which the job will be run.\n", - "\n", - "## 1.1 Import the required libraries" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f5d327ef-3466-40cf-a6a6-395529cfcf6e", - "metadata": {}, - "outputs": [], - "source": [ - "# Import required libraries\n", - "from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential\n", - "\n", - "from azure.ai.ml import MLClient, Input\n", - "from azure.ai.ml.dsl import pipeline\n", - "from azure.ai.ml import load_component" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "7bc31b17", - "metadata": {}, - "source": [ - "## 1.2 Configure credential\n", - "\n", - "We are using `DefaultAzureCredential` to get access to workspace which should be capable of handling most Azure SDK authentication scenarios. \n", - "\n", - "Reference for more available credentials if it does not work for you: [configure credential example](../../configuration.ipynb), [azure-identity reference doc](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity?view=azure-python).\n", - "Check the [configuration notebook](https://github.com/Azure/azureml-examples/blob/6142c51451561447befa665e8dd6fb3ff80bdb62/sdk/python/jobs/configuration.ipynb) for more details on how to configure credentials and connect to a workspace." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "c4169370-43a9-4bd7-adca-ec3accf14400", - "metadata": {}, - "outputs": [], - "source": [ - "try:\n", - " credential = DefaultAzureCredential()\n", - " # Check if given credential can get token successfully.\n", - " credential.get_token(\"https://management.azure.com/.default\")\n", - "except Exception as ex:\n", - " # Fall back to InteractiveBrowserCredential in case DefaultAzureCredential not work\n", - " credential = InteractiveBrowserCredential()" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "ac24d6f7", - "metadata": {}, - "source": [ - "## 1.3 Get a handle to the registry\n", - "\n", - "We need to initialize a MlClient pointed to the registry where the OpenAI components are available. [Check this api documentation for more details](https://learn.microsoft.com/en-us/python/api/azure-ai-ml/azure.ai.ml.mlclient?view=azure-python)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "e5b58d89-5ead-41fa-b295-972ad7b15acf", - "metadata": {}, - "outputs": [], - "source": [ - "# Get a handle to workspace\n", - "ml_client = MLClient(\n", - " credential=credential, registry_name=\"azureml-preview\", registry_location=\"eastus\"\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "b48af564", - "metadata": {}, - "source": [ - "# 2. Define and create components into workspace\n", - "## 2.1 Load components from registry" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "9fe7872f-d02e-49f1-b3f6-4e3c54c57186", - "metadata": {}, - "outputs": [], - "source": [ - "openai_completions_finetune_component = load_component(\n", - " client=ml_client, name=\"openai_completions_finetune\", version=\"0.0.7\"\n", - ")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "7a744987", - "metadata": {}, - "source": [ - "## 2.2 Inspect loaded components" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "17744c1b", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"Pipeline component loaded successfully. Component spec:\")\n", - "print(openai_completions_finetune_component)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "cb7335d1", - "metadata": {}, - "source": [ - "# 3. Sample pipeline job\n", - "## 3.1 Build pipeline" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "0246232c-dc7b-4d9a-b9c4-de43782121ee", - "metadata": {}, - "outputs": [], - "source": [ - "# Construct pipeline\n", - "@pipeline()\n", - "def pipeline_with_registered_components(\n", - " train_dataset, validation_dataset, training_max_epochs=20, model=\"ada\"\n", - "):\n", - " openai_completions_finetune_component_results = (\n", - " openai_completions_finetune_component(\n", - " train_dataset=train_dataset,\n", - " validation_dataset=validation_dataset,\n", - " n_epochs=training_max_epochs,\n", - " model=model,\n", - " registered_model_name=\"ft_ada_model_from_sdk\",\n", - " )\n", - " )\n", - "\n", - " return openai_completions_finetune_component_results\n", - "\n", - "\n", - "pipeline_job = pipeline_with_registered_components(\n", - " train_dataset=Input(type=\"uri_folder\", path=\"data/\"),\n", - " validation_dataset=Input(type=\"uri_folder\", path=\"data/\"),\n", - " training_max_epochs=1,\n", - " model=\"ada\", # Select any model from [\"ada\", \"babbage\", \"curie\", \"davinci\", \"text-davinci-fine-tune-002\"]\n", - ")\n", - "\n", - "# set pipeline level compute\n", - "pipeline_job.settings.default_compute = \"serverless\"\n", - "pipeline_job.settings.force_rerun = True" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "811cddb4", - "metadata": {}, - "source": [ - "## 3.2 Configure workspace details and get a handle to the workspace\n", - "\n", - "To connect to a workspace, we need identifier parameters - a subscription, resource group and workspace name. We will use these details in the `MLClient` from `azure.ai.ml` to get a handle to the required Azure Machine Learning workspace. We use the default [default azure authentication](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.defaultazurecredential?view=azure-python) as mentioned at the beginning of the notebook." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3e34a18f-0f70-4222-8812-78894f430e05", - "metadata": {}, - "outputs": [], - "source": [ - "# Get a handle to workspace\n", - "ml_client = None\n", - "try:\n", - " ml_client = MLClient.from_config(credential)\n", - "except Exception as ex:\n", - " print(ex)\n", - " # Enter details of your AML workspace\n", - " subscription_id = \"\"\n", - " resource_group = \"\"\n", - " workspace_name = \"\"\n", - "\n", - " ml_client = MLClient(credential, subscription_id, resource_group, workspace_name)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "3ab904e9", - "metadata": {}, - "source": [ - "## 3.3 Submit pipeline job" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "109caadf-ac2e-4a61-831b-13a9f4c78e14", - "metadata": {}, - "outputs": [], - "source": [ - "# Submit pipeline job to workspace\n", - "pipeline_job = ml_client.jobs.create_or_update(\n", - " pipeline_job, experiment_name=\"mrpc_pipeline_test\"\n", - ")\n", - "pipeline_job" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "daf966d8", - "metadata": {}, - "outputs": [], - "source": [ - "# Wait until the job completes\n", - "ml_client.jobs.stream(pipeline_job.name)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3.10 - SDK V2", - "language": "python", - "name": "python310-sdkv2" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/sdk/python/jobs/pipelines/README.md b/sdk/python/jobs/pipelines/README.md index 18b82bcfec..237b6b011c 100644 --- a/sdk/python/jobs/pipelines/README.md +++ b/sdk/python/jobs/pipelines/README.md @@ -9,7 +9,6 @@ Test Status is for branch - **_main_** |Area|Sub-Area|Notebook|Description|Status| |--|--|--|--|--| |jobs|pipelines|[pipeline_with_components_from_yaml](1a_pipeline_with_components_from_yaml/pipeline_with_components_from_yaml.ipynb)|Create pipeline with CommandComponents from local YAML file|[![pipeline_with_components_from_yaml](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1a_pipeline_with_components_from_yaml-pipeline_with_components_from_yaml.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1a_pipeline_with_components_from_yaml-pipeline_with_components_from_yaml.yml)| -|jobs|pipelines|[pipeline_with_components_from_yaml_serverless](1a_pipeline_with_components_from_yaml/pipeline_with_components_from_yaml_serverless.ipynb)|Create pipeline with CommandComponents from local YAML file|[![pipeline_with_components_from_yaml_serverless](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1a_pipeline_with_components_from_yaml-pipeline_with_components_from_yaml_serverless.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1a_pipeline_with_components_from_yaml-pipeline_with_components_from_yaml_serverless.yml)| |jobs|pipelines|[pipeline_with_python_function_components](1b_pipeline_with_python_function_components/pipeline_with_python_function_components.ipynb)|Create pipeline with command_component decorator|[![pipeline_with_python_function_components](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1b_pipeline_with_python_function_components-pipeline_with_python_function_components.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1b_pipeline_with_python_function_components-pipeline_with_python_function_components.yml)| |jobs|pipelines|[pipeline_with_hyperparameter_sweep](1c_pipeline_with_hyperparameter_sweep/pipeline_with_hyperparameter_sweep.ipynb)|Use sweep (hyperdrive) in pipeline to train mnist model using tensorflow|[![pipeline_with_hyperparameter_sweep](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1c_pipeline_with_hyperparameter_sweep-pipeline_with_hyperparameter_sweep.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1c_pipeline_with_hyperparameter_sweep-pipeline_with_hyperparameter_sweep.yml)| |jobs|pipelines|[pipeline_with_non_python_components](1d_pipeline_with_non_python_components/pipeline_with_non_python_components.ipynb)|Create a pipeline with command function|[![pipeline_with_non_python_components](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1d_pipeline_with_non_python_components-pipeline_with_non_python_components.yml/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/sdk-jobs-pipelines-1d_pipeline_with_non_python_components-pipeline_with_non_python_components.yml)|