From e6dcdff40abfa3d9b5763c4cee61a8cd959a82d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Mon, 19 Dec 2022 19:04:28 +0100 Subject: [PATCH 01/66] Add support for cirun on self-hosted GHA runners --- conda_smithy/configure_feedstock.py | 22 ++++++- conda_smithy/templates/cirun.yml.tmpl | 9 +++ ...b-actions.tmpl => github-actions.yml.tmpl} | 58 +++++++++++++++++-- 3 files changed, 81 insertions(+), 8 deletions(-) create mode 100644 conda_smithy/templates/cirun.yml.tmpl rename conda_smithy/templates/{github-actions.tmpl => github-actions.yml.tmpl} (82%) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 2c8fd4ec9..131b43a1b 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -1243,6 +1243,12 @@ def _github_actions_specific_setup( if forge_config["github_actions"]["store_build_artifacts"]: for tmpls in platform_templates.values(): tmpls.append(".scripts/create_conda_build_artifacts.sh") + if ( + forge_config["github_actions"]["self_hosted"] and + any(label.startswith("cirun-") for label in forge_config["github_actions"]["self_hosted_labels"]) + ): + platform_templates["linux"].append("cirun.yml") + template_files = platform_templates.get(platform, []) _render_template_exe_files( @@ -1259,7 +1265,7 @@ def render_github_actions( target_path = os.path.join( forge_dir, ".github", "workflows", "conda-build.yml" ) - template_filename = "github-actions.tmpl" + template_filename = "github-actions.yml.tmpl" fast_finish_text = "" ( @@ -1269,7 +1275,7 @@ def render_github_actions( upload_packages, ) = _get_platforms_of_provider("github_actions", forge_config) - logger.debug("github platforms retreived") + logger.debug("github platforms retrieved") remove_file_or_dir(target_path) return _render_ci_provider( @@ -1763,10 +1769,22 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): }, "github_actions": { "self_hosted": False, + "self_hosted_labels": [], + "self_hosted_triggers": ["push"], # Toggle creating artifacts for conda build_artifacts dir "store_build_artifacts": False, "artifact_retention_days": 14, }, + "cirun_runners": [ + { + "name": "cirun-openstack-gpu", + "labels": ["cirun-openstack-gpu"], + "cloud": "openstack", + "instance_type": "gpu_tiny", + "machine_image": "ubuntu-focal-nvidia-14112022", + "region": "RegionOne", + } + ], "recipe_dir": "recipe", "skip_render": [], "bot": {"automerge": False}, diff --git a/conda_smithy/templates/cirun.yml.tmpl b/conda_smithy/templates/cirun.yml.tmpl new file mode 100644 index 000000000..a8da5f51a --- /dev/null +++ b/conda_smithy/templates/cirun.yml.tmpl @@ -0,0 +1,9 @@ +{%- if github_actions.self_hosted -%} +runners: +{%- for runner in cirun_runners %} + - # Custom runner {{ loop.index }} +{%- for key, value in runner.items() %} + {{ key }}: {{ value }} +{%- endfor -%} +{%- endfor -%} +{%- endif -%} diff --git a/conda_smithy/templates/github-actions.tmpl b/conda_smithy/templates/github-actions.yml.tmpl similarity index 82% rename from conda_smithy/templates/github-actions.tmpl rename to conda_smithy/templates/github-actions.yml.tmpl index 951a8fc45..baab801e3 100644 --- a/conda_smithy/templates/github-actions.tmpl +++ b/conda_smithy/templates/github-actions.yml.tmpl @@ -12,7 +12,7 @@ name: Build conda package {%- if github_actions.self_hosted %} -on: [push] +on: {{ github_actions.self_hosted_triggers }} {%- else %} on: [push, pull_request] {%- endif %} @@ -38,24 +38,61 @@ jobs: {%- if data.build_platform.startswith("osx-64") %} os: macos {%- if github_actions.self_hosted %} - labels: ['macOS', 'self-hosted', 'x64'] + labels: + - macOS + - x64 + - self-hosted + {%- for label in github_actions.self_hosted_labels %} + {%- if label.startswith('cirun-') %} + {# See https://docs.cirun.io/reference/unique-runner-labels #} + {%- set label = label ~ "--${{ github.run_id }}-" ~ data.short_config_name %} + {%- endif %} + - '{{ label }}' + {%- endfor %} {%- endif %} {%- elif data.build_platform.startswith("osx-arm64") %} os: macos {%- if github_actions.self_hosted %} - labels: ['macOS', 'self-hosted', 'ARM64'] + labels: + - macOS + - ARM64 + - self-hosted + {%- for label in github_actions.self_hosted_labels %} + {%- if label.startswith('cirun-') %} + {%- set label = label ~ "--${{ github.run_id }}" ~ data.short_config_name %} + {%- endif %} + - '{{ label }}' + {%- endfor %} {%- endif %} {%- elif data.build_platform.startswith("linux-64") %} DOCKER_IMAGE: {{ data.config["docker_image"][-1] }} os: ubuntu {%- if github_actions.self_hosted %} - labels: ['linux', 'self-hosted', 'x64'] + labels: + - linux + - x64 + - self-hosted + {%- for label in github_actions.self_hosted_labels %} + {%- if label.startswith('cirun-') %} + {%- set label = label ~ "--${{ github.run_id }}" ~ data.short_config_name %} + {%- endif %} + - '{{ label }}' + {%- endfor %} {%- endif %} {%- elif data.build_platform.startswith("linux-aarch64") %} DOCKER_IMAGE: {{ data.config["docker_image"][-1] }} os: ubuntu {%- if github_actions.self_hosted %} - labels: ['linux', 'self-hosted', 'ARM64'] + labels: + - linux + - ARM64 + - self-hosted + {%- for label in github_actions.self_hosted_labels %} + {%- if label.startswith('cirun-') %} + {%- set label = label ~ "--${{ github.run_id }}" ~ data.short_config_name %} + {%- endif %} + - '{{ label }}' + {%- endfor %} {%- endif %} {%- elif data.build_platform.startswith("linux") %} DOCKER_IMAGE: {{ data.config["docker_image"][-1] }} @@ -63,7 +100,16 @@ jobs: {%- elif data.build_platform.startswith("win") %} os: windows {%- if github_actions.self_hosted %} - labels: ['windows', 'self-hosted', 'x64'] + labels: + - windows + - x64 + - self-hosted + {%- for label in github_actions.self_hosted_labels %} + {%- if label.startswith('cirun-') %} + {%- set label = label ~ "--${{ github.run_id }}" ~ data.short_config_name %} + {%- endif %} + - '{{ label }}' + {%- endfor %} {%- endif %} {%- endif %} {%- endfor %} From fd031cd2e1749bef2605fceed6e6e0a69e1365de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Mon, 19 Dec 2022 20:05:57 +0100 Subject: [PATCH 02/66] pre-commit --- conda_smithy/configure_feedstock.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 131b43a1b..421352e88 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -1243,9 +1243,9 @@ def _github_actions_specific_setup( if forge_config["github_actions"]["store_build_artifacts"]: for tmpls in platform_templates.values(): tmpls.append(".scripts/create_conda_build_artifacts.sh") - if ( - forge_config["github_actions"]["self_hosted"] and - any(label.startswith("cirun-") for label in forge_config["github_actions"]["self_hosted_labels"]) + if forge_config["github_actions"]["self_hosted"] and any( + label.startswith("cirun-") + for label in forge_config["github_actions"]["self_hosted_labels"] ): platform_templates["linux"].append("cirun.yml") From 7836ccedf6f72423237fce03b35e392590c152d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Tue, 27 Dec 2022 13:11:54 +0100 Subject: [PATCH 03/66] add more labels and runners --- conda_smithy/configure_feedstock.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 421352e88..1498be6d8 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -1778,11 +1778,29 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): "cirun_runners": [ { "name": "cirun-openstack-gpu", - "labels": ["cirun-openstack-gpu"], + "labels": [ + "linux", + "x64", + "self-hosted", + "cirun-openstack-gpu" + ], "cloud": "openstack", "instance_type": "gpu_tiny", "machine_image": "ubuntu-focal-nvidia-14112022", "region": "RegionOne", + }, + { + "name": "cirun-openstack-cpu", + "labels": [ + "linux", + "x64", + "self-hosted", + "cirun-openstack-cpu", + ], + "cloud": "openstack", + "instance_type": "ci_medium", + "machine_image": "ubuntu-2204-cloud-jammy-20221104", + "region": "RegionOne", } ], "recipe_dir": "recipe", From 2598d9ab63225f6fc6f05761c2afa24ac40bbafb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Mon, 9 Jan 2023 11:10:07 +0100 Subject: [PATCH 04/66] produce .cirun.yml --- conda_smithy/configure_feedstock.py | 18 ++++++++++-------- .../{cirun.yml.tmpl => .cirun.yml.tmpl} | 0 2 files changed, 10 insertions(+), 8 deletions(-) rename conda_smithy/templates/{cirun.yml.tmpl => .cirun.yml.tmpl} (100%) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 1498be6d8..32d5b25b0 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -1240,17 +1240,19 @@ def _github_actions_specific_setup( ], "win": [], } - if forge_config["github_actions"]["store_build_artifacts"]: - for tmpls in platform_templates.values(): - tmpls.append(".scripts/create_conda_build_artifacts.sh") - if forge_config["github_actions"]["self_hosted"] and any( - label.startswith("cirun-") - for label in forge_config["github_actions"]["self_hosted_labels"] - ): - platform_templates["linux"].append("cirun.yml") template_files = platform_templates.get(platform, []) + # Templates for all platforms + if forge_config["github_actions"]["store_build_artifacts"]: + template_files.append(".scripts/create_conda_build_artifacts.sh") + + if forge_config["github_actions"]["self_hosted"]: + for label in forge_config["github_actions"]["self_hosted_labels"]: + if label.startswith("cirun-"): + template_files.append(".cirun.yml") + break + _render_template_exe_files( forge_config=forge_config, jinja_env=jinja_env, diff --git a/conda_smithy/templates/cirun.yml.tmpl b/conda_smithy/templates/.cirun.yml.tmpl similarity index 100% rename from conda_smithy/templates/cirun.yml.tmpl rename to conda_smithy/templates/.cirun.yml.tmpl From e6158f9763ca8f6840a7db4eab6c0f5fbb6f7468 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Mon, 9 Jan 2023 11:16:38 +0100 Subject: [PATCH 05/66] update runner config --- conda_smithy/configure_feedstock.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 32d5b25b0..0743bc94f 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -1781,20 +1781,20 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): { "name": "cirun-openstack-gpu", "labels": [ - "linux", + "linux", "x64", "self-hosted", "cirun-openstack-gpu" ], "cloud": "openstack", - "instance_type": "gpu_tiny", - "machine_image": "ubuntu-focal-nvidia-14112022", + "instance_type": "gpu_medium", + "machine_image": "ubuntu-2204-nvidia-docker-20221229", "region": "RegionOne", }, { "name": "cirun-openstack-cpu", "labels": [ - "linux", + "linux", "x64", "self-hosted", "cirun-openstack-cpu", From 5f96e5effeb6297486f5bceab62542d5b79fc0c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Mon, 9 Jan 2023 11:17:09 +0100 Subject: [PATCH 06/66] refactor os-matrix building in gha template --- .../templates/github-actions.yml.tmpl | 119 ++++++++---------- 1 file changed, 52 insertions(+), 67 deletions(-) diff --git a/conda_smithy/templates/github-actions.yml.tmpl b/conda_smithy/templates/github-actions.yml.tmpl index baab801e3..48d19fb21 100644 --- a/conda_smithy/templates/github-actions.yml.tmpl +++ b/conda_smithy/templates/github-actions.yml.tmpl @@ -17,6 +17,40 @@ on: {{ github_actions.self_hosted_triggers }} on: [push, pull_request] {%- endif %} +{%- set platform_vars = { + 'osx-64': { + 'os': 'macos', + 'self_hosted_label_os': 'macOS', + 'self_hosted_label_arch': 'x64', + }, + 'osx-arm64': { + 'os': 'macos', + 'self_hosted_label_os': 'macOS', + 'self_hosted_label_arch': 'arm64', + }, + 'linux-64': { + 'os': 'ubuntu', + 'self_hosted_label_os': 'linux', + 'self_hosted_label_arch': 'x64', + }, + 'linux-aarch64': { + 'os': 'ubuntu', + 'self_hosted_label_os': 'linux', + 'self_hosted_label_arch': '', + }, + 'linux-ppcle64': { + 'os': 'ubuntu', + 'self_hosted_label_os': 'linux', + 'self_hosted_label_arch': '', + }, + 'win-64': { + 'os': 'windows', + 'self_hosted_label_os': 'windows', + 'self_hosted_label_arch': 'x64', + }, +} +%} + jobs: build: name: {% raw %}${{ matrix.CONFIG }}{% endraw %} @@ -35,82 +69,31 @@ jobs: SHORT_CONFIG: {{ data.short_config_name }} {%- endif %} UPLOAD_PACKAGES: {{ data.upload }} - {%- if data.build_platform.startswith("osx-64") %} - os: macos + os: {{ platform_vars[data.build_platform]['os'] }} + {#- namespace needed to handle jinja scoping rules #} + {%- set ns = namespace(gpu=False) %} {%- if github_actions.self_hosted %} labels: - - macOS - - x64 - self-hosted + - {{ platform_vars[data.build_platform]['self_hosted_label_os'] }} + {%- if platform_vars[data.build_platform]['self_hosted_label_arch'] %} + - {{ platform_vars[data.build_platform]['self_hosted_label_arch'] }} + {%- endif %} {%- for label in github_actions.self_hosted_labels %} - {%- if label.startswith('cirun-') %} - {# See https://docs.cirun.io/reference/unique-runner-labels #} - {%- set label = label ~ "--${{ github.run_id }}-" ~ data.short_config_name %} + {%- if 'gpu' in label %} + {%- set ns.gpu = True %} {%- endif %} - - '{{ label }}' - {%- endfor %} - {%- endif %} - {%- elif data.build_platform.startswith("osx-arm64") %} - os: macos - {%- if github_actions.self_hosted %} - labels: - - macOS - - ARM64 - - self-hosted - {%- for label in github_actions.self_hosted_labels %} {%- if label.startswith('cirun-') %} - {%- set label = label ~ "--${{ github.run_id }}" ~ data.short_config_name %} + {%- set label = label ~ "--${{ github.run_id }}-" ~ data.short_config_name %} {%- endif %} - - '{{ label }}' + - {{ label }} {%- endfor %} {%- endif %} - {%- elif data.build_platform.startswith("linux-64") %} + {%- if data.build_platform.startswith("linux") %} DOCKER_IMAGE: {{ data.config["docker_image"][-1] }} - os: ubuntu - {%- if github_actions.self_hosted %} - labels: - - linux - - x64 - - self-hosted - {%- for label in github_actions.self_hosted_labels %} - {%- if label.startswith('cirun-') %} - {%- set label = label ~ "--${{ github.run_id }}" ~ data.short_config_name %} - {%- endif %} - - '{{ label }}' - {%- endfor %} - {%- endif %} - {%- elif data.build_platform.startswith("linux-aarch64") %} - DOCKER_IMAGE: {{ data.config["docker_image"][-1] }} - os: ubuntu - {%- if github_actions.self_hosted %} - labels: - - linux - - ARM64 - - self-hosted - {%- for label in github_actions.self_hosted_labels %} - {%- if label.startswith('cirun-') %} - {%- set label = label ~ "--${{ github.run_id }}" ~ data.short_config_name %} - {%- endif %} - - '{{ label }}' - {%- endfor %} - {%- endif %} - {%- elif data.build_platform.startswith("linux") %} - DOCKER_IMAGE: {{ data.config["docker_image"][-1] }} - os: ubuntu - {%- elif data.build_platform.startswith("win") %} - os: windows - {%- if github_actions.self_hosted %} - labels: - - windows - - x64 - - self-hosted - {%- for label in github_actions.self_hosted_labels %} - {%- if label.startswith('cirun-') %} - {%- set label = label ~ "--${{ github.run_id }}" ~ data.short_config_name %} - {%- endif %} - - '{{ label }}' - {%- endfor %} - {%- endif %} + {%- if ns.gpu %} + CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" + {%- endif %} {%- endif %} {%- endfor %} steps: @@ -132,7 +115,9 @@ jobs: UPLOAD_ON_BRANCH: {{ upload_on_branch }} {%- endif %} {%- if docker.run_args is defined %} - CONDA_FORGE_DOCKER_RUN_ARGS: "{{ docker.run_args }}" + CONDA_FORGE_DOCKER_RUN_ARGS: "{{ docker.run_args }} {% raw %}${{ matrix.CONDA_FORGE_DOCKER_RUN_ARGS }}{% endraw %}" +{%- else %} + CONDA_FORGE_DOCKER_RUN_ARGS: "{% raw %}${{ matrix.CONDA_FORGE_DOCKER_RUN_ARGS }}{% endraw %}" {%- endif %} {%- for secret in secrets %} {{ secret }}: {% raw %}${{{% endraw %} secrets.{{ secret }} {% raw %}}}{% endraw %} From a366a1f12bdba55afd00cbf0249acf267052db8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Tue, 10 Jan 2023 18:52:09 +0100 Subject: [PATCH 07/66] update vm [ci skip] --- conda_smithy/configure_feedstock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 0743bc94f..77fab0327 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -1788,7 +1788,7 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): ], "cloud": "openstack", "instance_type": "gpu_medium", - "machine_image": "ubuntu-2204-nvidia-docker-20221229", + "machine_image": "ubuntu-2204-nvidia-container-runtime-20230107", "region": "RegionOne", }, { From 4104525d5a02a4d0b2db08660723e1d7a491180f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Mon, 23 Jan 2023 17:18:34 +0100 Subject: [PATCH 08/66] default to gpu_large --- conda_smithy/configure_feedstock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 77fab0327..ae470fd5a 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -1787,7 +1787,7 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): "cirun-openstack-gpu" ], "cloud": "openstack", - "instance_type": "gpu_medium", + "instance_type": "gpu_large", "machine_image": "ubuntu-2204-nvidia-container-runtime-20230107", "region": "RegionOne", }, From d6ae11ac39deda9c2c2aedf40fc7ea1e92a7e5a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Mon, 23 Jan 2023 18:00:32 +0100 Subject: [PATCH 09/66] support cancel_in_progress --- conda_smithy/configure_feedstock.py | 1 + conda_smithy/templates/github-actions.yml.tmpl | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index ae470fd5a..b59b08d90 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -1773,6 +1773,7 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): "self_hosted": False, "self_hosted_labels": [], "self_hosted_triggers": ["push"], + "cancel_in_progress": False, # Toggle creating artifacts for conda build_artifacts dir "store_build_artifacts": False, "artifact_retention_days": 14, diff --git a/conda_smithy/templates/github-actions.yml.tmpl b/conda_smithy/templates/github-actions.yml.tmpl index 48d19fb21..7582bb728 100644 --- a/conda_smithy/templates/github-actions.yml.tmpl +++ b/conda_smithy/templates/github-actions.yml.tmpl @@ -17,6 +17,12 @@ on: {{ github_actions.self_hosted_triggers }} on: [push, pull_request] {%- endif %} +{%- if github_actions.cancel_in_progress or github_actions.self_hosted %} +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true +{%- endif %} + {%- set platform_vars = { 'osx-64': { 'os': 'macos', From 9e014005bda2ef01625682c5257171b168c8e195 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Mon, 23 Jan 2023 18:05:02 +0100 Subject: [PATCH 10/66] raw --- conda_smithy/templates/github-actions.yml.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_smithy/templates/github-actions.yml.tmpl b/conda_smithy/templates/github-actions.yml.tmpl index 7582bb728..9871a10db 100644 --- a/conda_smithy/templates/github-actions.yml.tmpl +++ b/conda_smithy/templates/github-actions.yml.tmpl @@ -19,7 +19,7 @@ on: [push, pull_request] {%- if github_actions.cancel_in_progress or github_actions.self_hosted %} concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + group: {% raw %}${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}{% endraw %} cancel-in-progress: true {%- endif %} From bc96c16b646293a4052cc71d1aa8a9df6d45a2a5 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 26 Jul 2023 16:55:14 +0200 Subject: [PATCH 11/66] add stubs for opt-in ci registration --- conda_smithy/ci_register.py | 35 +++++++++++++++++++++++++++ conda_smithy/cli.py | 47 +++++++++++++++++++++++++++++++++++-- 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/conda_smithy/ci_register.py b/conda_smithy/ci_register.py index b6a2117ed..85d6484e5 100755 --- a/conda_smithy/ci_register.py +++ b/conda_smithy/ci_register.py @@ -423,6 +423,11 @@ def add_project_to_travis(user, project): print(" * {}/{} registered on travis-ci".format(user, project)) +def remove_project_from_travis(user, repo): + "TODO: Revoke access to the repo for travis." + pass + + def travis_token_update_conda_forge_config( feedstock_config_path, user, project ): @@ -571,6 +576,36 @@ def travis_cleanup(org, project): github.remove_from_project(gh, org, project) +def ensure_cirun_app_installed(owner): + "Install the cirun.io app for this owner (user or org). Only the first time." + pass + + +def enable_cirun_for_project(owner, project): + "Enable the cirun.io app for a particular repository." + pass + + +def disable_cirun_for_project(owner, project): + "Disable the cirun.io app for a particular repository." + pass + + +def add_project_to_cirun_resource(owner, project, resource): + "Grant access to a cirun resource to a particular repository, with a particular policy." + pass + + +def revoke_access_to_cirun_resource(owner, project, resource): + "Revoke access to a cirun resource to a particular repository, with a particular policy." + pass + + +def enabled_cirun_resources(owner, project): + "Which resources are currently enabled for this project" + pass + + def get_conda_hook_info(hook_url, events): payload = { "name": "web", diff --git a/conda_smithy/cli.py b/conda_smithy/cli.py index cf5558933..f4b172d32 100644 --- a/conda_smithy/cli.py +++ b/conda_smithy/cli.py @@ -208,7 +208,7 @@ def __init__(self, parser): # conda-smithy register-ci ./ super(RegisterCI, self).__init__( parser, - "Register a feedstock at the CI " "services which do the builds.", + "Register a feedstock at the CI services which do the builds.", ) scp = self.subcommand_parser scp.add_argument( @@ -238,6 +238,7 @@ def __init__(self, parser): "Appveyor", "Drone", "Webservice", + "Cirun", ]: scp.add_argument( "--without-{}".format(ci.lower()), @@ -257,6 +258,17 @@ def __init__(self, parser): action="append", help="drone server URL to register this repo. multiple values allowed", ) + scp.add_argument( + "--cirun-resources", + action="append", + help="cirun resources to enable for this repo. multiple values allowed", + ) + scp.add_argument( + "--remove", + action="store_true", + help="Revoke access to the configured CI services. " + "Only available for Travis and Cirun for now", + ) def __call__(self, args): from conda_smithy import ci_register @@ -278,7 +290,17 @@ def __call__(self, args): ) print("CI Summary for {}/{} (can take ~30s):".format(owner, repo)) - + if args.remove and any([ + args.azure, + args.circle, + args.appveyor, + args.drone, + args.webservice, + args.anaconda_token, + ]): + raise RuntimeError( + "The --remove flag is only supported for Cirun for now" + ) if not args.anaconda_token: print( "Warning: By not registering an Anaconda/Binstar token" @@ -345,6 +367,27 @@ def __call__(self, args): else: print("Drone registration disabled.") + if args.cirun: + if args.remove: + if args.cirun_resources: + to_remove = args.cirun_resources + else: + to_remove = ci_register.enabled_cirun_resources(owner, repo) + for resource in to_remove: + ci_register.remove_project_from_cirun_resource( + owner, repo, resource + ) + current_resources = ci_register.enabled_cirun_resources(owner, repo) + if not current_resources: + ci_register.remove_project_from_cirun(owner, repo) + else: + ci_register.ensure_cirun_app_installed(owner) + ci_register.enable_cirun_for_project(owner, repo) + for resource in args.cirun_resources: + ci_register.add_project_to_cirun_resource(owner, repo, resource) + else: + print("Cirun registration disabled.") + if args.webservice: ci_register.add_conda_forge_webservice_hooks(owner, repo) else: From d3258f94149ae2586e740a01fa56e0d33bfde477 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Sat, 5 Aug 2023 15:06:04 +0100 Subject: [PATCH 12/66] TEMP --- conda_smithy/ci_register.py | 28 ++++++++++++++++++++-------- conda_smithy/cli.py | 3 ++- requirements.txt | 1 + 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/conda_smithy/ci_register.py b/conda_smithy/ci_register.py index 85d6484e5..07ce1fa7c 100755 --- a/conda_smithy/ci_register.py +++ b/conda_smithy/ci_register.py @@ -4,6 +4,8 @@ import time import sys +from cirun import Cirun + from . import github from .utils import update_conda_forge_config @@ -576,33 +578,43 @@ def travis_cleanup(org, project): github.remove_from_project(gh, org, project) -def ensure_cirun_app_installed(owner): +def ensure_cirun_app_installed(owner, repo): "Install the cirun.io app for this owner (user or org). Only the first time." pass def enable_cirun_for_project(owner, project): - "Enable the cirun.io app for a particular repository." - pass + """Enable the cirun.io app for a particular repository.""" + cirun = Cirun() + return cirun.set_repo(f"{owner}/{project}", installation_id=123) def disable_cirun_for_project(owner, project): - "Disable the cirun.io app for a particular repository." - pass + """Disable the cirun.io app for a particular repository.""" + cirun = Cirun() + return cirun.set_repo(f"{owner}/{project}", active=False) def add_project_to_cirun_resource(owner, project, resource): - "Grant access to a cirun resource to a particular repository, with a particular policy." + """Grant access to a cirun resource to a particular repository, with a particular policy.""" pass def revoke_access_to_cirun_resource(owner, project, resource): - "Revoke access to a cirun resource to a particular repository, with a particular policy." + """Revoke access to a cirun resource to a particular repository, with a particular policy.""" pass def enabled_cirun_resources(owner, project): - "Which resources are currently enabled for this project" + """Which resources are currently enabled for this project""" + pass + + +def remove_project_from_cirun_resource(owner, repo, resource): + pass + + +def remove_project_from_cirun(owner, repo): pass diff --git a/conda_smithy/cli.py b/conda_smithy/cli.py index f4b172d32..35be007c7 100644 --- a/conda_smithy/cli.py +++ b/conda_smithy/cli.py @@ -373,6 +373,7 @@ def __call__(self, args): to_remove = args.cirun_resources else: to_remove = ci_register.enabled_cirun_resources(owner, repo) + for resource in to_remove: ci_register.remove_project_from_cirun_resource( owner, repo, resource @@ -381,7 +382,7 @@ def __call__(self, args): if not current_resources: ci_register.remove_project_from_cirun(owner, repo) else: - ci_register.ensure_cirun_app_installed(owner) + # ci_register.ensure_cirun_app_installed(owner, repo) ci_register.enable_cirun_for_project(owner, repo) for resource in args.cirun_resources: ci_register.add_project_to_cirun_resource(owner, repo, resource) diff --git a/requirements.txt b/requirements.txt index e0400e9b5..f4f24fb6e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ shellcheck scrypt license-expression setuptools_scm >=7 +cirun From f5dad7528a62b15a501a9bc4e29a7329242086bb Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Sun, 6 Aug 2023 16:19:23 +0100 Subject: [PATCH 13/66] add utilities to handle adding cirun --- conda_smithy/ci_register.py | 25 +++++++++++++++++++------ conda_smithy/cli.py | 10 ++++++---- requirements.txt | 2 +- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/conda_smithy/ci_register.py b/conda_smithy/ci_register.py index 07ce1fa7c..83de79526 100755 --- a/conda_smithy/ci_register.py +++ b/conda_smithy/ci_register.py @@ -585,33 +585,36 @@ def ensure_cirun_app_installed(owner, repo): def enable_cirun_for_project(owner, project): """Enable the cirun.io app for a particular repository.""" - cirun = Cirun() + cirun = _get_cirun_client() return cirun.set_repo(f"{owner}/{project}", installation_id=123) def disable_cirun_for_project(owner, project): """Disable the cirun.io app for a particular repository.""" - cirun = Cirun() + cirun = cirun = _get_cirun_client() return cirun.set_repo(f"{owner}/{project}", active=False) def add_project_to_cirun_resource(owner, project, resource): """Grant access to a cirun resource to a particular repository, with a particular policy.""" - pass + cirun = _get_cirun_client() + cirun.add_repo_to_resources("conda-forge", project, resource) def revoke_access_to_cirun_resource(owner, project, resource): """Revoke access to a cirun resource to a particular repository, with a particular policy.""" - pass + cirun = _get_cirun_client() + cirun.remove_repo_from_resources("conda-forge", project, [resource]) def enabled_cirun_resources(owner, project): """Which resources are currently enabled for this project""" - pass + cirun = _get_cirun_client() + return cirun.get_repo_resources("conda-forge", project) def remove_project_from_cirun_resource(owner, repo, resource): - pass + revoke_access_to_cirun_resource(owner, project=repo, resource=resource) def remove_project_from_cirun(owner, repo): @@ -692,6 +695,16 @@ def _get_anaconda_token(): ) +def _get_cirun_client(): + try: + return Cirun() + except KeyError: + raise RuntimeError( + "You must have CIRUN_API_KEY defined to do Cirun CI registration" + "This requirement can be overriden by specifying `--without-cirun`" + ) + + if __name__ == "__main__": import argparse diff --git a/conda_smithy/cli.py b/conda_smithy/cli.py index 35be007c7..9e6de885c 100644 --- a/conda_smithy/cli.py +++ b/conda_smithy/cli.py @@ -372,15 +372,17 @@ def __call__(self, args): if args.cirun_resources: to_remove = args.cirun_resources else: - to_remove = ci_register.enabled_cirun_resources(owner, repo) + # to_remove = ci_register.enabled_cirun_resources(owner, repo) + # This will remove everything + to_remove = ["*"] for resource in to_remove: ci_register.remove_project_from_cirun_resource( owner, repo, resource ) - current_resources = ci_register.enabled_cirun_resources(owner, repo) - if not current_resources: - ci_register.remove_project_from_cirun(owner, repo) + # current_resources = ci_register.enabled_cirun_resources(owner, repo) + # if not current_resources: + # ci_register.remove_project_from_cirun(owner, repo) else: # ci_register.ensure_cirun_app_installed(owner, repo) ci_register.enable_cirun_for_project(owner, repo) diff --git a/requirements.txt b/requirements.txt index f4f24fb6e..7ccf53c15 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ setuptools conda -conda-build >=3.4.2 +conda-build==2.1.5 jinja2 requests pycryptodome From eb8811eac35c00c003111e794388d7fabd05e215 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Thu, 17 Aug 2023 13:15:27 +0530 Subject: [PATCH 14/66] move cirun stuff to cirun utils --- conda_smithy/ci_register.py | 55 -------------------------- conda_smithy/cirun_utils.py | 54 +++++++++++++++++++++++++ conda_smithy/cli.py | 10 +++-- conda_smithy/templates/.cirun.yml.tmpl | 9 ----- 4 files changed, 60 insertions(+), 68 deletions(-) create mode 100644 conda_smithy/cirun_utils.py delete mode 100644 conda_smithy/templates/.cirun.yml.tmpl diff --git a/conda_smithy/ci_register.py b/conda_smithy/ci_register.py index 83de79526..db0bd4c64 100755 --- a/conda_smithy/ci_register.py +++ b/conda_smithy/ci_register.py @@ -4,8 +4,6 @@ import time import sys -from cirun import Cirun - from . import github from .utils import update_conda_forge_config @@ -578,49 +576,6 @@ def travis_cleanup(org, project): github.remove_from_project(gh, org, project) -def ensure_cirun_app_installed(owner, repo): - "Install the cirun.io app for this owner (user or org). Only the first time." - pass - - -def enable_cirun_for_project(owner, project): - """Enable the cirun.io app for a particular repository.""" - cirun = _get_cirun_client() - return cirun.set_repo(f"{owner}/{project}", installation_id=123) - - -def disable_cirun_for_project(owner, project): - """Disable the cirun.io app for a particular repository.""" - cirun = cirun = _get_cirun_client() - return cirun.set_repo(f"{owner}/{project}", active=False) - - -def add_project_to_cirun_resource(owner, project, resource): - """Grant access to a cirun resource to a particular repository, with a particular policy.""" - cirun = _get_cirun_client() - cirun.add_repo_to_resources("conda-forge", project, resource) - - -def revoke_access_to_cirun_resource(owner, project, resource): - """Revoke access to a cirun resource to a particular repository, with a particular policy.""" - cirun = _get_cirun_client() - cirun.remove_repo_from_resources("conda-forge", project, [resource]) - - -def enabled_cirun_resources(owner, project): - """Which resources are currently enabled for this project""" - cirun = _get_cirun_client() - return cirun.get_repo_resources("conda-forge", project) - - -def remove_project_from_cirun_resource(owner, repo, resource): - revoke_access_to_cirun_resource(owner, project=repo, resource=resource) - - -def remove_project_from_cirun(owner, repo): - pass - - def get_conda_hook_info(hook_url, events): payload = { "name": "web", @@ -695,16 +650,6 @@ def _get_anaconda_token(): ) -def _get_cirun_client(): - try: - return Cirun() - except KeyError: - raise RuntimeError( - "You must have CIRUN_API_KEY defined to do Cirun CI registration" - "This requirement can be overriden by specifying `--without-cirun`" - ) - - if __name__ == "__main__": import argparse diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py new file mode 100644 index 000000000..6208783ac --- /dev/null +++ b/conda_smithy/cirun_utils.py @@ -0,0 +1,54 @@ +from cirun import Cirun + + +def ensure_cirun_app_installed(owner, repo): + "Install the cirun.io app for this owner (user or org). Only the first time." + pass + + +def enable_cirun_for_project(owner, project): + """Enable the cirun.io app for a particular repository.""" + cirun = _get_cirun_client() + return cirun.set_repo(f"{owner}/{project}", installation_id=123) + + +def disable_cirun_for_project(owner, project): + """Disable the cirun.io app for a particular repository.""" + cirun = _get_cirun_client() + return cirun.set_repo(f"{owner}/{project}", active=False) + + +def add_project_to_cirun_resource(owner, project, resource): + """Grant access to a cirun resource to a particular repository, with a particular policy.""" + cirun = _get_cirun_client() + cirun.add_repo_to_resources("conda-forge", project, resource) + + +def revoke_access_to_cirun_resource(owner, project, resource): + """Revoke access to a cirun resource to a particular repository, with a particular policy.""" + cirun = _get_cirun_client() + cirun.remove_repo_from_resources("conda-forge", project, [resource]) + + +def enabled_cirun_resources(owner, project): + """Which resources are currently enabled for this project""" + cirun = _get_cirun_client() + return cirun.get_repo_resources("conda-forge", project) + + +def remove_project_from_cirun_resource(owner, repo, resource): + revoke_access_to_cirun_resource(owner, project=repo, resource=resource) + + +def remove_project_from_cirun(owner, repo): + pass + + +def _get_cirun_client(): + try: + return Cirun() + except KeyError: + raise RuntimeError( + "You must have CIRUN_API_KEY defined to do Cirun CI registration" + "This requirement can be overriden by specifying `--without-cirun`" + ) diff --git a/conda_smithy/cli.py b/conda_smithy/cli.py index 9e6de885c..21a6022a5 100644 --- a/conda_smithy/cli.py +++ b/conda_smithy/cli.py @@ -12,6 +12,8 @@ import conda # noqa import conda_build.api from conda_build.metadata import MetaData + +import conda_smithy.cirun_utils from conda_smithy.utils import get_feedstock_name_from_meta, merge_dict from ruamel.yaml import YAML @@ -299,7 +301,7 @@ def __call__(self, args): args.anaconda_token, ]): raise RuntimeError( - "The --remove flag is only supported for Cirun for now" + "The --remove flag is only supported for Cirun for now" ) if not args.anaconda_token: print( @@ -377,7 +379,7 @@ def __call__(self, args): to_remove = ["*"] for resource in to_remove: - ci_register.remove_project_from_cirun_resource( + conda_smithy.cirun_utils.remove_project_from_cirun_resource( owner, repo, resource ) # current_resources = ci_register.enabled_cirun_resources(owner, repo) @@ -385,9 +387,9 @@ def __call__(self, args): # ci_register.remove_project_from_cirun(owner, repo) else: # ci_register.ensure_cirun_app_installed(owner, repo) - ci_register.enable_cirun_for_project(owner, repo) + conda_smithy.cirun_utils.enable_cirun_for_project(owner, repo) for resource in args.cirun_resources: - ci_register.add_project_to_cirun_resource(owner, repo, resource) + conda_smithy.cirun_utils.add_project_to_cirun_resource(owner, repo, resource) else: print("Cirun registration disabled.") diff --git a/conda_smithy/templates/.cirun.yml.tmpl b/conda_smithy/templates/.cirun.yml.tmpl deleted file mode 100644 index a8da5f51a..000000000 --- a/conda_smithy/templates/.cirun.yml.tmpl +++ /dev/null @@ -1,9 +0,0 @@ -{%- if github_actions.self_hosted -%} -runners: -{%- for runner in cirun_runners %} - - # Custom runner {{ loop.index }} -{%- for key, value in runner.items() %} - {{ key }}: {{ value }} -{%- endfor -%} -{%- endfor -%} -{%- endif -%} From f672f73014cd8ab92329fb1c448e41b5b101a78c Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 21 Aug 2023 23:23:04 +0530 Subject: [PATCH 15/66] some cleanups --- conda_smithy/cirun_utils.py | 41 +++++++++++++------------------------ conda_smithy/cli.py | 7 ++++--- conda_smithy/utils.py | 1 - 3 files changed, 18 insertions(+), 31 deletions(-) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index 6208783ac..fc46c8592 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -1,47 +1,34 @@ from cirun import Cirun -def ensure_cirun_app_installed(owner, repo): - "Install the cirun.io app for this owner (user or org). Only the first time." - pass +def enable_cirun_for_project(owner, repo): + """Enable the cirun.io Github Application for a particular repository.""" + cirun = _get_cirun_client() + return cirun.set_repo(f"{owner}/{repo}", installation_id=123) -def enable_cirun_for_project(owner, project): - """Enable the cirun.io app for a particular repository.""" +def disable_cirun_for_project(owner, repo): + """Disable the cirun.io Github Application for a particular repository.""" cirun = _get_cirun_client() - return cirun.set_repo(f"{owner}/{project}", installation_id=123) + return cirun.set_repo(f"{owner}/{repo}", active=False) -def disable_cirun_for_project(owner, project): - """Disable the cirun.io app for a particular repository.""" +def enabled_cirun_resources(owner, repo): + """Which resources are currently enabled for this project""" cirun = _get_cirun_client() - return cirun.set_repo(f"{owner}/{project}", active=False) + return cirun.get_repo_resources("conda-forge", repo) -def add_project_to_cirun_resource(owner, project, resource): +def add_repo_to_cirun_resource(repo, resource): """Grant access to a cirun resource to a particular repository, with a particular policy.""" cirun = _get_cirun_client() - cirun.add_repo_to_resources("conda-forge", project, resource) + cirun.add_repo_to_resources("conda-forge", repo, resource) -def revoke_access_to_cirun_resource(owner, project, resource): +def remove_repo_from_cirun_resource(repo, resource): """Revoke access to a cirun resource to a particular repository, with a particular policy.""" cirun = _get_cirun_client() - cirun.remove_repo_from_resources("conda-forge", project, [resource]) - - -def enabled_cirun_resources(owner, project): - """Which resources are currently enabled for this project""" - cirun = _get_cirun_client() - return cirun.get_repo_resources("conda-forge", project) - - -def remove_project_from_cirun_resource(owner, repo, resource): - revoke_access_to_cirun_resource(owner, project=repo, resource=resource) - - -def remove_project_from_cirun(owner, repo): - pass + cirun.remove_repo_from_resources("conda-forge", repo, [resource]) def _get_cirun_client(): diff --git a/conda_smithy/cli.py b/conda_smithy/cli.py index 21a6022a5..ca1dc1565 100644 --- a/conda_smithy/cli.py +++ b/conda_smithy/cli.py @@ -23,6 +23,7 @@ from . import __version__ + if sys.version_info[0] == 2: raise Exception("Conda-smithy does not support python 2!") @@ -379,8 +380,8 @@ def __call__(self, args): to_remove = ["*"] for resource in to_remove: - conda_smithy.cirun_utils.remove_project_from_cirun_resource( - owner, repo, resource + conda_smithy.cirun_utils.remove_repo_from_cirun_resource( + repo, resource ) # current_resources = ci_register.enabled_cirun_resources(owner, repo) # if not current_resources: @@ -389,7 +390,7 @@ def __call__(self, args): # ci_register.ensure_cirun_app_installed(owner, repo) conda_smithy.cirun_utils.enable_cirun_for_project(owner, repo) for resource in args.cirun_resources: - conda_smithy.cirun_utils.add_project_to_cirun_resource(owner, repo, resource) + conda_smithy.cirun_utils.add_repo_to_cirun_resource(repo, resource) else: print("Cirun registration disabled.") diff --git a/conda_smithy/utils.py b/conda_smithy/utils.py index b37870c6a..574534270 100644 --- a/conda_smithy/utils.py +++ b/conda_smithy/utils.py @@ -5,7 +5,6 @@ import datetime import time import os -import sys from pathlib import Path from collections import defaultdict from contextlib import contextmanager From 156e27dfe888ad39957703ab626413b490681c82 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 21 Aug 2023 23:23:38 +0530 Subject: [PATCH 16/66] undo conda-build version change --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7ccf53c15..f4f24fb6e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ setuptools conda -conda-build==2.1.5 +conda-build >=3.4.2 jinja2 requests pycryptodome From 0a7bdd94ddb9775ce0fa7de59ebe3cb8f58b02de Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Tue, 22 Aug 2023 16:52:38 +0530 Subject: [PATCH 17/66] make precommit happy --- conda_smithy/cli.py | 23 +++++++++++-------- conda_smithy/configure_feedstock.py | 4 ++-- .../templates/github-actions.yml.tmpl | 2 +- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/conda_smithy/cli.py b/conda_smithy/cli.py index ca1dc1565..264668488 100644 --- a/conda_smithy/cli.py +++ b/conda_smithy/cli.py @@ -23,7 +23,6 @@ from . import __version__ - if sys.version_info[0] == 2: raise Exception("Conda-smithy does not support python 2!") @@ -293,14 +292,16 @@ def __call__(self, args): ) print("CI Summary for {}/{} (can take ~30s):".format(owner, repo)) - if args.remove and any([ - args.azure, - args.circle, - args.appveyor, - args.drone, - args.webservice, - args.anaconda_token, - ]): + if args.remove and any( + [ + args.azure, + args.circle, + args.appveyor, + args.drone, + args.webservice, + args.anaconda_token, + ] + ): raise RuntimeError( "The --remove flag is only supported for Cirun for now" ) @@ -390,7 +391,9 @@ def __call__(self, args): # ci_register.ensure_cirun_app_installed(owner, repo) conda_smithy.cirun_utils.enable_cirun_for_project(owner, repo) for resource in args.cirun_resources: - conda_smithy.cirun_utils.add_repo_to_cirun_resource(repo, resource) + conda_smithy.cirun_utils.add_repo_to_cirun_resource( + repo, resource + ) else: print("Cirun registration disabled.") diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 3ac5e7592..9050d3380 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -1846,7 +1846,7 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): "linux", "x64", "self-hosted", - "cirun-openstack-gpu" + "cirun-openstack-gpu", ], "cloud": "openstack", "instance_type": "gpu_large", @@ -1865,7 +1865,7 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): "instance_type": "ci_medium", "machine_image": "ubuntu-2204-cloud-jammy-20221104", "region": "RegionOne", - } + }, ], "recipe_dir": "recipe", "skip_render": [], diff --git a/conda_smithy/templates/github-actions.yml.tmpl b/conda_smithy/templates/github-actions.yml.tmpl index f45aef687..53997fbd7 100644 --- a/conda_smithy/templates/github-actions.yml.tmpl +++ b/conda_smithy/templates/github-actions.yml.tmpl @@ -54,7 +54,7 @@ concurrency: 'self_hosted_label_os': 'windows', 'self_hosted_label_arch': 'x64', }, -} +} %} jobs: From 7b0c3e3cecad0ddb6dc039a9c5975b8062471844 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Tue, 22 Aug 2023 17:01:40 +0530 Subject: [PATCH 18/66] add installation id --- conda_smithy/cirun_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index fc46c8592..617dc1eeb 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -1,10 +1,14 @@ from cirun import Cirun +CONDA_FORGE_CIRUN_INSTALLATION_ID = 18453316 + def enable_cirun_for_project(owner, repo): """Enable the cirun.io Github Application for a particular repository.""" cirun = _get_cirun_client() - return cirun.set_repo(f"{owner}/{repo}", installation_id=123) + return cirun.set_repo( + f"{owner}/{repo}", installation_id=CONDA_FORGE_CIRUN_INSTALLATION_ID + ) def disable_cirun_for_project(owner, repo): From fd37814a08bb078edc43423e36b7da60e1ed36fc Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Sat, 26 Aug 2023 10:59:42 +0530 Subject: [PATCH 19/66] add policy args --- conda_smithy/cirun_utils.py | 8 ++++++-- conda_smithy/cli.py | 10 +++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index 617dc1eeb..d93efe2c8 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -23,10 +23,14 @@ def enabled_cirun_resources(owner, repo): return cirun.get_repo_resources("conda-forge", repo) -def add_repo_to_cirun_resource(repo, resource): +def add_repo_to_cirun_resource(repo, resource, cirun_policy_args): """Grant access to a cirun resource to a particular repository, with a particular policy.""" cirun = _get_cirun_client() - cirun.add_repo_to_resources("conda-forge", repo, resource) + policy_args = {} + if cirun_policy_args: + if "pull_request" in cirun_policy_args: + policy_args.update({"pull_request": True}) + cirun.add_repo_to_resources("conda-forge", repo, resource, policy_args) def remove_repo_from_cirun_resource(repo, resource): diff --git a/conda_smithy/cli.py b/conda_smithy/cli.py index 264668488..a1236507b 100644 --- a/conda_smithy/cli.py +++ b/conda_smithy/cli.py @@ -265,6 +265,11 @@ def __init__(self, parser): action="append", help="cirun resources to enable for this repo. multiple values allowed", ) + scp.add_argument( + "--cirun-policy-args", + action="append", + help="extra arguments for cirun policy to create for this repo. multiple values allowed", + ) scp.add_argument( "--remove", action="store_true", @@ -372,6 +377,7 @@ def __call__(self, args): print("Drone registration disabled.") if args.cirun: + print("Cirun Registration") if args.remove: if args.cirun_resources: to_remove = args.cirun_resources @@ -380,6 +386,7 @@ def __call__(self, args): # This will remove everything to_remove = ["*"] + print(f"Cirun Registration: resources to remove: {to_remove}") for resource in to_remove: conda_smithy.cirun_utils.remove_repo_from_cirun_resource( repo, resource @@ -389,10 +396,11 @@ def __call__(self, args): # ci_register.remove_project_from_cirun(owner, repo) else: # ci_register.ensure_cirun_app_installed(owner, repo) + print(f"Cirun Registration: resources to add to: {owner}/{repo}") conda_smithy.cirun_utils.enable_cirun_for_project(owner, repo) for resource in args.cirun_resources: conda_smithy.cirun_utils.add_repo_to_cirun_resource( - repo, resource + repo, resource, args.cirun_policy_args ) else: print("Cirun registration disabled.") From e72cb954ce1169505310870e481a3d992a345186 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Sat, 26 Aug 2023 15:23:30 +0530 Subject: [PATCH 20/66] set installation id as environment variable --- conda_smithy/cirun_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index d93efe2c8..cd92c982e 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -1,13 +1,15 @@ +import os + from cirun import Cirun -CONDA_FORGE_CIRUN_INSTALLATION_ID = 18453316 +CIRUN_INSTALLATION_ID = os.environ.get("CIRUN_INSTALLATION_ID", 18453316) def enable_cirun_for_project(owner, repo): """Enable the cirun.io Github Application for a particular repository.""" cirun = _get_cirun_client() return cirun.set_repo( - f"{owner}/{repo}", installation_id=CONDA_FORGE_CIRUN_INSTALLATION_ID + f"{owner}/{repo}", installation_id=CIRUN_INSTALLATION_ID ) From 832f85d32597aec9a091c8754a7fecb3c31694a2 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Sat, 26 Aug 2023 16:29:43 +0530 Subject: [PATCH 21/66] move conda-forge to environ variable --- conda_smithy/cirun_utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index cd92c982e..5994f8e90 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -3,6 +3,7 @@ from cirun import Cirun CIRUN_INSTALLATION_ID = os.environ.get("CIRUN_INSTALLATION_ID", 18453316) +GH_ORG = os.environ.get("GH_ORG", "conda-forge") def enable_cirun_for_project(owner, repo): @@ -22,7 +23,7 @@ def disable_cirun_for_project(owner, repo): def enabled_cirun_resources(owner, repo): """Which resources are currently enabled for this project""" cirun = _get_cirun_client() - return cirun.get_repo_resources("conda-forge", repo) + return cirun.get_repo_resources(GH_ORG, repo) def add_repo_to_cirun_resource(repo, resource, cirun_policy_args): @@ -32,13 +33,13 @@ def add_repo_to_cirun_resource(repo, resource, cirun_policy_args): if cirun_policy_args: if "pull_request" in cirun_policy_args: policy_args.update({"pull_request": True}) - cirun.add_repo_to_resources("conda-forge", repo, resource, policy_args) + cirun.add_repo_to_resources(GH_ORG, repo, resource, policy_args) def remove_repo_from_cirun_resource(repo, resource): """Revoke access to a cirun resource to a particular repository, with a particular policy.""" cirun = _get_cirun_client() - cirun.remove_repo_from_resources("conda-forge", repo, [resource]) + cirun.remove_repo_from_resources(GH_ORG, repo, [resource]) def _get_cirun_client(): From d384958f94779c516c522bd784681764e7139cca Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Sat, 26 Aug 2023 16:55:18 +0530 Subject: [PATCH 22/66] use owner/org from cli instead of env var --- conda_smithy/cirun_utils.py | 11 +++++------ conda_smithy/cli.py | 4 ++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index 5994f8e90..a81f476f2 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -3,7 +3,6 @@ from cirun import Cirun CIRUN_INSTALLATION_ID = os.environ.get("CIRUN_INSTALLATION_ID", 18453316) -GH_ORG = os.environ.get("GH_ORG", "conda-forge") def enable_cirun_for_project(owner, repo): @@ -23,23 +22,23 @@ def disable_cirun_for_project(owner, repo): def enabled_cirun_resources(owner, repo): """Which resources are currently enabled for this project""" cirun = _get_cirun_client() - return cirun.get_repo_resources(GH_ORG, repo) + return cirun.get_repo_resources(owner, repo) -def add_repo_to_cirun_resource(repo, resource, cirun_policy_args): +def add_repo_to_cirun_resource(owner, repo, resource, cirun_policy_args): """Grant access to a cirun resource to a particular repository, with a particular policy.""" cirun = _get_cirun_client() policy_args = {} if cirun_policy_args: if "pull_request" in cirun_policy_args: policy_args.update({"pull_request": True}) - cirun.add_repo_to_resources(GH_ORG, repo, resource, policy_args) + cirun.add_repo_to_resources(owner, repo, resource, policy_args) -def remove_repo_from_cirun_resource(repo, resource): +def remove_repo_from_cirun_resource(owner, repo, resource): """Revoke access to a cirun resource to a particular repository, with a particular policy.""" cirun = _get_cirun_client() - cirun.remove_repo_from_resources(GH_ORG, repo, [resource]) + cirun.remove_repo_from_resources(owner, repo, [resource]) def _get_cirun_client(): diff --git a/conda_smithy/cli.py b/conda_smithy/cli.py index a1236507b..c6be256ad 100644 --- a/conda_smithy/cli.py +++ b/conda_smithy/cli.py @@ -389,7 +389,7 @@ def __call__(self, args): print(f"Cirun Registration: resources to remove: {to_remove}") for resource in to_remove: conda_smithy.cirun_utils.remove_repo_from_cirun_resource( - repo, resource + owner, repo, resource ) # current_resources = ci_register.enabled_cirun_resources(owner, repo) # if not current_resources: @@ -400,7 +400,7 @@ def __call__(self, args): conda_smithy.cirun_utils.enable_cirun_for_project(owner, repo) for resource in args.cirun_resources: conda_smithy.cirun_utils.add_repo_to_cirun_resource( - repo, resource, args.cirun_policy_args + owner, repo, resource, args.cirun_policy_args ) else: print("Cirun registration disabled.") From d0844345cf834a3f8e3d012302e097ca19faf825 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Sun, 27 Aug 2023 02:19:19 +0530 Subject: [PATCH 23/66] minor fixes --- conda_smithy/cirun_utils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index a81f476f2..aa8718cc3 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -7,6 +7,7 @@ def enable_cirun_for_project(owner, repo): """Enable the cirun.io Github Application for a particular repository.""" + print(f"Enabling cirun for {owner}/{repo} ...") cirun = _get_cirun_client() return cirun.set_repo( f"{owner}/{repo}", installation_id=CIRUN_INSTALLATION_ID @@ -28,11 +29,12 @@ def enabled_cirun_resources(owner, repo): def add_repo_to_cirun_resource(owner, repo, resource, cirun_policy_args): """Grant access to a cirun resource to a particular repository, with a particular policy.""" cirun = _get_cirun_client() - policy_args = {} + policy_args = {"pull_request": False} if cirun_policy_args: if "pull_request" in cirun_policy_args: - policy_args.update({"pull_request": True}) - cirun.add_repo_to_resources(owner, repo, resource, policy_args) + policy_args["pull_request"] = True + + cirun.add_repo_to_resources(owner, repo, resources=[resource], teams=[repo], policy_args=policy_args) def remove_repo_from_cirun_resource(owner, repo, resource): From 6355aa803349bde0025ea1f295e24aba7c685e01 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Sun, 27 Aug 2023 13:08:13 +0530 Subject: [PATCH 24/66] assert installation id --- conda_smithy/cirun_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index aa8718cc3..c54ef6037 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -9,6 +9,7 @@ def enable_cirun_for_project(owner, repo): """Enable the cirun.io Github Application for a particular repository.""" print(f"Enabling cirun for {owner}/{repo} ...") cirun = _get_cirun_client() + assert CIRUN_INSTALLATION_ID return cirun.set_repo( f"{owner}/{repo}", installation_id=CIRUN_INSTALLATION_ID ) From 37d0252b0aa4ef96133ff52c7093466560666faa Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Sun, 27 Aug 2023 13:10:16 +0530 Subject: [PATCH 25/66] add some more logging --- conda_smithy/cirun_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index c54ef6037..3c756dcb2 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -34,13 +34,14 @@ def add_repo_to_cirun_resource(owner, repo, resource, cirun_policy_args): if cirun_policy_args: if "pull_request" in cirun_policy_args: policy_args["pull_request"] = True - + print(f"Adding repo {owner}/{repo} to resource {resource} with policy_args: {policy_args}") cirun.add_repo_to_resources(owner, repo, resources=[resource], teams=[repo], policy_args=policy_args) def remove_repo_from_cirun_resource(owner, repo, resource): """Revoke access to a cirun resource to a particular repository, with a particular policy.""" cirun = _get_cirun_client() + print(f"Removing repo {owner}/{repo} from resource {resource}.") cirun.remove_repo_from_resources(owner, repo, [resource]) From 94aac9e8dbfebf2ec74925c0520b21e9eea9af37 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Sun, 27 Aug 2023 14:05:04 +0530 Subject: [PATCH 26/66] log cirun response --- conda_smithy/cirun_utils.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index 3c756dcb2..50dc37f76 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -35,14 +35,20 @@ def add_repo_to_cirun_resource(owner, repo, resource, cirun_policy_args): if "pull_request" in cirun_policy_args: policy_args["pull_request"] = True print(f"Adding repo {owner}/{repo} to resource {resource} with policy_args: {policy_args}") - cirun.add_repo_to_resources(owner, repo, resources=[resource], teams=[repo], policy_args=policy_args) + response = cirun.add_repo_to_resources( + owner, repo, resources=[resource], teams=[repo], policy_args=policy_args + ) + print(f"response: {response} | {response.json().keys()}") + return response def remove_repo_from_cirun_resource(owner, repo, resource): """Revoke access to a cirun resource to a particular repository, with a particular policy.""" cirun = _get_cirun_client() print(f"Removing repo {owner}/{repo} from resource {resource}.") - cirun.remove_repo_from_resources(owner, repo, [resource]) + response = cirun.remove_repo_from_resources(owner, repo, [resource]) + print(f"response: {response} | {response.json().keys()}") + return response def _get_cirun_client(): From c2db57e663620addc3900b426d29902ad7c8dfdf Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Sun, 27 Aug 2023 20:47:33 +0530 Subject: [PATCH 27/66] remove commented out code and cirun file json --- conda_smithy/cirun_utils.py | 12 ------------ conda_smithy/cli.py | 6 ------ conda_smithy/configure_feedstock.py | 28 ---------------------------- 3 files changed, 46 deletions(-) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index 50dc37f76..3a998e4d3 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -15,18 +15,6 @@ def enable_cirun_for_project(owner, repo): ) -def disable_cirun_for_project(owner, repo): - """Disable the cirun.io Github Application for a particular repository.""" - cirun = _get_cirun_client() - return cirun.set_repo(f"{owner}/{repo}", active=False) - - -def enabled_cirun_resources(owner, repo): - """Which resources are currently enabled for this project""" - cirun = _get_cirun_client() - return cirun.get_repo_resources(owner, repo) - - def add_repo_to_cirun_resource(owner, repo, resource, cirun_policy_args): """Grant access to a cirun resource to a particular repository, with a particular policy.""" cirun = _get_cirun_client() diff --git a/conda_smithy/cli.py b/conda_smithy/cli.py index c6be256ad..54aaa3e00 100644 --- a/conda_smithy/cli.py +++ b/conda_smithy/cli.py @@ -382,8 +382,6 @@ def __call__(self, args): if args.cirun_resources: to_remove = args.cirun_resources else: - # to_remove = ci_register.enabled_cirun_resources(owner, repo) - # This will remove everything to_remove = ["*"] print(f"Cirun Registration: resources to remove: {to_remove}") @@ -391,11 +389,7 @@ def __call__(self, args): conda_smithy.cirun_utils.remove_repo_from_cirun_resource( owner, repo, resource ) - # current_resources = ci_register.enabled_cirun_resources(owner, repo) - # if not current_resources: - # ci_register.remove_project_from_cirun(owner, repo) else: - # ci_register.ensure_cirun_app_installed(owner, repo) print(f"Cirun Registration: resources to add to: {owner}/{repo}") conda_smithy.cirun_utils.enable_cirun_for_project(owner, repo) for resource in args.cirun_resources: diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 9050d3380..342074247 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -1839,34 +1839,6 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): "store_build_artifacts": False, "artifact_retention_days": 14, }, - "cirun_runners": [ - { - "name": "cirun-openstack-gpu", - "labels": [ - "linux", - "x64", - "self-hosted", - "cirun-openstack-gpu", - ], - "cloud": "openstack", - "instance_type": "gpu_large", - "machine_image": "ubuntu-2204-nvidia-container-runtime-20230107", - "region": "RegionOne", - }, - { - "name": "cirun-openstack-cpu", - "labels": [ - "linux", - "x64", - "self-hosted", - "cirun-openstack-cpu", - ], - "cloud": "openstack", - "instance_type": "ci_medium", - "machine_image": "ubuntu-2204-cloud-jammy-20221104", - "region": "RegionOne", - }, - ], "recipe_dir": "recipe", "skip_render": [], "bot": {"automerge": False}, From 63c566aee007cd4e711a91895c10f5631aea0ced Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Sun, 27 Aug 2023 20:50:05 +0530 Subject: [PATCH 28/66] remove .cirun.yml template reference --- conda_smithy/configure_feedstock.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 342074247..b07cb0166 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -1286,12 +1286,6 @@ def _github_actions_specific_setup( if forge_config["github_actions"]["store_build_artifacts"]: template_files.append(".scripts/create_conda_build_artifacts.sh") - if forge_config["github_actions"]["self_hosted"]: - for label in forge_config["github_actions"]["self_hosted_labels"]: - if label.startswith("cirun-"): - template_files.append(".cirun.yml") - break - _render_template_exe_files( forge_config=forge_config, jinja_env=jinja_env, From ce81aa2a32846587aa9e62749f85721cfc7a131d Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 28 Aug 2023 11:18:16 +0530 Subject: [PATCH 29/66] pre-commit fixes --- conda_smithy/cirun_utils.py | 10 ++++++++-- conda_smithy/cli.py | 4 +++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index 3a998e4d3..f4bef659d 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -22,9 +22,15 @@ def add_repo_to_cirun_resource(owner, repo, resource, cirun_policy_args): if cirun_policy_args: if "pull_request" in cirun_policy_args: policy_args["pull_request"] = True - print(f"Adding repo {owner}/{repo} to resource {resource} with policy_args: {policy_args}") + print( + f"Adding repo {owner}/{repo} to resource {resource} with policy_args: {policy_args}" + ) response = cirun.add_repo_to_resources( - owner, repo, resources=[resource], teams=[repo], policy_args=policy_args + owner, + repo, + resources=[resource], + teams=[repo], + policy_args=policy_args, ) print(f"response: {response} | {response.json().keys()}") return response diff --git a/conda_smithy/cli.py b/conda_smithy/cli.py index 54aaa3e00..92b8466fa 100644 --- a/conda_smithy/cli.py +++ b/conda_smithy/cli.py @@ -390,7 +390,9 @@ def __call__(self, args): owner, repo, resource ) else: - print(f"Cirun Registration: resources to add to: {owner}/{repo}") + print( + f"Cirun Registration: resources to add to: {owner}/{repo}" + ) conda_smithy.cirun_utils.enable_cirun_for_project(owner, repo) for resource in args.cirun_resources: conda_smithy.cirun_utils.add_repo_to_cirun_resource( From 829201e70e85f1c0741c291f89ce15c77ef02fe1 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 28 Aug 2023 14:27:41 +0530 Subject: [PATCH 30/66] undo changes to github actions template --- ...b-actions.yml.tmpl => github-actions.tmpl} | 95 ++++++------------- 1 file changed, 29 insertions(+), 66 deletions(-) rename conda_smithy/templates/{github-actions.yml.tmpl => github-actions.tmpl} (80%) diff --git a/conda_smithy/templates/github-actions.yml.tmpl b/conda_smithy/templates/github-actions.tmpl similarity index 80% rename from conda_smithy/templates/github-actions.yml.tmpl rename to conda_smithy/templates/github-actions.tmpl index 53997fbd7..f413796d8 100644 --- a/conda_smithy/templates/github-actions.yml.tmpl +++ b/conda_smithy/templates/github-actions.tmpl @@ -12,51 +12,11 @@ name: Build conda package {%- if github_actions.self_hosted %} -on: {{ github_actions.self_hosted_triggers }} +on: [push] {%- else %} on: [push, pull_request] {%- endif %} -{%- if github_actions.cancel_in_progress or github_actions.self_hosted %} -concurrency: - group: {% raw %}${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}{% endraw %} - cancel-in-progress: true -{%- endif %} - -{%- set platform_vars = { - 'osx-64': { - 'os': 'macos', - 'self_hosted_label_os': 'macOS', - 'self_hosted_label_arch': 'x64', - }, - 'osx-arm64': { - 'os': 'macos', - 'self_hosted_label_os': 'macOS', - 'self_hosted_label_arch': 'arm64', - }, - 'linux-64': { - 'os': 'ubuntu', - 'self_hosted_label_os': 'linux', - 'self_hosted_label_arch': 'x64', - }, - 'linux-aarch64': { - 'os': 'ubuntu', - 'self_hosted_label_os': 'linux', - 'self_hosted_label_arch': '', - }, - 'linux-ppcle64': { - 'os': 'ubuntu', - 'self_hosted_label_os': 'linux', - 'self_hosted_label_arch': '', - }, - 'win-64': { - 'os': 'windows', - 'self_hosted_label_os': 'windows', - 'self_hosted_label_arch': 'x64', - }, -} -%} - jobs: build: name: {% raw %}${{ matrix.CONFIG }}{% endraw %} @@ -78,31 +38,36 @@ jobs: SHORT_CONFIG: {{ data.short_config_name }} {%- endif %} UPLOAD_PACKAGES: {{ data.upload }} - os: {{ platform_vars[data.build_platform]['os'] }} - {#- namespace needed to handle jinja scoping rules #} - {%- set ns = namespace(gpu=False) %} + {%- if data.build_platform.startswith("osx-64") %} + os: macos + {%- if github_actions.self_hosted %} + labels: ['macOS', 'self-hosted', 'x64'] + {%- endif %} + {%- elif data.build_platform.startswith("osx-arm64") %} + os: macos + {%- if github_actions.self_hosted %} + labels: ['macOS', 'self-hosted', 'ARM64'] + {%- endif %} + {%- elif data.build_platform.startswith("linux-64") %} + DOCKER_IMAGE: {{ data.config["docker_image"][-1] }} + os: ubuntu {%- if github_actions.self_hosted %} - labels: - - self-hosted - - {{ platform_vars[data.build_platform]['self_hosted_label_os'] }} - {%- if platform_vars[data.build_platform]['self_hosted_label_arch'] %} - - {{ platform_vars[data.build_platform]['self_hosted_label_arch'] }} - {%- endif %} - {%- for label in github_actions.self_hosted_labels %} - {%- if 'gpu' in label %} - {%- set ns.gpu = True %} - {%- endif %} - {%- if label.startswith('cirun-') %} - {%- set label = label ~ "--${{ github.run_id }}-" ~ data.short_config_name %} - {%- endif %} - - {{ label }} - {%- endfor %} + labels: ['linux', 'self-hosted', 'x64'] {%- endif %} - {%- if data.build_platform.startswith("linux") %} + {%- elif data.build_platform.startswith("linux-aarch64") %} DOCKER_IMAGE: {{ data.config["docker_image"][-1] }} - {%- if ns.gpu %} - CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" - {%- endif %} + os: ubuntu + {%- if github_actions.self_hosted %} + labels: ['linux', 'self-hosted', 'ARM64'] + {%- endif %} + {%- elif data.build_platform.startswith("linux") %} + DOCKER_IMAGE: {{ data.config["docker_image"][-1] }} + os: ubuntu + {%- elif data.build_platform.startswith("win") %} + os: windows + {%- if github_actions.self_hosted %} + labels: ['windows', 'self-hosted', 'x64'] + {%- endif %} {%- endif %} {%- endfor %} steps: @@ -151,9 +116,7 @@ jobs: UPLOAD_ON_BRANCH: {{ upload_on_branch }} {%- endif %} {%- if docker.run_args is defined %} - CONDA_FORGE_DOCKER_RUN_ARGS: "{{ docker.run_args }} {% raw %}${{ matrix.CONDA_FORGE_DOCKER_RUN_ARGS }}{% endraw %}" -{%- else %} - CONDA_FORGE_DOCKER_RUN_ARGS: "{% raw %}${{ matrix.CONDA_FORGE_DOCKER_RUN_ARGS }}{% endraw %}" + CONDA_FORGE_DOCKER_RUN_ARGS: "{{ docker.run_args }}" {%- endif %} {%- for secret in secrets %} {{ secret }}: {% raw %}${{{% endraw %} secrets.{{ secret }} {% raw %}}}{% endraw %} From c68e3b049a5732c20c674a7db5c289c6e9e35a93 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 28 Aug 2023 14:28:26 +0530 Subject: [PATCH 31/66] undo changes to configure feedstock --- conda_smithy/configure_feedstock.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index b07cb0166..7b4830000 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -1279,12 +1279,10 @@ def _github_actions_specific_setup( ], "win": [], } - - template_files = platform_templates.get(platform, []) - - # Templates for all platforms if forge_config["github_actions"]["store_build_artifacts"]: - template_files.append(".scripts/create_conda_build_artifacts.sh") + for tmpls in platform_templates.values(): + tmpls.append(".scripts/create_conda_build_artifacts.sh") + template_files = platform_templates.get(platform, []) _render_template_exe_files( forge_config=forge_config, @@ -1300,7 +1298,7 @@ def render_github_actions( target_path = os.path.join( forge_dir, ".github", "workflows", "conda-build.yml" ) - template_filename = "github-actions.yml.tmpl" + template_filename = "github-actions.tmpl" fast_finish_text = "" ( @@ -1310,7 +1308,7 @@ def render_github_actions( upload_packages, ) = _get_platforms_of_provider("github_actions", forge_config) - logger.debug("github platforms retrieved") + logger.debug("github platforms retreived") remove_file_or_dir(target_path) return _render_ci_provider( @@ -1824,9 +1822,6 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): }, "github_actions": { "self_hosted": False, - "self_hosted_labels": [], - "self_hosted_triggers": ["push"], - "cancel_in_progress": False, # Set maximum parallel jobs "max_parallel": None, # Toggle creating artifacts for conda build_artifacts dir From 03b5cea3a754111d5e1e8b32dc77a562cbc0cdcc Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 28 Aug 2023 14:29:47 +0530 Subject: [PATCH 32/66] remove stub travis func --- conda_smithy/ci_register.py | 5 ----- 1 file changed, 5 deletions(-) mode change 100755 => 100644 conda_smithy/ci_register.py diff --git a/conda_smithy/ci_register.py b/conda_smithy/ci_register.py old mode 100755 new mode 100644 index db0bd4c64..b6a2117ed --- a/conda_smithy/ci_register.py +++ b/conda_smithy/ci_register.py @@ -423,11 +423,6 @@ def add_project_to_travis(user, project): print(" * {}/{} registered on travis-ci".format(user, project)) -def remove_project_from_travis(user, repo): - "TODO: Revoke access to the repo for travis." - pass - - def travis_token_update_conda_forge_config( feedstock_config_path, user, project ): From d4a594cb68300ca90647ed6712dfbd00712a17c3 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 28 Aug 2023 14:31:23 +0530 Subject: [PATCH 33/66] undo mode change --- conda_smithy/ci_register.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 conda_smithy/ci_register.py diff --git a/conda_smithy/ci_register.py b/conda_smithy/ci_register.py old mode 100644 new mode 100755 From dee38433b8106f0b1102d9cf16085d19472911f3 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 8 Sep 2023 12:16:47 +0530 Subject: [PATCH 34/66] Undo removal of github actions template --- conda_smithy/configure_feedstock.py | 64 ++++++--- ...b-actions.tmpl => github-actions.yml.tmpl} | 124 ++++++++++-------- 2 files changed, 111 insertions(+), 77 deletions(-) rename conda_smithy/templates/{github-actions.tmpl => github-actions.yml.tmpl} (78%) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 7b4830000..d029da996 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -51,20 +51,6 @@ conda_forge_content = os.path.abspath(os.path.dirname(__file__)) logger = logging.getLogger(__name__) -# feedstocks listed here are allowed to use GHA on -# conda-forge -# this should solve issues where other CI proviers have too many -# jobs and we need to change something via CI -SERVICE_FEEDSTOCKS = [ - "conda-forge-pinning-feedstock", - "conda-forge-repodata-patches-feedstock", - "conda-smithy-feedstock", -] -if "CONDA_SMITHY_SERVICE_FEEDSTOCKS" in os.environ: - SERVICE_FEEDSTOCKS += os.environ["CONDA_SMITHY_SERVICE_FEEDSTOCKS"].split( - "," - ) - def package_key(config, used_loop_vars, subdir): # get the build string from whatever conda-build makes of the configuration @@ -687,7 +673,6 @@ def _render_ci_provider( channel_target.startswith("conda-forge ") and provider_name == "github_actions" and not forge_config["github_actions"]["self_hosted"] - and os.path.basename(forge_dir) not in SERVICE_FEEDSTOCKS ): raise RuntimeError( "Using github_actions as the CI provider inside " @@ -1279,11 +1264,19 @@ def _github_actions_specific_setup( ], "win": [], } - if forge_config["github_actions"]["store_build_artifacts"]: - for tmpls in platform_templates.values(): - tmpls.append(".scripts/create_conda_build_artifacts.sh") + template_files = platform_templates.get(platform, []) + # Templates for all platforms + if forge_config["github_actions"]["store_build_artifacts"]: + template_files.append(".scripts/create_conda_build_artifacts.sh") + + if forge_config["github_actions"]["self_hosted"]: + for label in forge_config["github_actions"]["self_hosted_labels"]: + if label.startswith("cirun-"): + template_files.append(".cirun.yml") + break + _render_template_exe_files( forge_config=forge_config, jinja_env=jinja_env, @@ -1298,7 +1291,7 @@ def render_github_actions( target_path = os.path.join( forge_dir, ".github", "workflows", "conda-build.yml" ) - template_filename = "github-actions.tmpl" + template_filename = "github-actions.yml.tmpl" fast_finish_text = "" ( @@ -1308,7 +1301,7 @@ def render_github_actions( upload_packages, ) = _get_platforms_of_provider("github_actions", forge_config) - logger.debug("github platforms retreived") + logger.debug("github platforms retrieved") remove_file_or_dir(target_path) return _render_ci_provider( @@ -1822,12 +1815,43 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): }, "github_actions": { "self_hosted": False, + "self_hosted_labels": [], + "self_hosted_triggers": ["push"], + "cancel_in_progress": False, # Set maximum parallel jobs "max_parallel": None, # Toggle creating artifacts for conda build_artifacts dir "store_build_artifacts": False, "artifact_retention_days": 14, }, + "cirun_runners": [ + { + "name": "cirun-openstack-gpu", + "labels": [ + "linux", + "x64", + "self-hosted", + "cirun-openstack-gpu" + ], + "cloud": "openstack", + "instance_type": "gpu_large", + "machine_image": "ubuntu-2204-nvidia-container-runtime-20230107", + "region": "RegionOne", + }, + { + "name": "cirun-openstack-cpu", + "labels": [ + "linux", + "x64", + "self-hosted", + "cirun-openstack-cpu", + ], + "cloud": "openstack", + "instance_type": "ci_medium", + "machine_image": "ubuntu-2204-cloud-jammy-20221104", + "region": "RegionOne", + } + ], "recipe_dir": "recipe", "skip_render": [], "bot": {"automerge": False}, diff --git a/conda_smithy/templates/github-actions.tmpl b/conda_smithy/templates/github-actions.yml.tmpl similarity index 78% rename from conda_smithy/templates/github-actions.tmpl rename to conda_smithy/templates/github-actions.yml.tmpl index f413796d8..a8bd3561b 100644 --- a/conda_smithy/templates/github-actions.tmpl +++ b/conda_smithy/templates/github-actions.yml.tmpl @@ -12,11 +12,51 @@ name: Build conda package {%- if github_actions.self_hosted %} -on: [push] +on: {{ github_actions.self_hosted_triggers }} {%- else %} on: [push, pull_request] {%- endif %} +{%- if github_actions.cancel_in_progress or github_actions.self_hosted %} +concurrency: + group: {% raw %}${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}{% endraw %} + cancel-in-progress: true +{%- endif %} + +{%- set platform_vars = { + 'osx-64': { + 'os': 'macos', + 'self_hosted_label_os': 'macOS', + 'self_hosted_label_arch': 'x64', + }, + 'osx-arm64': { + 'os': 'macos', + 'self_hosted_label_os': 'macOS', + 'self_hosted_label_arch': 'arm64', + }, + 'linux-64': { + 'os': 'ubuntu', + 'self_hosted_label_os': 'linux', + 'self_hosted_label_arch': 'x64', + }, + 'linux-aarch64': { + 'os': 'ubuntu', + 'self_hosted_label_os': 'linux', + 'self_hosted_label_arch': '', + }, + 'linux-ppcle64': { + 'os': 'ubuntu', + 'self_hosted_label_os': 'linux', + 'self_hosted_label_arch': '', + }, + 'win-64': { + 'os': 'windows', + 'self_hosted_label_os': 'windows', + 'self_hosted_label_arch': 'x64', + }, +} +%} + jobs: build: name: {% raw %}${{ matrix.CONFIG }}{% endraw %} @@ -38,66 +78,34 @@ jobs: SHORT_CONFIG: {{ data.short_config_name }} {%- endif %} UPLOAD_PACKAGES: {{ data.upload }} - {%- if data.build_platform.startswith("osx-64") %} - os: macos - {%- if github_actions.self_hosted %} - labels: ['macOS', 'self-hosted', 'x64'] - {%- endif %} - {%- elif data.build_platform.startswith("osx-arm64") %} - os: macos + os: {{ platform_vars[data.build_platform]['os'] }} + {#- namespace needed to handle jinja scoping rules #} + {%- set ns = namespace(gpu=False) %} {%- if github_actions.self_hosted %} - labels: ['macOS', 'self-hosted', 'ARM64'] + labels: + - self-hosted + - {{ platform_vars[data.build_platform]['self_hosted_label_os'] }} + {%- if platform_vars[data.build_platform]['self_hosted_label_arch'] %} + - {{ platform_vars[data.build_platform]['self_hosted_label_arch'] }} + {%- endif %} + {%- for label in github_actions.self_hosted_labels %} + {%- if 'gpu' in label %} + {%- set ns.gpu = True %} + {%- endif %} + {%- if label.startswith('cirun-') %} + {%- set label = label ~ "--${{ github.run_id }}-" ~ data.short_config_name %} + {%- endif %} + - {{ label }} + {%- endfor %} {%- endif %} - {%- elif data.build_platform.startswith("linux-64") %} + {%- if data.build_platform.startswith("linux") %} DOCKER_IMAGE: {{ data.config["docker_image"][-1] }} - os: ubuntu - {%- if github_actions.self_hosted %} - labels: ['linux', 'self-hosted', 'x64'] - {%- endif %} - {%- elif data.build_platform.startswith("linux-aarch64") %} - DOCKER_IMAGE: {{ data.config["docker_image"][-1] }} - os: ubuntu - {%- if github_actions.self_hosted %} - labels: ['linux', 'self-hosted', 'ARM64'] - {%- endif %} - {%- elif data.build_platform.startswith("linux") %} - DOCKER_IMAGE: {{ data.config["docker_image"][-1] }} - os: ubuntu - {%- elif data.build_platform.startswith("win") %} - os: windows - {%- if github_actions.self_hosted %} - labels: ['windows', 'self-hosted', 'x64'] - {%- endif %} + {%- if ns.gpu %} + CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" + {%- endif %} {%- endif %} {%- endfor %} steps: -{%- if github_actions.free_disk_space %} - - - name: Manage disk space - if: matrix.os == 'ubuntu' - run: | - sudo mkdir -p /opt/empty_dir || true - for d in \ - /opt/ghc \ - /opt/hostedtoolcache \ - /usr/lib/jvm \ - /usr/local/.ghcup \ - /usr/local/lib/android \ - /usr/local/share/powershell \ - /usr/share/dotnet \ - /usr/share/swift \ - ; do - sudo rsync --stats -a --delete /opt/empty_dir/ $d || true - done - sudo apt-get purge -y -f firefox \ - google-chrome-stable \ - microsoft-edge-stable - sudo apt-get autoremove -y >& /dev/null - sudo apt-get autoclean -y >& /dev/null - sudo docker image prune --all --force - df -h -{%- endif %} - - name: Checkout code uses: actions/checkout@v3 {%- if clone_depth is not none %} @@ -116,7 +124,9 @@ jobs: UPLOAD_ON_BRANCH: {{ upload_on_branch }} {%- endif %} {%- if docker.run_args is defined %} - CONDA_FORGE_DOCKER_RUN_ARGS: "{{ docker.run_args }}" + CONDA_FORGE_DOCKER_RUN_ARGS: "{{ docker.run_args }} {% raw %}${{ matrix.CONDA_FORGE_DOCKER_RUN_ARGS }}{% endraw %}" +{%- else %} + CONDA_FORGE_DOCKER_RUN_ARGS: "{% raw %}${{ matrix.CONDA_FORGE_DOCKER_RUN_ARGS }}{% endraw %}" {%- endif %} {%- for secret in secrets %} {{ secret }}: {% raw %}${{{% endraw %} secrets.{{ secret }} {% raw %}}}{% endraw %} @@ -191,7 +201,7 @@ jobs: {%- endif %} setup_conda_rc .\ ".\{{ recipe_dir }}" .\.ci_support\%CONFIG%.yaml if errorlevel 1 exit 1 - {%- if build_setup %} + {% if build_setup -%} {{ build_setup.replace("\n", "\n ").rstrip() }} {%- endif %} From 655aec28725ae8313dee740045b58823bfb89ead Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Fri, 8 Sep 2023 12:28:09 +0530 Subject: [PATCH 35/66] pre-commit fixes --- conda_smithy/configure_feedstock.py | 4 ++-- conda_smithy/templates/github-actions.yml.tmpl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 10b103784..0a1f44769 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -1833,7 +1833,7 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): "linux", "x64", "self-hosted", - "cirun-openstack-gpu" + "cirun-openstack-gpu", ], "cloud": "openstack", "instance_type": "gpu_large", @@ -1852,7 +1852,7 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): "instance_type": "ci_medium", "machine_image": "ubuntu-2204-cloud-jammy-20221104", "region": "RegionOne", - } + }, ], "recipe_dir": "recipe", "skip_render": [], diff --git a/conda_smithy/templates/github-actions.yml.tmpl b/conda_smithy/templates/github-actions.yml.tmpl index fc4dac727..05877f112 100644 --- a/conda_smithy/templates/github-actions.yml.tmpl +++ b/conda_smithy/templates/github-actions.yml.tmpl @@ -54,7 +54,7 @@ concurrency: 'self_hosted_label_os': 'windows', 'self_hosted_label_arch': 'x64', }, -} +} %} jobs: From cf9e9bfc06f0bb8c39a6fa2f8ce49a23f118da62 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Fri, 8 Sep 2023 12:29:02 +0530 Subject: [PATCH 36/66] not available for travis --- conda_smithy/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_smithy/cli.py b/conda_smithy/cli.py index 92b8466fa..e0475a12b 100644 --- a/conda_smithy/cli.py +++ b/conda_smithy/cli.py @@ -274,7 +274,7 @@ def __init__(self, parser): "--remove", action="store_true", help="Revoke access to the configured CI services. " - "Only available for Travis and Cirun for now", + "Only available for Cirun for now", ) def __call__(self, args): From 589fb8ad955ea793c0a4663a1a175a8102d8e1a6 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Fri, 8 Sep 2023 12:36:39 +0530 Subject: [PATCH 37/66] add a note about getting installation id --- conda_smithy/cirun_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index f4bef659d..d569f3d68 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -2,6 +2,10 @@ from cirun import Cirun +# To get this id, got to https://github.com/organizations//settings/installations +# and then click on the Configure button next to Cirun Application, then copy +# the installation id from the URL, it would look something like: +# https://github.com/organizations/conda-forge/settings/installations/18453316 CIRUN_INSTALLATION_ID = os.environ.get("CIRUN_INSTALLATION_ID", 18453316) From 117fc1ea935ab08502be57680abe39076a64bc88 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Fri, 8 Sep 2023 12:40:39 +0530 Subject: [PATCH 38/66] cache cirun client --- conda_smithy/cirun_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index d569f3d68..cef8f5416 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -1,4 +1,5 @@ import os +from functools import cache from cirun import Cirun @@ -49,6 +50,7 @@ def remove_repo_from_cirun_resource(owner, repo, resource): return response +@cache def _get_cirun_client(): try: return Cirun() From c02cbdd16923c521e0c56270cafa99810733a65c Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Fri, 8 Sep 2023 13:20:28 +0530 Subject: [PATCH 39/66] add type hints, docs --- conda_smithy/cirun_utils.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index cef8f5416..9804ce8a5 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -1,5 +1,6 @@ import os -from functools import cache +from functools import lru_cache +from typing import List, Dict, Any from cirun import Cirun @@ -10,7 +11,7 @@ CIRUN_INSTALLATION_ID = os.environ.get("CIRUN_INSTALLATION_ID", 18453316) -def enable_cirun_for_project(owner, repo): +def enable_cirun_for_project(owner: str, repo: str) -> Dict[str, Any]: """Enable the cirun.io Github Application for a particular repository.""" print(f"Enabling cirun for {owner}/{repo} ...") cirun = _get_cirun_client() @@ -20,10 +21,12 @@ def enable_cirun_for_project(owner, repo): ) -def add_repo_to_cirun_resource(owner, repo, resource, cirun_policy_args): +def add_repo_to_cirun_resource( + owner: str, repo: str, resource: str, cirun_policy_args: List[str] +) -> Dict[str, Any]: """Grant access to a cirun resource to a particular repository, with a particular policy.""" cirun = _get_cirun_client() - policy_args = {"pull_request": False} + policy_args: Dict[str, Any] = {"pull_request": False} if cirun_policy_args: if "pull_request" in cirun_policy_args: policy_args["pull_request"] = True @@ -41,7 +44,7 @@ def add_repo_to_cirun_resource(owner, repo, resource, cirun_policy_args): return response -def remove_repo_from_cirun_resource(owner, repo, resource): +def remove_repo_from_cirun_resource(owner: str, repo: str, resource: str): """Revoke access to a cirun resource to a particular repository, with a particular policy.""" cirun = _get_cirun_client() print(f"Removing repo {owner}/{repo} from resource {resource}.") @@ -50,8 +53,8 @@ def remove_repo_from_cirun_resource(owner, repo, resource): return response -@cache -def _get_cirun_client(): +@lru_cache +def _get_cirun_client() -> Cirun: try: return Cirun() except KeyError: From e7257d7bbb8c27c8c731dabac24a5d9123f1668e Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Fri, 8 Sep 2023 13:36:21 +0530 Subject: [PATCH 40/66] set policy args to None by default --- conda_smithy/cirun_utils.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index 9804ce8a5..6cb7894f9 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -1,6 +1,6 @@ import os from functools import lru_cache -from typing import List, Dict, Any +from typing import List, Dict, Any, Optional from cirun import Cirun @@ -26,10 +26,9 @@ def add_repo_to_cirun_resource( ) -> Dict[str, Any]: """Grant access to a cirun resource to a particular repository, with a particular policy.""" cirun = _get_cirun_client() - policy_args: Dict[str, Any] = {"pull_request": False} - if cirun_policy_args: - if "pull_request" in cirun_policy_args: - policy_args["pull_request"] = True + policy_args: Optional[Dict[str, Any]] = None + if cirun_policy_args and "pull_request" in cirun_policy_args: + policy_args = {"pull_request": True} print( f"Adding repo {owner}/{repo} to resource {resource} with policy_args: {policy_args}" ) From 3876bbf92f364b50fa2aa2418086216b04a941ac Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Fri, 8 Sep 2023 13:46:06 +0530 Subject: [PATCH 41/66] set default value for cirun_resources to iterable --- conda_smithy/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conda_smithy/cli.py b/conda_smithy/cli.py index e0475a12b..adee63223 100644 --- a/conda_smithy/cli.py +++ b/conda_smithy/cli.py @@ -262,6 +262,7 @@ def __init__(self, parser): ) scp.add_argument( "--cirun-resources", + default=[], action="append", help="cirun resources to enable for this repo. multiple values allowed", ) From 85390fa5ca221f45bb9357bb573f59fb6b118b93 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Fri, 8 Sep 2023 13:47:12 +0530 Subject: [PATCH 42/66] make polcy args as optional --- conda_smithy/cirun_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index 6cb7894f9..b24aaeca0 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -22,7 +22,7 @@ def enable_cirun_for_project(owner: str, repo: str) -> Dict[str, Any]: def add_repo_to_cirun_resource( - owner: str, repo: str, resource: str, cirun_policy_args: List[str] + owner: str, repo: str, resource: str, cirun_policy_args: Optional[List[str]] = None ) -> Dict[str, Any]: """Grant access to a cirun resource to a particular repository, with a particular policy.""" cirun = _get_cirun_client() From 2204933655eb850ffae8828009698ed6de879986 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Fri, 8 Sep 2023 14:02:32 +0530 Subject: [PATCH 43/66] precommit fixes --- conda_smithy/cirun_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index b24aaeca0..de8e291a9 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -22,7 +22,10 @@ def enable_cirun_for_project(owner: str, repo: str) -> Dict[str, Any]: def add_repo_to_cirun_resource( - owner: str, repo: str, resource: str, cirun_policy_args: Optional[List[str]] = None + owner: str, + repo: str, + resource: str, + cirun_policy_args: Optional[List[str]] = None, ) -> Dict[str, Any]: """Grant access to a cirun resource to a particular repository, with a particular policy.""" cirun = _get_cirun_client() From 357e5949cca875744374f9e81103d7bef0dbb44c Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Fri, 8 Sep 2023 14:03:08 +0530 Subject: [PATCH 44/66] Update conda_smithy/cirun_utils.py Co-authored-by: jaimergp --- conda_smithy/cirun_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index de8e291a9..cfd92c77f 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -61,6 +61,6 @@ def _get_cirun_client() -> Cirun: return Cirun() except KeyError: raise RuntimeError( - "You must have CIRUN_API_KEY defined to do Cirun CI registration" + "You must have CIRUN_API_KEY defined to do Cirun CI registration. " "This requirement can be overriden by specifying `--without-cirun`" ) From 97456f4a7a41b457d9dba02afaed4c7e4b0a78c3 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Fri, 8 Sep 2023 14:05:35 +0530 Subject: [PATCH 45/66] remove reference to .cirun.yml template --- conda_smithy/configure_feedstock.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 0a1f44769..5100a9b7d 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -1271,12 +1271,6 @@ def _github_actions_specific_setup( if forge_config["github_actions"]["store_build_artifacts"]: template_files.append(".scripts/create_conda_build_artifacts.sh") - if forge_config["github_actions"]["self_hosted"]: - for label in forge_config["github_actions"]["self_hosted_labels"]: - if label.startswith("cirun-"): - template_files.append(".cirun.yml") - break - _render_template_exe_files( forge_config=forge_config, jinja_env=jinja_env, From f42717c13cc9c2b9d1e7b4e16cf7d08a351a287d Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Fri, 8 Sep 2023 20:15:02 +0530 Subject: [PATCH 46/66] add link to client docs --- conda_smithy/cirun_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index cfd92c77f..d1349c8b7 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -1,3 +1,6 @@ +""" +See http://py.cirun.io/api.html for cirun client docs +""" import os from functools import lru_cache from typing import List, Dict, Any, Optional From ab740b85bca8c55eda6a38e7d7ae7fa6b1f15578 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Fri, 8 Sep 2023 20:17:17 +0530 Subject: [PATCH 47/66] remove cirun_runners from forge config --- conda_smithy/configure_feedstock.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 5100a9b7d..38fc89d5c 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -1820,34 +1820,6 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): "store_build_artifacts": False, "artifact_retention_days": 14, }, - "cirun_runners": [ - { - "name": "cirun-openstack-gpu", - "labels": [ - "linux", - "x64", - "self-hosted", - "cirun-openstack-gpu", - ], - "cloud": "openstack", - "instance_type": "gpu_large", - "machine_image": "ubuntu-2204-nvidia-container-runtime-20230107", - "region": "RegionOne", - }, - { - "name": "cirun-openstack-cpu", - "labels": [ - "linux", - "x64", - "self-hosted", - "cirun-openstack-cpu", - ], - "cloud": "openstack", - "instance_type": "ci_medium", - "machine_image": "ubuntu-2204-cloud-jammy-20221104", - "region": "RegionOne", - }, - ], "recipe_dir": "recipe", "skip_render": [], "bot": {"automerge": False}, From 3472ee56bec1941a2d49826d3b3f39b836d20418 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 4 Oct 2023 17:07:05 +0200 Subject: [PATCH 48/66] fix ppc64le name --- conda_smithy/templates/github-actions.yml.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_smithy/templates/github-actions.yml.tmpl b/conda_smithy/templates/github-actions.yml.tmpl index c665418a3..5312ded47 100644 --- a/conda_smithy/templates/github-actions.yml.tmpl +++ b/conda_smithy/templates/github-actions.yml.tmpl @@ -44,7 +44,7 @@ concurrency: 'self_hosted_label_os': 'linux', 'self_hosted_label_arch': '', }, - 'linux-ppcle64': { + 'linux-ppc64le': { 'os': 'ubuntu', 'self_hosted_label_os': 'linux', 'self_hosted_label_arch': '', From 84cc27f90cd83dc094798edb331362ae4cf5e992 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 4 Oct 2023 17:07:21 +0200 Subject: [PATCH 49/66] add self hosted timeout minutes --- conda_smithy/configure_feedstock.py | 1 + conda_smithy/templates/github-actions.yml.tmpl | 3 +++ 2 files changed, 4 insertions(+) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 5fcecf22f..16502c338 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -1839,6 +1839,7 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): "self_hosted": False, "self_hosted_labels": [], "self_hosted_triggers": ["push"], + "self_hosted_timeout_minutes": 360, "cancel_in_progress": False, # Set maximum parallel jobs "max_parallel": None, diff --git a/conda_smithy/templates/github-actions.yml.tmpl b/conda_smithy/templates/github-actions.yml.tmpl index 5312ded47..8c505f062 100644 --- a/conda_smithy/templates/github-actions.yml.tmpl +++ b/conda_smithy/templates/github-actions.yml.tmpl @@ -62,6 +62,9 @@ jobs: name: {% raw %}${{ matrix.CONFIG }}{% endraw %} {%- if github_actions.self_hosted %} runs-on: {% raw %}${{ matrix.labels }}{% endraw %} + {%- if github_actions.self_hosted_timeout_minutes %} + timeout-minutes: {{ github_actions.self_hosted_timeout_minutes }} + {%- endif %} {%- else %} runs-on: {% raw %}${{ matrix.os }}{% endraw %}-latest {%- endif %} From af967c969e653197fa0d7e5eb91cf974a87aeb5d Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 6 Oct 2023 13:20:14 +0200 Subject: [PATCH 50/66] add more platform labels --- conda_smithy/templates/github-actions.yml.tmpl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/conda_smithy/templates/github-actions.yml.tmpl b/conda_smithy/templates/github-actions.yml.tmpl index 8c505f062..95eb3cf55 100644 --- a/conda_smithy/templates/github-actions.yml.tmpl +++ b/conda_smithy/templates/github-actions.yml.tmpl @@ -42,7 +42,7 @@ concurrency: 'linux-aarch64': { 'os': 'ubuntu', 'self_hosted_label_os': 'linux', - 'self_hosted_label_arch': '', + 'self_hosted_label_arch': 'ARM64', }, 'linux-ppc64le': { 'os': 'ubuntu', @@ -54,6 +54,11 @@ concurrency: 'self_hosted_label_os': 'windows', 'self_hosted_label_arch': 'x64', }, + 'win-arm64': { + 'os': 'windows', + 'self_hosted_label_os': 'windows', + 'self_hosted_label_arch': 'ARM64', + }, } %} From 94055403827b0888707f412b039d6290b6e464ea Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 17 Oct 2023 13:07:24 +0200 Subject: [PATCH 51/66] move labels config to configure_feedstock --- conda_smithy/configure_feedstock.py | 64 +++++++++++++++ .../templates/github-actions.yml.tmpl | 77 ++----------------- 2 files changed, 70 insertions(+), 71 deletions(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 16502c338..6909c73e5 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -1265,6 +1265,64 @@ def _github_actions_specific_setup( jinja_env, forge_config, forge_dir, platform ): + # Handle GH-hosted and self-hosted runners runs-on config + # Do it before the deepcopy below so these changes can be used by the + # .github/worfkflows/conda-build.yml template + runs_on = { + 'osx-64': { + 'os': 'macos', + 'self_hosted_labels': ('macOS', 'x64'), + }, + 'osx-arm64': { + 'os': 'macos', + 'self_hosted_labels': ('macOS', 'arm64'), + }, + 'linux-64': { + 'os': 'ubuntu', + 'self_hosted_labels': ('linux', 'x64'), + }, + 'linux-aarch64': { + 'os': 'ubuntu', + 'self_hosted_labels': ('linux', 'ARM64'), + }, + 'linux-ppc64le': { + 'os': 'ubuntu', + 'self_hosted_labels': ('linux', ''), + }, + 'win-64': { + 'os': 'windows', + 'self_hosted_labels': ('windows', 'x64'), + }, + 'win-arm64': { + 'os': 'windows', + 'self_hosted_labels': ('windows', 'ARM64'), + }, + } + for data in forge_config["configs"]: + if not data["build_platform"].startswith(platform): + continue + # This Github Actions specific configs are prefixed with "gha_" + # because we are not deepcopying the data dict intentionally + # so it can be used in the general "render_github_actions" function + # This avoid potential collisions with other CI providers :crossed_fingers: + data["gha_os"] = runs_on[data["build_platform"]]["os"] + data["gha_with_gpu"] = False + if forge_config["github_actions"]["self_hosted"]: + data["gha_runs_on"] = [ + "self-hosted", + # default, per-platform labels + *runs_on[data["build_platform"]]["self_hosted_labels"], + ] + # labels provided in conda-forge.yml + for label in forge_config["github_actions"]["self_hosted_labels"]: + if label.startswith("cirun-"): + label += "--${{ github.run_id }}-" + data["short_config_name"] + if "gpu" in label.lower(): + data["gha_with_gpu"] = True + data["gha_runs_on"].append(label) + else: + data["gha_runs_on"] = data["gha_os"] + "-latest" + build_setup = _get_build_setup_line(forge_dir, platform, forge_config) if platform == "linux": @@ -1320,6 +1378,12 @@ def render_github_actions( logger.debug("github platforms retrieved") + if forge_config["github_actions"]["self_hosted"]: + forge_config["github_actions"]["triggers"] = \ + forge_config["github_actions"]["self_hosted_triggers"] + else: + forge_config["github_actions"]["triggers"] = ["push", "pull_request"] + remove_file_or_dir(target_path) return _render_ci_provider( "github_actions", diff --git a/conda_smithy/templates/github-actions.yml.tmpl b/conda_smithy/templates/github-actions.yml.tmpl index 95eb3cf55..08a7ac328 100644 --- a/conda_smithy/templates/github-actions.yml.tmpl +++ b/conda_smithy/templates/github-actions.yml.tmpl @@ -11,11 +11,7 @@ {%- endfor %} name: Build conda package -{%- if github_actions.self_hosted %} -on: {{ github_actions.self_hosted_triggers }} -{%- else %} -on: [push, pull_request] -{%- endif %} +on: {{ github_actions.triggers }} {%- if github_actions.cancel_in_progress or github_actions.self_hosted %} concurrency: @@ -23,55 +19,12 @@ concurrency: cancel-in-progress: true {%- endif %} -{%- set platform_vars = { - 'osx-64': { - 'os': 'macos', - 'self_hosted_label_os': 'macOS', - 'self_hosted_label_arch': 'x64', - }, - 'osx-arm64': { - 'os': 'macos', - 'self_hosted_label_os': 'macOS', - 'self_hosted_label_arch': 'arm64', - }, - 'linux-64': { - 'os': 'ubuntu', - 'self_hosted_label_os': 'linux', - 'self_hosted_label_arch': 'x64', - }, - 'linux-aarch64': { - 'os': 'ubuntu', - 'self_hosted_label_os': 'linux', - 'self_hosted_label_arch': 'ARM64', - }, - 'linux-ppc64le': { - 'os': 'ubuntu', - 'self_hosted_label_os': 'linux', - 'self_hosted_label_arch': '', - }, - 'win-64': { - 'os': 'windows', - 'self_hosted_label_os': 'windows', - 'self_hosted_label_arch': 'x64', - }, - 'win-arm64': { - 'os': 'windows', - 'self_hosted_label_os': 'windows', - 'self_hosted_label_arch': 'ARM64', - }, -} -%} - jobs: build: name: {% raw %}${{ matrix.CONFIG }}{% endraw %} - {%- if github_actions.self_hosted %} - runs-on: {% raw %}${{ matrix.labels }}{% endraw %} - {%- if github_actions.self_hosted_timeout_minutes %} + runs-on: {% raw %}${{ matrix.runs_on }}{% endraw %} + {%- if github_actions.self_hosted and github_actions.self_hosted_timeout_minutes %} timeout-minutes: {{ github_actions.self_hosted_timeout_minutes }} - {%- endif %} - {%- else %} - runs-on: {% raw %}${{ matrix.os }}{% endraw %}-latest {%- endif %} strategy: fail-fast: false @@ -86,29 +39,11 @@ jobs: SHORT_CONFIG: {{ data.short_config_name }} {%- endif %} UPLOAD_PACKAGES: {{ data.upload }} - os: {{ platform_vars[data.build_platform]['os'] }} - {#- namespace needed to handle jinja scoping rules #} - {%- set ns = namespace(gpu=False) %} - {%- if github_actions.self_hosted %} - labels: - - self-hosted - - {{ platform_vars[data.build_platform]['self_hosted_label_os'] }} - {%- if platform_vars[data.build_platform]['self_hosted_label_arch'] %} - - {{ platform_vars[data.build_platform]['self_hosted_label_arch'] }} - {%- endif %} - {%- for label in github_actions.self_hosted_labels %} - {%- if 'gpu' in label %} - {%- set ns.gpu = True %} - {%- endif %} - {%- if label.startswith('cirun-') %} - {%- set label = label ~ "--${{ github.run_id }}-" ~ data.short_config_name %} - {%- endif %} - - {{ label }} - {%- endfor %} - {%- endif %} + os: {{ data.gha_os }} + runs_on: {{ data.gha_runs_on }} {%- if data.build_platform.startswith("linux") %} DOCKER_IMAGE: {{ data.config["docker_image"][-1] }} - {%- if ns.gpu %} + {%- if data.gha_with_gpu %} CONDA_FORGE_DOCKER_RUN_ARGS: "--gpus all" {%- endif %} {%- endif %} From 4c4c898f47f570246504f3d965cb33e5e958fef8 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 17 Oct 2023 13:09:27 +0200 Subject: [PATCH 52/66] pre-commit --- conda_smithy/configure_feedstock.py | 51 +++++++++++++++-------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 6909c73e5..08e9789b3 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -1269,33 +1269,33 @@ def _github_actions_specific_setup( # Do it before the deepcopy below so these changes can be used by the # .github/worfkflows/conda-build.yml template runs_on = { - 'osx-64': { - 'os': 'macos', - 'self_hosted_labels': ('macOS', 'x64'), + "osx-64": { + "os": "macos", + "self_hosted_labels": ("macOS", "x64"), }, - 'osx-arm64': { - 'os': 'macos', - 'self_hosted_labels': ('macOS', 'arm64'), + "osx-arm64": { + "os": "macos", + "self_hosted_labels": ("macOS", "arm64"), }, - 'linux-64': { - 'os': 'ubuntu', - 'self_hosted_labels': ('linux', 'x64'), + "linux-64": { + "os": "ubuntu", + "self_hosted_labels": ("linux", "x64"), }, - 'linux-aarch64': { - 'os': 'ubuntu', - 'self_hosted_labels': ('linux', 'ARM64'), + "linux-aarch64": { + "os": "ubuntu", + "self_hosted_labels": ("linux", "ARM64"), }, - 'linux-ppc64le': { - 'os': 'ubuntu', - 'self_hosted_labels': ('linux', ''), + "linux-ppc64le": { + "os": "ubuntu", + "self_hosted_labels": ("linux", ""), }, - 'win-64': { - 'os': 'windows', - 'self_hosted_labels': ('windows', 'x64'), + "win-64": { + "os": "windows", + "self_hosted_labels": ("windows", "x64"), }, - 'win-arm64': { - 'os': 'windows', - 'self_hosted_labels': ('windows', 'ARM64'), + "win-arm64": { + "os": "windows", + "self_hosted_labels": ("windows", "ARM64"), }, } for data in forge_config["configs"]: @@ -1316,7 +1316,9 @@ def _github_actions_specific_setup( # labels provided in conda-forge.yml for label in forge_config["github_actions"]["self_hosted_labels"]: if label.startswith("cirun-"): - label += "--${{ github.run_id }}-" + data["short_config_name"] + label += ( + "--${{ github.run_id }}-" + data["short_config_name"] + ) if "gpu" in label.lower(): data["gha_with_gpu"] = True data["gha_runs_on"].append(label) @@ -1379,8 +1381,9 @@ def render_github_actions( logger.debug("github platforms retrieved") if forge_config["github_actions"]["self_hosted"]: - forge_config["github_actions"]["triggers"] = \ - forge_config["github_actions"]["self_hosted_triggers"] + forge_config["github_actions"]["triggers"] = forge_config[ + "github_actions" + ]["self_hosted_triggers"] else: forge_config["github_actions"]["triggers"] = ["push", "pull_request"] From 98eb99cce31ba83fb052d616cfe77ebae6441c32 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Fri, 20 Oct 2023 15:15:21 -0500 Subject: [PATCH 53/66] Use recipe/conda_build_config.yaml for labels for more fine-grained control --- conda_smithy/configure_feedstock.py | 64 ++++++++++++------- .../templates/github-actions.yml.tmpl | 6 +- 2 files changed, 44 insertions(+), 26 deletions(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 08e9789b3..ed36f32a6 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -367,6 +367,7 @@ def _collapse_subpackage_variants( "channel_targets", "docker_image", "build_number_decrement", + "github_actions_labels", # The following keys are required for some of our aarch64 builds # Added in https://github.com/conda-forge/conda-forge-pinning-feedstock/pull/180 "cdt_arch", @@ -1285,10 +1286,6 @@ def _github_actions_specific_setup( "os": "ubuntu", "self_hosted_labels": ("linux", "ARM64"), }, - "linux-ppc64le": { - "os": "ubuntu", - "self_hosted_labels": ("linux", ""), - }, "win-64": { "os": "windows", "self_hosted_labels": ("windows", "x64"), @@ -1307,14 +1304,34 @@ def _github_actions_specific_setup( # This avoid potential collisions with other CI providers :crossed_fingers: data["gha_os"] = runs_on[data["build_platform"]]["os"] data["gha_with_gpu"] = False + + self_hosted_default = list( + runs_on[data["build_platform"]]["self_hosted_labels"] + ) + self_hosted_default += ["self-hosted"] + hosted_default = [data["gha_os"] + "-latest"] + + labels_default = ( + ["hosted"] + if forge_config["github_actions"]["self_hosted"] + else ["self-hosted"] + ) + labels = list( + data["config"].get("github_actions_labels", labels_default) + ) + + if len(labels) == 1 and labels[0] == "hosted": + labels = hosted_default + elif len(labels) == 1 and labels[0] in "self-hosted": + labels = self_hosted_default + else: + # Prepend the required ones + labels += self_hosted_default + if forge_config["github_actions"]["self_hosted"]: - data["gha_runs_on"] = [ - "self-hosted", - # default, per-platform labels - *runs_on[data["build_platform"]]["self_hosted_labels"], - ] + data["gha_runs_on"] = [] # labels provided in conda-forge.yml - for label in forge_config["github_actions"]["self_hosted_labels"]: + for label in labels: if label.startswith("cirun-"): label += ( "--${{ github.run_id }}-" + data["short_config_name"] @@ -1323,7 +1340,7 @@ def _github_actions_specific_setup( data["gha_with_gpu"] = True data["gha_runs_on"].append(label) else: - data["gha_runs_on"] = data["gha_os"] + "-latest" + data["gha_runs_on"] = hosted_default build_setup = _get_build_setup_line(forge_dir, platform, forge_config) @@ -1380,13 +1397,6 @@ def render_github_actions( logger.debug("github platforms retrieved") - if forge_config["github_actions"]["self_hosted"]: - forge_config["github_actions"]["triggers"] = forge_config[ - "github_actions" - ]["self_hosted_triggers"] - else: - forge_config["github_actions"]["triggers"] = ["push", "pull_request"] - remove_file_or_dir(target_path) return _render_ci_provider( "github_actions", @@ -1904,10 +1914,9 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): }, "github_actions": { "self_hosted": False, - "self_hosted_labels": [], - "self_hosted_triggers": ["push"], - "self_hosted_timeout_minutes": 360, - "cancel_in_progress": False, + "triggers": "default", + "timeout_minutes": 360, + "cancel_in_progress": "default", # Set maximum parallel jobs "max_parallel": None, # Toggle creating artifacts for conda build_artifacts dir @@ -2065,6 +2074,17 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): if config["test"] is None: config["test"] = "all" + if config["github_actions"]["cancel_in_progress"] == "default": + config["github_actions"]["cancel_in_progress"] = config[ + "github_actions" + ]["self_hosted"] + + if config["github_actions"]["triggers"] == "default": + self_hosted = config["github_actions"]["self_hosted"] + config["github_actions"]["triggers"] = ( + ["push"] if self_hosted else ["push", "pull_request"] + ) + # An older conda-smithy used to have some files which should no longer exist, # remove those now. old_files = [ diff --git a/conda_smithy/templates/github-actions.yml.tmpl b/conda_smithy/templates/github-actions.yml.tmpl index 08a7ac328..1e9a8addb 100644 --- a/conda_smithy/templates/github-actions.yml.tmpl +++ b/conda_smithy/templates/github-actions.yml.tmpl @@ -13,7 +13,7 @@ name: Build conda package on: {{ github_actions.triggers }} -{%- if github_actions.cancel_in_progress or github_actions.self_hosted %} +{%- if github_actions.cancel_in_progress %} concurrency: group: {% raw %}${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}{% endraw %} cancel-in-progress: true @@ -23,9 +23,7 @@ jobs: build: name: {% raw %}${{ matrix.CONFIG }}{% endraw %} runs-on: {% raw %}${{ matrix.runs_on }}{% endraw %} - {%- if github_actions.self_hosted and github_actions.self_hosted_timeout_minutes %} - timeout-minutes: {{ github_actions.self_hosted_timeout_minutes }} - {%- endif %} + timeout-minutes: {{ github_actions.timeout_minutes }} strategy: fail-fast: false {%- if github_actions.max_parallel %} From 3c079487ae3a3773c0bdea75dcc3e4cfa22ffa28 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Fri, 20 Oct 2023 16:12:13 -0500 Subject: [PATCH 54/66] Keep github_actions_labels only if self-hosted is turned on --- conda_smithy/configure_feedstock.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index da8e8b176..8ee12a9de 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -367,7 +367,6 @@ def _collapse_subpackage_variants( "channel_targets", "docker_image", "build_number_decrement", - "github_actions_labels", # The following keys are required for some of our aarch64 builds # Added in https://github.com/conda-forge/conda-forge-pinning-feedstock/pull/180 "cdt_arch", @@ -378,6 +377,9 @@ def _collapse_subpackage_variants( if not is_noarch: always_keep_keys.add("target_platform") + if forge_config["github_actions"]["self_hosted"]: + always_keep_keys.add("github_actions_labels") + all_used_vars.update(always_keep_keys) all_used_vars.update(top_level_vars) From c29e9cdb3223cff36c665b67f49bb32be61ab9e6 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Fri, 20 Oct 2023 16:16:11 -0500 Subject: [PATCH 55/66] Bring back free_disk_space --- .../templates/github-actions.yml.tmpl | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/conda_smithy/templates/github-actions.yml.tmpl b/conda_smithy/templates/github-actions.yml.tmpl index 1e9a8addb..88286e97f 100644 --- a/conda_smithy/templates/github-actions.yml.tmpl +++ b/conda_smithy/templates/github-actions.yml.tmpl @@ -47,6 +47,33 @@ jobs: {%- endif %} {%- endfor %} steps: +{%- if github_actions.free_disk_space %} + + - name: Manage disk space + if: matrix.os == 'ubuntu' + run: | + sudo mkdir -p /opt/empty_dir || true + for d in \ + /opt/ghc \ + /opt/hostedtoolcache \ + /usr/lib/jvm \ + /usr/local/.ghcup \ + /usr/local/lib/android \ + /usr/local/share/powershell \ + /usr/share/dotnet \ + /usr/share/swift \ + ; do + sudo rsync --stats -a --delete /opt/empty_dir/ $d || true + done + sudo apt-get purge -y -f firefox \ + google-chrome-stable \ + microsoft-edge-stable + sudo apt-get autoremove -y >& /dev/null + sudo apt-get autoclean -y >& /dev/null + sudo docker image prune --all --force + df -h +{%- endif %} + - name: Checkout code uses: actions/checkout@v3 {%- if clone_depth is not none %} From 41d65b2dc640b3232b0c7ea85c86b375f68bc17d Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Mon, 23 Oct 2023 09:04:08 -0500 Subject: [PATCH 56/66] Support other orgs for cirun --- conda_smithy/cirun_utils.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index d1349c8b7..ea6002290 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -7,11 +7,25 @@ from cirun import Cirun -# To get this id, got to https://github.com/organizations//settings/installations -# and then click on the Configure button next to Cirun Application, then copy -# the installation id from the URL, it would look something like: -# https://github.com/organizations/conda-forge/settings/installations/18453316 -CIRUN_INSTALLATION_ID = os.environ.get("CIRUN_INSTALLATION_ID", 18453316) + +@lru_cache +def get_cirun_installation_id(owner: str) -> int: + # This ID needs a token with admin: org privileges. + # Hard-code instead for easier use + if owner == "conda-forge": + return 18453316 + else: + from .github import gh_token, Github + gh = Github(gh_token) + user = gh.get_user() + if (user.login == owner): + user_or_org = user + else: + user_or_org = gh.get_organization(owner) + for inst in user_or_org.get_installations: + if inst.raw_data["app_slug"] == "cirun-application": + return inst.app_id + raise ValueError(f"cirun not found for owner {owner}") def enable_cirun_for_project(owner: str, repo: str) -> Dict[str, Any]: From d631fbdc44c38d7fb59c5600d69ab0b635c96ad3 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Mon, 23 Oct 2023 09:05:21 -0500 Subject: [PATCH 57/66] pre-commit --- conda_smithy/cirun_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index ea6002290..318232ff9 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -16,9 +16,10 @@ def get_cirun_installation_id(owner: str) -> int: return 18453316 else: from .github import gh_token, Github + gh = Github(gh_token) user = gh.get_user() - if (user.login == owner): + if user.login == owner: user_or_org = user else: user_or_org = gh.get_organization(owner) From 549d7b9cb3ae903477779f0c63e458661dee6edf Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Tue, 31 Oct 2023 08:56:03 -0500 Subject: [PATCH 58/66] Move import to top-level --- conda_smithy/cirun_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index 318232ff9..b509cbf73 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -6,6 +6,7 @@ from typing import List, Dict, Any, Optional from cirun import Cirun +from .github import gh_token, Github @lru_cache @@ -15,7 +16,6 @@ def get_cirun_installation_id(owner: str) -> int: if owner == "conda-forge": return 18453316 else: - from .github import gh_token, Github gh = Github(gh_token) user = gh.get_user() From 83079c485b25e388ddb11f346c00b19eae2680ba Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Tue, 31 Oct 2023 08:56:15 -0500 Subject: [PATCH 59/66] fix cirun installation id --- conda_smithy/cirun_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index b509cbf73..57ef9a8d3 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -33,9 +33,8 @@ def enable_cirun_for_project(owner: str, repo: str) -> Dict[str, Any]: """Enable the cirun.io Github Application for a particular repository.""" print(f"Enabling cirun for {owner}/{repo} ...") cirun = _get_cirun_client() - assert CIRUN_INSTALLATION_ID return cirun.set_repo( - f"{owner}/{repo}", installation_id=CIRUN_INSTALLATION_ID + f"{owner}/{repo}", installation_id=get_cirun_installation_id(owner) ) From fc4a0b000b0478287f1752548a94f4f737f1b958 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Tue, 31 Oct 2023 08:57:39 -0500 Subject: [PATCH 60/66] formatting --- conda_smithy/cirun_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conda_smithy/cirun_utils.py b/conda_smithy/cirun_utils.py index 57ef9a8d3..f393892fc 100644 --- a/conda_smithy/cirun_utils.py +++ b/conda_smithy/cirun_utils.py @@ -12,11 +12,10 @@ @lru_cache def get_cirun_installation_id(owner: str) -> int: # This ID needs a token with admin: org privileges. - # Hard-code instead for easier use + # Hard-code instead for easier use. if owner == "conda-forge": return 18453316 else: - gh = Github(gh_token) user = gh.get_user() if user.login == owner: From 87f7004e83c87a5b568a622cc120769086dec5a5 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Tue, 31 Oct 2023 09:07:03 -0500 Subject: [PATCH 61/66] Add news entry --- news/1703-cirun-self-hosted.rst | 46 +++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 news/1703-cirun-self-hosted.rst diff --git a/news/1703-cirun-self-hosted.rst b/news/1703-cirun-self-hosted.rst new file mode 100644 index 000000000..07e512839 --- /dev/null +++ b/news/1703-cirun-self-hosted.rst @@ -0,0 +1,46 @@ +**Added:** + +* For self-hosted github actions runs, a user can add custom labels + by adding `github_actions_labels` yaml key in `recipe/conda_build_config.yaml`. + The value `hosted` can be used for Microsoft hosted free runners + and the value `self-hosted` can be used for the default self-hosted labels. + +* `github_actions: timeout_minutes` option added to change the timeout in minutes. + The default value is `360`. + +* `github_actions: cancel_in_progress` option added to cancel in progress runs. + The default value is the value of `github_actions: self-hosted` which in turn + defaults to `false`. + +* `github_actions: triggers` is a list of triggers which defaults to + `push, pull_request` when not self-hosted and `push` when self-hosted. + +* Added a `--cirun` argument to `conda-smithy ci-register` command to register + `cirun` as a CI service. This makes `cirun` conda package a dependency of + conda-smithy. + +* Added support for `cirun` by generating a unique label when the self-hosted + label starts with `cirun`. + +* When a label is added that starts with `gpu` for a self-hosted runner, the docker + build will pass the GPUs to the docker instance. + +**Changed:** + +* + +**Deprecated:** + +* + +**Removed:** + +* + +**Fixed:** + +* + +**Security:** + +* From a72ca1e5a0c9830ef88a01ba6560bda0ff320bd2 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Tue, 31 Oct 2023 09:18:14 -0500 Subject: [PATCH 62/66] Fix default values --- conda_smithy/configure_feedstock.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 8ee12a9de..16a5c6440 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -1926,9 +1926,9 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): }, "github_actions": { "self_hosted": False, - "triggers": "default", + "triggers": [], "timeout_minutes": 360, - "cancel_in_progress": "default", + "cancel_in_progress": None, # Set maximum parallel jobs "max_parallel": None, # Toggle creating artifacts for conda build_artifacts dir @@ -2091,12 +2091,12 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): if config["test"] is None: config["test"] = "all" - if config["github_actions"]["cancel_in_progress"] == "default": + if config["github_actions"]["cancel_in_progress"] is None: config["github_actions"]["cancel_in_progress"] = config[ "github_actions" ]["self_hosted"] - if config["github_actions"]["triggers"] == "default": + if not config["github_actions"]["triggers"]: self_hosted = config["github_actions"]["self_hosted"] config["github_actions"]["triggers"] = ( ["push"] if self_hosted else ["push", "pull_request"] From 20252d39db7a07c80b42f8761f4441bdbebe78f0 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Tue, 31 Oct 2023 09:24:15 -0500 Subject: [PATCH 63/66] fix news --- news/1703-cirun-self-hosted.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/news/1703-cirun-self-hosted.rst b/news/1703-cirun-self-hosted.rst index 07e512839..71e40465c 100644 --- a/news/1703-cirun-self-hosted.rst +++ b/news/1703-cirun-self-hosted.rst @@ -22,8 +22,8 @@ * Added support for `cirun` by generating a unique label when the self-hosted label starts with `cirun`. -* When a label is added that starts with `gpu` for a self-hosted runner, the docker - build will pass the GPUs to the docker instance. +* When a label is added that has the string with `gpu` or `GPU` for a self-hosted + runner, the docker build will pass the GPUs to the docker instance. **Changed:** From e50cf3a704ab90361e5a5e7518b4309a4d9d92c9 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Tue, 31 Oct 2023 09:29:48 -0500 Subject: [PATCH 64/66] list of lists for zipping --- conda_smithy/configure_feedstock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 16a5c6440..615222e47 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -1319,8 +1319,8 @@ def _github_actions_specific_setup( if forge_config["github_actions"]["self_hosted"] else ["self-hosted"] ) - labels = list( - data["config"].get("github_actions_labels", labels_default) + labels = conda_build.utils.ensure_list( + data["config"].get("github_actions_labels", [labels_default])[0] ) if len(labels) == 1 and labels[0] == "hosted": From 2bbcd4d4f055582aaf3bb6e750b36edb02d38153 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Mon, 6 Nov 2023 13:30:54 -0600 Subject: [PATCH 65/66] change default value of cancel_in_progress --- conda_smithy/configure_feedstock.py | 7 +------ news/1703-cirun-self-hosted.rst | 7 ++----- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 1ce7f744c..3b7bd14f8 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -1924,7 +1924,7 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): "self_hosted": False, "triggers": [], "timeout_minutes": 360, - "cancel_in_progress": None, + "cancel_in_progress": True, # Set maximum parallel jobs "max_parallel": None, # Toggle creating artifacts for conda build_artifacts dir @@ -2085,11 +2085,6 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None): if config["test"] is None: config["test"] = "all" - if config["github_actions"]["cancel_in_progress"] is None: - config["github_actions"]["cancel_in_progress"] = config[ - "github_actions" - ]["self_hosted"] - if not config["github_actions"]["triggers"]: self_hosted = config["github_actions"]["self_hosted"] config["github_actions"]["triggers"] = ( diff --git a/news/1703-cirun-self-hosted.rst b/news/1703-cirun-self-hosted.rst index 71e40465c..fdf075a0a 100644 --- a/news/1703-cirun-self-hosted.rst +++ b/news/1703-cirun-self-hosted.rst @@ -8,10 +8,6 @@ * `github_actions: timeout_minutes` option added to change the timeout in minutes. The default value is `360`. -* `github_actions: cancel_in_progress` option added to cancel in progress runs. - The default value is the value of `github_actions: self-hosted` which in turn - defaults to `false`. - * `github_actions: triggers` is a list of triggers which defaults to `push, pull_request` when not self-hosted and `push` when self-hosted. @@ -27,7 +23,8 @@ **Changed:** -* +* `github_actions: cancel_in_progress` option added to cancel in progress runs. + The default value was changed to `true`. **Deprecated:** From 62d2a4b7023d69804147f4b35645f459fa05e550 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Mon, 6 Nov 2023 14:13:00 -0600 Subject: [PATCH 66/66] pre-commit --- conda_smithy/configure_feedstock.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 3b7bd14f8..5f49009d0 100644 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -1268,7 +1268,6 @@ def render_appveyor(jinja_env, forge_config, forge_dir, return_metadata=False): def _github_actions_specific_setup( jinja_env, forge_config, forge_dir, platform ): - # Handle GH-hosted and self-hosted runners runs-on config # Do it before the deepcopy below so these changes can be used by the # .github/worfkflows/conda-build.yml template