Merge branch 'ansible:devel' into devel

delinea-sagar · Jul 6, 2023 · 1355314 · 1355314
2 parents 4adfc0c + 82abd18
commit 1355314
Show file tree

Hide file tree

Showing 16 changed files with 150 additions and 44 deletions.
diff --git a/Makefile b/Makefile
@@ -27,6 +27,8 @@ COLLECTION_TEMPLATE_VERSION ?= false
 # NOTE: This defaults the container image version to the branch that's active
 COMPOSE_TAG ?= $(GIT_BRANCH)
 MAIN_NODE_TYPE ?= hybrid
+# If set to true docker-compose will also start a pgbouncer instance and use it
+PGBOUNCER ?= false
 # If set to true docker-compose will also start a keycloak instance
 KEYCLOAK ?= false
 # If set to true docker-compose will also start an ldap instance
@@ -522,6 +524,7 @@ docker-compose-sources: .git/hooks/pre-commit
 	    -e control_plane_node_count=$(CONTROL_PLANE_NODE_COUNT) \
 	    -e execution_node_count=$(EXECUTION_NODE_COUNT) \
 	    -e minikube_container_group=$(MINIKUBE_CONTAINER_GROUP) \
+	    -e enable_pgbouncer=$(PGBOUNCER) \
 	    -e enable_keycloak=$(KEYCLOAK) \
 	    -e enable_ldap=$(LDAP) \
 	    -e enable_splunk=$(SPLUNK) \

diff --git a/awx/api/templates/instance_install_bundle/group_vars/all.yml b/awx/api/templates/instance_install_bundle/group_vars/all.yml
@@ -12,7 +12,7 @@ receptor_work_commands:
 custom_worksign_public_keyfile: receptor/work_public_key.pem
 custom_tls_certfile: receptor/tls/receptor.crt
 custom_tls_keyfile: receptor/tls/receptor.key
-custom_ca_certfile: receptor/tls/ca/receptor-ca.crt
+custom_ca_certfile: receptor/tls/ca/mesh-CA.crt
 receptor_protocol: 'tcp'
 receptor_listener: true
 receptor_port: {{ instance.listener_port }}

diff --git a/awx/conf/apps.py b/awx/conf/apps.py
@@ -14,7 +14,7 @@ class ConfConfig(AppConfig):
     def ready(self):
         self.module.autodiscover()
 
-        if not set(sys.argv) & {'migrate', 'check_migrations'}:
+        if not set(sys.argv) & {'migrate', 'check_migrations', 'showmigrations'}:
             from .settings import SettingsWrapper
 
             SettingsWrapper.initialize()
diff --git a/awx/main/conf.py b/awx/main/conf.py
@@ -94,6 +94,20 @@
     category_slug='system',
 )
 
+register(
+    'CSRF_TRUSTED_ORIGINS',
+    default=[],
+    field_class=fields.StringListField,
+    label=_('CSRF Trusted Origins List'),
+    help_text=_(
+        "If the service is behind a reverse proxy/load balancer, use this setting "
+        "to configure the schema://addresses from which the service should trust "
+        "Origin header values. "
+    ),
+    category=_('System'),
+    category_slug='system',
+)
+
 register(
     'LICENSE',
     field_class=fields.DictField,

diff --git a/awx/main/models/workflow.py b/awx/main/models/workflow.py
@@ -661,7 +661,11 @@ def workflow_nodes(self):
 
     @property
     def event_processing_finished(self):
-        return True
+        return True  # workflow jobs do not have events
+
+    @property
+    def has_unpartitioned_events(self):
+        return False  # workflow jobs do not have events
 
     def _get_parent_field_name(self):
         if self.job_template_id:
@@ -914,7 +918,11 @@ def signal_start(self, **kwargs):
 
     @property
     def event_processing_finished(self):
-        return True
+        return True  # approval jobs do not have events
+
+    @property
+    def has_unpartitioned_events(self):
+        return False  # approval jobs do not have events
 
     def send_approval_notification(self, approval_status):
         from awx.main.tasks.system import send_notifications  # avoid circular import

diff --git a/awx/playbooks/project_update.yml b/awx/playbooks/project_update.yml
@@ -189,11 +189,12 @@
   connection: local
   name: Install content with ansible-galaxy command if necessary
   vars:
-    galaxy_task_env:  # configure in settings
-    additional_collections_env:
-      # These environment variables are used for installing collections, in addition to galaxy_task_env
-      # setting the collections paths silences warnings
+    galaxy_task_env:  # configured in settings
+    # additional_galaxy_env contains environment variables are used for installing roles and collections and will take precedence over items in galaxy_task_env
+    additional_galaxy_env:
+      # These paths control where ansible-galaxy installs collections and roles on top the filesystem
       ANSIBLE_COLLECTIONS_PATHS: "{{ projects_root }}/.__awx_cache/{{ local_path }}/stage/requirements_collections"
+      ANSIBLE_ROLES_PATH: "{{ projects_root }}/.__awx_cache/{{ local_path }}/stage/requirements_roles"
       # Put the local tmp directory in same volume as collection destination
       # otherwise, files cannot be moved accross volumes and will cause error
       ANSIBLE_LOCAL_TEMP: "{{ projects_root }}/.__awx_cache/{{ local_path }}/stage/tmp"
@@ -212,40 +213,50 @@
         - name: End play due to disabled content sync
           ansible.builtin.meta: end_play
 
-    - name: Fetch galaxy roles from requirements.(yml/yaml)
-      ansible.builtin.command: >
-        ansible-galaxy role install -r {{ item }}
-        --roles-path {{ projects_root }}/.__awx_cache/{{ local_path }}/stage/requirements_roles
-        {{ ' -' + 'v' * ansible_verbosity if ansible_verbosity else '' }}
-      args:
-        chdir: "{{ project_path | quote }}"
-      register: galaxy_result
-      with_fileglob:
-        - "{{ project_path | quote }}/roles/requirements.yaml"
-        - "{{ project_path | quote }}/roles/requirements.yml"
-      changed_when: "'was installed successfully' in galaxy_result.stdout"
-      environment: "{{ galaxy_task_env }}"
-      when: roles_enabled | bool
-      tags:
-        - install_roles
+    - block:
+        - name: Fetch galaxy roles from roles/requirements.(yml/yaml)
+          ansible.builtin.command:
+            cmd: "ansible-galaxy role install -r {{ item }} {{ verbosity }}"
+          register: galaxy_result
+          with_fileglob:
+            - "{{ project_path | quote }}/roles/requirements.yaml"
+            - "{{ project_path | quote }}/roles/requirements.yml"
+          changed_when: "'was installed successfully' in galaxy_result.stdout"
+          when: roles_enabled | bool
+          tags:
+            - install_roles
 
-    - name: Fetch galaxy collections from collections/requirements.(yml/yaml)
-      ansible.builtin.command: >
-        ansible-galaxy collection install -r {{ item }}
-        --collections-path {{ projects_root }}/.__awx_cache/{{ local_path }}/stage/requirements_collections
-        {{ ' -' + 'v' * ansible_verbosity if ansible_verbosity else '' }}
-      args:
-        chdir: "{{ project_path | quote }}"
-      register: galaxy_collection_result
-      with_fileglob:
-        - "{{ project_path | quote }}/collections/requirements.yaml"
-        - "{{ project_path | quote }}/collections/requirements.yml"
-        - "{{ project_path | quote }}/requirements.yaml"
-        - "{{ project_path | quote }}/requirements.yml"
-      changed_when: "'Installing ' in galaxy_collection_result.stdout"
-      environment: "{{ additional_collections_env | combine(galaxy_task_env) }}"
-      when:
-        - "ansible_version.full is version_compare('2.9', '>=')"
-        - collections_enabled | bool
-      tags:
-        - install_collections
+        - name: Fetch galaxy collections from collections/requirements.(yml/yaml)
+          ansible.builtin.command:
+            cmd: "ansible-galaxy collection install -r {{ item }} {{ verbosity }}"
+          register: galaxy_collection_result
+          with_fileglob:
+            - "{{ project_path | quote }}/collections/requirements.yaml"
+            - "{{ project_path | quote }}/collections/requirements.yml"
+          changed_when: "'Nothing to do.' not in galaxy_collection_result.stdout"
+          when:
+            - "ansible_version.full is version_compare('2.9', '>=')"
+            - collections_enabled | bool
+          tags:
+            - install_collections
+
+        - name: Fetch galaxy roles and collections from requirements.(yml/yaml)
+          ansible.builtin.command:
+            cmd: "ansible-galaxy install -r {{ item }} {{ verbosity }}"
+          register: galaxy_combined_result
+          with_fileglob:
+            - "{{ project_path | quote }}/requirements.yaml"
+            - "{{ project_path | quote }}/requirements.yml"
+          changed_when: "'Nothing to do.' not in galaxy_combined_result.stdout"
+          when:
+            - "ansible_version.full is version_compare('2.10', '>=')"
+            - collections_enabled | bool
+            - roles_enabled | bool
+          tags:
+            - install_collections
+            - install_roles
+
+      # We combine our additional_galaxy_env into galaxy_task_env so that our values are preferred over anything a user would set
+      environment: "{{ galaxy_task_env | combine(additional_galaxy_env) }}"
+      vars:
+        verbosity: "{{ (ansible_verbosity) | ternary('-'+'v'*ansible_verbosity, '') }}"
diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py
@@ -158,6 +158,11 @@
 # REMOTE_HOST_HEADERS will be trusted unconditionally')
 PROXY_IP_ALLOWED_LIST = []
 
+# If we are behind a reverse proxy/load balancer, use this setting to
+# allow the scheme://addresses from which Tower should trust csrf requests from
+# If this setting is an empty list (the default), we will only trust ourself
+CSRF_TRUSTED_ORIGINS = []
+
 CUSTOM_VENV_PATHS = []
 
 # Warning: this is a placeholder for a database setting

diff --git a/awx/ui/src/screens/Setting/MiscSystem/MiscSystemDetail/MiscSystemDetail.js b/awx/ui/src/screens/Setting/MiscSystem/MiscSystemDetail/MiscSystemDetail.js
@@ -59,6 +59,7 @@ function MiscSystemDetail() {
         'TOWER_URL_BASE',
         'DEFAULT_EXECUTION_ENVIRONMENT',
         'PROXY_IP_ALLOWED_LIST',
+        'CSRF_TRUSTED_ORIGINS',
         'AUTOMATION_ANALYTICS_LAST_GATHER',
         'AUTOMATION_ANALYTICS_LAST_ENTRIES',
         'UI_NEXT'

diff --git a/awx/ui/src/screens/Setting/MiscSystem/MiscSystemDetail/MiscSystemDetail.test.js b/awx/ui/src/screens/Setting/MiscSystem/MiscSystemDetail/MiscSystemDetail.test.js
@@ -29,6 +29,7 @@ describe('<MiscSystemDetail />', () => {
         TOWER_URL_BASE: 'https://towerhost',
         REMOTE_HOST_HEADERS: [],
         PROXY_IP_ALLOWED_LIST: [],
+        CSRF_TRUSTED_ORIGINS: [],
         LICENSE: null,
         REDHAT_USERNAME: 'name1',
         REDHAT_PASSWORD: '$encrypted$',

diff --git a/awx/ui/src/screens/Setting/MiscSystem/MiscSystemEdit/MiscSystemEdit.js b/awx/ui/src/screens/Setting/MiscSystem/MiscSystemEdit/MiscSystemEdit.js
@@ -53,6 +53,7 @@ function MiscSystemEdit() {
         'TOWER_URL_BASE',
         'DEFAULT_EXECUTION_ENVIRONMENT',
         'PROXY_IP_ALLOWED_LIST',
+        'CSRF_TRUSTED_ORIGINS',
         'UI_NEXT'
       );
 
@@ -95,6 +96,7 @@ function MiscSystemEdit() {
     await submitForm({
       ...form,
       PROXY_IP_ALLOWED_LIST: formatJson(form.PROXY_IP_ALLOWED_LIST),
+      CSRF_TRUSTED_ORIGINS: formatJson(form.CSRF_TRUSTED_ORIGINS),
       REMOTE_HOST_HEADERS: formatJson(form.REMOTE_HOST_HEADERS),
       DEFAULT_EXECUTION_ENVIRONMENT:
         form.DEFAULT_EXECUTION_ENVIRONMENT?.id || null,
@@ -239,6 +241,11 @@ function MiscSystemEdit() {
                   config={system.PROXY_IP_ALLOWED_LIST}
                   isRequired
                 />
+                <ObjectField
+                  name="CSRF_TRUSTED_ORIGINS"
+                  config={system.CSRF_TRUSTED_ORIGINS}
+                  isRequired
+                />
                 {submitError && <FormSubmitError error={submitError} />}
                 {revertError && <FormSubmitError error={revertError} />}
               </FormColumnLayout>

diff --git a/awx/ui/src/screens/Setting/MiscSystem/MiscSystemEdit/MiscSystemEdit.test.js b/awx/ui/src/screens/Setting/MiscSystem/MiscSystemEdit/MiscSystemEdit.test.js
@@ -39,6 +39,7 @@ const systemData = {
   REMOTE_HOST_HEADERS: ['REMOTE_ADDR', 'REMOTE_HOST'],
   TOWER_URL_BASE: 'https://localhost:3000',
   PROXY_IP_ALLOWED_LIST: [],
+  CSRF_TRUSTED_ORIGINS: [],
   UI_NEXT: false,
 };
 

diff --git a/awx/ui/src/screens/Setting/shared/data.allSettingOptions.json b/awx/ui/src/screens/Setting/shared/data.allSettingOptions.json
@@ -78,6 +78,20 @@
                   "read_only": false
               }
           },
+          "CSRF_TRUSTED_ORIGINS": {
+              "type": "list",
+              "required": true,
+              "label": "CSRF Origins List",
+              "help_text": "If the service is behind a reverse proxy/load balancer, use this setting to configure the schema://addresses from which the service should trust Origin header values. ",
+              "category": "System",
+              "category_slug": "system",
+              "default": [],
+              "child": {
+                  "type": "string",
+                  "required": true,
+                  "read_only": false
+              }
+          },
           "REDHAT_USERNAME": {
               "type": "string",
               "required": false,
@@ -4487,6 +4501,17 @@
                   "type": "string"
               }
           },
+          "CSRF_TRUSTED_ORIGINS": {
+              "type": "list",
+              "label": "CSRF Origins List",
+              "help_text": "If the service is behind a reverse proxy/load balancer, use this setting to configure the schema://addresses from which the service should trust Origin header values. ",
+              "category": "System",
+              "category_slug": "system",
+              "defined_in_file": false,
+              "child": {
+                  "type": "string"
+              }
+          },
           "LICENSE": {
               "type": "nested object",
               "label": "License",

diff --git a/awx/ui/src/screens/Setting/shared/data.allSettings.json b/awx/ui/src/screens/Setting/shared/data.allSettings.json
@@ -9,6 +9,7 @@
       "REMOTE_HOST"
   ],
   "PROXY_IP_ALLOWED_LIST": [],
+  "CSRF_TRUSTED_ORIGINS": [],
   "LICENSE": {},
   "REDHAT_USERNAME": "",
   "REDHAT_PASSWORD": "",

diff --git a/tools/docker-compose/ansible/roles/sources/defaults/main.yml b/tools/docker-compose/ansible/roles/sources/defaults/main.yml
@@ -35,3 +35,8 @@ enable_splunk: false
 enable_grafana: false
 enable_prometheus: false
 scrape_interval: '5s'
+
+# pgbouncer
+enable_pgbouncer: false
+pgbouncer_port: 6432
+pgbouncer_max_pool_size: 70
diff --git a/tools/docker-compose/ansible/roles/sources/templates/database.py.j2 b/tools/docker-compose/ansible/roles/sources/templates/database.py.j2
@@ -5,7 +5,12 @@ DATABASES = {
         'NAME': "{{ pg_database }}",
         'USER': "{{ pg_username }}",
         'PASSWORD': "{{ pg_password }}",
+{% if enable_pgbouncer|bool %}
+        'HOST': "pgbouncer",
+        'PORT': "{{ pgbouncer_port }}",
+{% else %}
         'HOST': "{{ pg_hostname | default('postgres') }}",
         'PORT': "{{ pg_port }}",
+{% endif %}
     }
 }
diff --git a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2
@@ -201,6 +201,25 @@ services:
       POSTGRES_PASSWORD: {{ pg_password }}
     volumes:
       - "awx_db:/var/lib/postgresql/data"
+{% if enable_pgbouncer|bool %}
+  pgbouncer:
+    image: bitnami/pgbouncer:latest
+    container_name: tools_pgbouncer_1
+    hostname: pgbouncer
+    environment:
+      POSTGRESQL_USERNAME: {{ pg_username }}
+      POSTGRESQL_DATABASE: {{ pg_database }}
+      PGBOUNCER_DATABASE: {{ pg_database }}
+      POSTGRESQL_PASSWORD: {{ pg_password }}
+      POSTGRESQL_HOST: {{ pg_hostname | default('postgres') }}
+      POSTGRESQL_PORT: {{ pg_port }}
+      PGBOUNCER_AUTH_TYPE: trust
+      PGBOUNCER_PORT: {{ pgbouncer_port }}
+      PGBOUNCER_DEFAULT_POOL_SIZE: {{ pgbouncer_max_pool_size }}
+      # This is the default, but we're being explicit here because it's important:
+      # pg_notify will NOT work in transaction mode.
+      PGBOUNCER_POOL_MODE: session
+{% endif %}
 {% if execution_node_count|int > 0 %}
   receptor-hop:
     image: {{ receptor_image }}