Modernise ansible

.github/workflows/test.yml

@@ -79,7 +79,7 @@ jobs:
       - name: "Install python dependencies"
         run: |
           echo ::group::PYDEPS
-          pip install towncrier twine wheel httpie docker netaddr boto3 'ansible~=10.3.0' mkdocs jq jsonpatch bump-my-version
+          pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs jq jsonpatch bump-my-version
           echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_catdog/.ci/assets/httpie/" >> $GITHUB_ENV
           echo ::endgroup::
 

plugin-template

@@ -145,7 +145,6 @@ DEFAULT_SETTINGS = {
     "test_gcp": False,
     "test_lowerbounds": True,
     "test_performance": False,
-    "test_reroute": True,
     "test_s3": False,
     "use_issue_template": True,
 }

templates/github/.ci/ansible/Containerfile.j2.copy

@@ -1,34 +1,28 @@
-FROM {{ ci_base | default(pulp_default_container) }}
+FROM {{ image.ci_base }}
+{%- if image.webserver_snippet %}
 
-# Add source directories to container
-{% for item in plugins %}
-ADD ./{{ item.name }} ./{{ item.name }}
-{% endfor %}
+ADD ./{{ plugin_name }}/{{ plugin_name | replace("-", "_") }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ plugin_name }}.conf
+{%- endif %}
+
+{%- for item in extra_files | default([]) %}
 
-{% for item in extra_files | default([]) %}
 ADD ./{{ item.origin }} {{ item.destination }}
-{% endfor %}
+{%- endfor %}
 
 # This MUST be the ONLY call to pip install in inside the container.
 RUN pip3 install --upgrade pip setuptools wheel && \
   rm -rf /root/.cache/pip && \
-  pip3 install
-{%- if s3_test | default(false) -%}
-{{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue
-{%- endif -%}
-{%- for item in plugins -%}
-{{ " " }}{{ item.source }}
-{%- if item.upperbounds | default(false) -%}
-{{ " " }}-c ./{{ item.name }}/upperbounds_constraints.txt
+  pip3 install {{ image.source }}
+{%- if image.upperbounds | default(false) -%}
+{{ " " }}-c ./{{ plugin_name }}/upperbounds_constraints.txt
 {%- endif -%}
-{%- if item.lowerbounds | default(false) -%}
-{{ " " }}-c ./{{ item.name }}/lowerbounds_constraints.txt
+{%- if image.lowerbounds | default(false) -%}
+{{ " " }}-c ./{{ plugin_name }}/lowerbounds_constraints.txt
 {%- endif -%}
-{%- if item.ci_requirements | default(false) -%}
-{{ " " }}-r ./{{ item.name }}/ci_requirements.txt
+{%- if image.ci_requirements | default(false) -%}
+{{ " " }}-r ./{{ plugin_name }}/ci_requirements.txt
 {%- endif -%}
-{%- endfor %}
-{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \
+{{ " " }}-c ./{{ plugin_name }}/.ci/assets/ci_constraints.txt && \
   rm -rf /root/.cache/pip
 
 {% if pulp_env is defined and pulp_env %}
@@ -46,11 +40,8 @@ ENV {{ key | upper }}={{ value }}
 USER pulp:pulp
 RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \
        /usr/local/bin/pulpcore-manager collectstatic --clear --noinput --link
-USER root:root
 
-{% for item in plugins %}
-RUN export plugin_path="$(pip3 show {{ item.name }} | sed -n -e 's/Location: //p')/{{ item.name }}" && \
-    ln $plugin_path/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true
-{% endfor %}
+RUN mkdir /var/lib/pulp/.config
+USER root:root
 
 ENTRYPOINT ["/init"]

templates/github/.ci/ansible/build_container.yaml

@@ -1,15 +1,14 @@
 # Ansible playbook to create the pulp service containers image
 ---
-- hosts: localhost
+- hosts: "localhost"
   gather_facts: false
   vars_files:
-    - vars/main.yaml
+    - "vars/main.yaml"
   tasks:
     - name: "Generate Containerfile from template"
-      template:
-        src: Containerfile.j2
-        dest: Containerfile
-
+      ansible.builtin.template:
+        src: "Containerfile.j2"
+        dest: "Containerfile"
     - name: "Build pulp image"
       # We build from the ../.. (parent dir of pulpcore git repo) Docker build
       # "context" so that repos like pulp-smash are accessible to Docker

templates/github/.ci/ansible/start_container.yaml

@@ -1,33 +1,26 @@
 # Ansible playbook to start the pulp service container and its supporting services
 ---
-- hosts: localhost
+- hosts: "localhost"
   gather_facts: false
   vars_files:
-    - vars/main.yaml
+    - "vars/main.yaml"
   tasks:
     - name: "Create Settings Directories"
-      file:
+      ansible.builtin.file:
         path: "{{ item }}"
-        state: directory
+        state: "directory"
         mode: "0755"
       loop:
-        - settings
-        - ssh
-        - ~/.config/pulp_smash
+        - "settings"
 
     - name: "Generate Pulp Settings"
       template:
-        src: settings.py.j2
-        dest: settings/settings.py
-
-    - name: "Configure pulp-smash"
-      copy:
-        src: smash-config.json
-        dest: ~/.config/pulp_smash/settings.json
+        src: "settings.py.j2"
+        dest: "settings/settings.py"
 
     - name: "Setup docker networking"
       docker_network:
-        name: pulp_ci_bridge
+        name: "pulp_ci_bridge"
 
     - name: "Start Service Containers"
       docker_container:
@@ -37,24 +30,24 @@
         recreate: true
         privileged: true
         networks:
-          - name: pulp_ci_bridge
+          - name: "pulp_ci_bridge"
             aliases: "{{ item.name }}"
         volumes: "{{ item.volumes | default(omit) }}"
         env: "{{ item.env | default(omit) }}"
         command: "{{ item.command | default(omit) }}"
-        state: started
+        state: "started"
       loop: "{{ services | default([]) }}"
 
     - name: "Retrieve Docker Network Info"
       docker_network_info:
-        name: pulp_ci_bridge
-      register: pulp_ci_bridge_info
+        name: "pulp_ci_bridge"
+      register: "pulp_ci_bridge_info"
 
     - name: "Update /etc/hosts"
       lineinfile:
-        path: /etc/hosts
+        path: "/etc/hosts"
         regexp: "\\s{{ item.value.Name }}\\s*$"
-        line: "{{ item.value.IPv4Address | ipaddr('address') }}\t{{ item.value.Name }}"
+        line: "{{ item.value.IPv4Address | ansible.utils.ipaddr('address') }}\t{{ item.value.Name }}"
       loop: "{{ pulp_ci_bridge_info.network.Containers | dict2items }}"
       become: true
 
@@ -63,19 +56,19 @@
         aws_access_key: "{{ minio_access_key }}"
         aws_secret_key: "{{ minio_secret_key }}"
         s3_url: "http://minio:9000"
-        region: eu-central-1
-        name: pulp3
-        state: present
-      when: s3_test | default(false)
+        region: "eu-central-1"
+        name: "pulp3"
+        state: "present"
+      when: "s3_test | default(false)"
 
     - block:
         - name: "Wait for Pulp"
           uri:
             url: "http://pulp{{ lookup('env', 'PULP_API_ROOT') | default('\/pulp\/', True) }}api/v3/status/"
-            follow_redirects: all
-            validate_certs: no
-          register: result
-          until: result.status == 200
+            follow_redirects: "all"
+            validate_certs: "no"
+          register: "result"
+          until: "result.status == 200"
           retries: 12
           delay: 5
       rescue:
@@ -86,7 +79,7 @@
     - name: "Check version of component being tested"
       assert:
         that:
-          - (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] | canonical_semver == (component_version | canonical_semver)
+          - "(result.json.versions | items2dict(key_name='component', value_name='version'))[item.app_label] | canonical_semver == (component_version | canonical_semver)"
         fail_msg: |
           Component {{ item.app_label }} was expected to be installed in version {{ component_version }}.
           Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] }}.
@@ -100,9 +93,20 @@
           login admin
           password password
 
-- hosts: pulp
+- hosts: "pulp"
   gather_facts: false
   tasks:
+    - name: "Create directory for pulp-smash config"
+      ansible.builtin.file:
+        path: "/var/lib/pulp/.config/pulp_smash/"
+        state: "directory"
+        mode: "0755"
+
+    - name: "Configure pulp-smash"
+      ansible.builtin.copy:
+        src: "smash-config.json"
+        dest: "/var/lib/pulp/.config/pulp_smash/settings.json"
+
     - name: "Set pulp admin password"
       command:
         cmd: "pulpcore-manager reset-admin-password --password password"

templates/github/.github/workflows/scripts/before_install.sh.j2

@@ -24,7 +24,7 @@ COMPONENT_SOURCE="./{{ plugin_name }}/dist/{{ plugin_name | snake }}-${COMPONENT
 {%- set PULPCORE_PREFIX = "" if plugin_name == "pulpcore" else " pulpcore" %}
 {%- if test_s3 %}
 if [ "$TEST" = "s3" ]; then
-  COMPONENT_SOURCE="${COMPONENT_SOURCE}{{ PULPCORE_PREFIX }}[s3]"
+  COMPONENT_SOURCE="${COMPONENT_SOURCE}{{ PULPCORE_PREFIX }}[s3] git+https://github.com/gerrod3/botocore.git@fix-100-continue"
 fi
 {%- endif %}
 {%- if test_azure %}
@@ -45,11 +45,7 @@ if [[ "$TEST" = "lowerbounds" ]]; then
   python3 .ci/scripts/calc_constraints.py {% if setup_py -%} requirements.txt {% else -%} pyproject.toml {% endif -%} > lowerbounds_constraints.txt
 fi
 
-{%- if test_reroute %}
 export PULP_API_ROOT=$(test "${TEST}" = "s3" && echo "/rerouted/djnd/" || echo "{{ api_root }}")
-{%- else %}
-export PULP_API_ROOT="{{ api_root }}"
-{%- endif %}
 
 echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV"
 
@@ -59,41 +55,39 @@ mkdir -p .ci/ansible/vars
 cat > .ci/ansible/vars/main.yaml << VARSYAML
 ---
 scenario: "${TEST}"
+plugin_name: "{{ plugin_name }}"
 legacy_component_name: "{{ plugin_name | snake }}"
 component_name: "{{ plugin_app_label }}"
 component_version: "${COMPONENT_VERSION}"
 pulp_env: {{ pulp_env | tojson }}
 pulp_settings: {{ pulp_settings | tojson }}
 pulp_scheme: "{{ pulp_scheme }}"
-{%- if ci_base_image.count(":") %}
-pulp_default_container: "{{ ci_base_image }}"
-{% else %}
-pulp_default_container: "{{ ci_base_image + ":latest" }}"
-{% endif -%}
 api_root: "${PULP_API_ROOT}"
 image:
   name: "pulp"
   tag: "ci_build"
-plugins:
-  - name: "{{ plugin_name }}"
-    source: "${COMPONENT_SOURCE}"
-    ci_requirements: $(test -f ci_requirements.txt && echo -n true || echo -n false)
-    upperbounds: $(test "${TEST}" = "pulp" && echo -n true || echo -n false)
-    lowerbounds: $(test "${TEST}" = "lowerbounds" && echo -n true || echo -n false)
-{%- if extra_files %}
+  {%- if ci_base_image.count(":") %}
+  ci_base: "{{ ci_base_image }}"
+  {% else %}
+  ci_base: "{{ ci_base_image + ":latest" }}"
+  {% endif -%}
+  source: "${COMPONENT_SOURCE}"
+  ci_requirements: $(test -f ci_requirements.txt && echo -n true || echo -n false)
+  upperbounds: $(test "${TEST}" = "pulp" && echo -n true || echo -n false)
+  lowerbounds: $(test "${TEST}" = "lowerbounds" && echo -n true || echo -n false)
+  webserver_snippet: $(test -f {{ plugin_name | snake }}/app/webserver_snippets/nginx.conf && echo -n true || echo -n false )
 extra_files:
+  - origin: "{{ plugin_name }}"
+    destination: "{{ plugin_name }}"
   {%- for item in extra_files %}
   - origin: "{{ item.origin }}"
     destination: "{{ item.destination }}"
   {%- endfor %}
-{%- endif %}
 services:
   - name: "pulp"
     image: "pulp:ci_build"
     volumes:
       - "./settings:/etc/pulp"
-      - "./ssh:/keys/"
-      - "~/.config:/var/lib/pulp/.config"
       - "../../../pulp-openapi-generator:/root/pulp-openapi-generator"
     env:
       PULP_WORKERS: "4"

templates/github/.github/workflows/scripts/before_script.sh.j2

@@ -18,7 +18,7 @@ fi
 # Developers often want to know the final pulp config
 echo
 echo "# Pulp config:"
-tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json
+tail -v -n +1 .ci/ansible/settings/settings.*
 
 echo
 echo "# Containerfile:"

templates/github/.github/workflows/scripts/install.sh.j2

@@ -43,10 +43,6 @@ cp ~/.config/pulp/cli.toml "${REPO_ROOT}/../{{ cli_package }}/tests/cli.toml"
 ansible-playbook build_container.yaml
 ansible-playbook start_container.yaml
 
-# .config needs to be accessible by the pulp user in the container, but some
-# files will likely be modified on the host by post/pre scripts.
-chmod 777 ~/.config/pulp_smash/
-chmod 666 ~/.config/pulp_smash/settings.json
 {% if test_cli -%}
 # Plugins often write to ~/.config/pulp/cli.toml from the host
 chmod 777 ~/.config/pulp