fix: do not use run_once instructions as it is wildly unreliable

playbooks/group_vars/all/consul.yml

@@ -5,6 +5,8 @@
 #                                                   #
 #####################################################
 
+consul_init_server: "{{ (inventory_hostname == groups['consul_servers'][0]) | bool }}"
+
 #####################
 # consul api config #
 #####################

playbooks/group_vars/all/nomad.yml

@@ -5,6 +5,8 @@
 #                                                   #
 #####################################################
 
+nomad_init_server: "{{ (inventory_hostname == groups['nomad_servers'][0]) | bool }}"
+
 ####################
 # nomad api config #
 ####################

playbooks/group_vars/all/vault.yml

@@ -5,6 +5,8 @@
 #                                                   #
 #####################################################
 
+vault_init_cluster: "{{ (inventory_hostname == groups['vault_servers'][0]) | bool }}"
+
 #########################
 # vault haproxy backend #
 #########################

playbooks/tasks/consul/consul_deploy.yml

@@ -32,17 +32,17 @@
         port: "{{ consul_api_port[consul_api_scheme] }}"
         scheme: "{{ consul_api_scheme }}"
         state: present
-      run_once: true
-      delegate_to: "{{ groups['consul_servers'] | first }}"
       register: _consul_init_secret
-      when: hashicorp_consul_configuration.acl.enabled
+      when:
+        - consul_init_server
+        - hashicorp_consul_configuration.acl.enabled
 
     - name: "Create consul agents token"
       when:
-        - consul_acl_configuration.enabled
+        - consul_init_server
+        - hashicorp_consul_configuration.acl.enabled
       block:
         - name: "Create consul agents token" # noqa: run-once[task] no-handler
-          run_once: true
           block:
             - name: "Create consul agent policy"
               community.general.consul_policy:

playbooks/tasks/nomad/nomad_deploy.yml

@@ -3,14 +3,13 @@
   block:
     - name: "Create consul tokens for service registration"
       when:
+        - nomad_init_server
         - enable_consul
         - nomad_enable_consul_integration
-      delegate_to: "{{ groups['consul_servers'] | first }}"
       vars:
         _consul_host: "{{ hostvars[groups['consul_servers'][0]].api_interface_address }}"
         _consul_port: "{{ hostvars[groups['consul_servers'][0]].consul_api_port[hostvars[groups['consul_servers'][0]].consul_api_scheme] }}"
         _consul_scheme: "{{ hostvars[groups['consul_servers'][0]].consul_api_scheme }}"
-      run_once: true
       block:
         - name: "Create server credentials"
           block:
@@ -77,7 +76,7 @@
         bootstrap_secret: "{{ _credentials.nomad.root_token.secret_id }}"
         api_url: "{{ nomad_api_addr }}"
         tls_verify: false
-      run_once: true
-      delegate_to: "{{ groups['nomad_servers'] | first }}"
       register: _nomad_init_secret
-      when: hashicorp_nomad_configuration.acl.enabled
+      when:
+        - nomad_init_server
+        - hashicorp_nomad_configuration.acl.enabled

playbooks/tasks/vault/vault_deploy.yml

@@ -3,14 +3,13 @@
   block:
     - name: "Create consul token for service registration"
       when:
+        - vault_init_cluster
         - enable_consul
         - vault_enable_service_registration
-      delegate_to: "{{ groups['consul_servers'] | first }}"
       vars:
         _consul_vault_sr_host: "{{ hostvars[groups['consul_servers'][0]].api_interface_address }}"
         _consul_vault_sr_port: "{{ hostvars[groups['consul_servers'][0]].consul_api_port[hostvars[groups['consul_servers'][0]].consul_api_scheme] }}"
         _consul_vault_sr_scheme: "{{ hostvars[groups['consul_servers'][0]].consul_api_scheme }}"
-      run_once: true
       block:
         - name: "Create consul vault policy"
           community.general.consul_policy:
@@ -48,12 +47,11 @@
         tls_verify: "{{ vault_tls_verify }}"
         key_shares: "{{ vault_seal_configuration['key_shares'] }}"
         key_threshold: "{{ vault_seal_configuration['key_threshold'] }}"
-      run_once: true
       retries: 5
       delay: 5
-      delegate_to: "{{ groups['vault_servers'] | first }}"
       register: _vault_init_secret
       until: not _vault_init_secret.failed
+      when: vault_init_cluster
 
     - name: "Write vault configuration to file" # noqa: run-once[task] no-handler
       ansible.builtin.copy:
@@ -62,15 +60,11 @@
         owner: "{{ lookup('env', 'USER') }}"
         group: "{{ lookup('env', 'USER') }}"
         mode: "0644"
-      when: _vault_init_secret.changed
+      when:
-      run_once: true
+        - vault_init_cluster
+        - _vault_init_secret.changed
       delegate_to: localhost
 
-    # - name: "Load vault cluster variables necessary for unseal operation"
-    #   ansible.builtin.include_vars:
-    #     file: "{{ sub_configuration_directories.vault_servers }}/vault_config.yml"
-    #     name: _vault_cluster_config
-
     - name: "Load vault cluster variables necessary for unseal operation"
       ansible.builtin.import_tasks:
         file: ../misc/load_credentials_vars.yml
@@ -80,9 +74,9 @@
         api_url: "{{ hashicorp_vault_configuration['api_addr'] }}"
         tls_verify: "{{ vault_tls_verify }}"
         key_shares: "{{ _credentials.vault['keys'] }}"
-      run_once: true
+      when:
-      delegate_to: "{{ groups['vault_servers'] | first }}"
+        - vault_init_cluster
-      when: _vault_init_secret.changed
+        - _vault_init_secret.changed
       register: _vault_unseal_secret
 
     - name: "Unseal all vault nodes"