feat(nomad): move variables to globals.yml, adjust bootstrap module for nomad ACLs

molecule/no_tls_multi_node/etc/hashistack/globals.yml

@@ -9,7 +9,7 @@
 # enable_nomad: "yes"
 
 # haproxy_version: "2.8"
-# nomad_version: "1.7.7"
+# nomad_version: "1.8.1"
 # consul_version: "1.18.1"
 # vault_version: "1.16.2"
 
@@ -112,9 +112,9 @@ consul_enable_tls: true
 
 # extra_consul_container_volumes: []
 
-#######################
+##############################
-# extra configuration #
+# consul extra configuration #
-#######################
+##############################
 
 # consul_extra_configuration: {}
 # consul_extra_files_list: []
@@ -202,9 +202,76 @@ vault_enable_tls: true
 
 # extra_vault_container_volumes: []
 
-#####################
+#############################
-# extra configuration
+# vault extra configuration #
-#####################
+#############################
 
 # vault_extra_configuration: {}
 # vault_extra_files_list: []
+
+#####################################################
+#                                                   #
+#                Nomad                              #
+#                                                   #
+#####################################################
+
+# nomad_datacenter: dc1
+# nomad_region: global
+
+###########################
+# nomad ACL configuration #
+###########################
+
+# nomad_acl_configuration:
+#   enabled: true
+#   token_ttl: 30s
+#   policy_ttl: 60s
+#   role_ttl: 60s
+
+############################
+# nomad consul integration #
+############################
+
+# nomad_enable_consul_integration: "{{ enable_consul | bool }}"
+# nomad_consul_integration_configuration:
+#   address: "127.0.0.1:{{ hashicorp_consul_configuration.ports.https if consul_enable_tls else hashicorp_consul_configuration.ports.http }}"
+#   auto_advertise: true
+#   ssl: "{{ consul_enable_tls | bool }}"
+#   token: "{{ _credentials.consul.tokens.nomad.server.secret_id if nomad_enable_server else _credentials.consul.tokens.nomad.client.secret_id}}"
+#   tags: []
+
+############################
+# nomad vault integration #
+############################
+
+# nomad_enable_vault_integration: false
+# nomad_vault_integration_configuration: {}
+
+###############################
+# nomad drivers configuration #
+###############################
+
+######################
+# nomad internal tls #
+######################
+
+nomad_enable_tls: true
+nomad_tls_configuration:
+  http: true
+  rpc: true
+  rpc_upgrade_mode: true
+  ca_file: "/etc/ssl/certs/ca-certificates.crt"
+  cert_file: "{{ nomad_certificates_directory }}/cert.pem"
+  key_file: "{{ nomad_certificates_directory }}/key.pem"
+  verify_server_hostname: true
+# nomad_certificates_directory: "{{ hashicorp_nomad_config_dir }}/tls"
+# nomad_certificates_extra_files_dir:
+#   - src: "{{ sub_configuration_directories['certificates'] }}/nomad/{{ inventory_hostname }}"
+#     dest: "{{ nomad_certificates_directory }}"
+
+#############################
+# nomad extra configuration #
+#############################
+
+# nomad_extra_configuration: {}
+# nomad_extra_files_list: []

playbooks/generate_certs.yml

@@ -277,3 +277,89 @@
                     owner: "{{ lookup('env', 'USER') }}"
                     group: "{{ lookup('env', 'USER') }}"
                     mode: "0644"
+
+        - name: "Create Nomad certificates"
+          when:
+            - "('nomad_servers' in group_names) or ('nomad_clients' in group_names)"
+          vars:
+            nomad_private_key_path: "{{ sub_configuration_directories['certificates'] }}/nomad/{{ inventory_hostname }}/key.pem"
+            nomad_certificate_path: "{{ sub_configuration_directories['certificates'] }}/nomad/{{ inventory_hostname }}/cert.pem"
+          block:
+            - name: "Create temporary cert directory in {{ sub_configuration_directories['certificates'] }}"  # noqa: run-once[task]
+              ansible.builtin.file:
+                path: "{{ sub_configuration_directories['certificates'] }}/nomad/{{ inventory_hostname }}"
+                state: directory
+                owner: "{{ lookup('env', 'USER') }}"
+                group: "{{ lookup('env', 'USER') }}"
+                mode: "0755"
+
+            - name: "Create Nomad certificate keys"
+              community.crypto.openssl_privatekey:
+                path: "{{ nomad_private_key_path }}"
+                owner: "{{ lookup('env', 'USER') }}"
+                group: "{{ lookup('env', 'USER') }}"
+
+            - name: "Create CSRs for Nomad servers"
+              vars:
+                nomad_csr_sans: >-
+                  {%- set sans_list = [
+                      'DNS:' + inventory_hostname,
+                      'DNS:localhost',
+                      'IP:' + api_interface_address,
+                      'IP:127.0.0.1'
+                  ] -%}
+                  {%- if hashicorp_nomad_configuration.server.enabled -%}
+                    {%- set _ = sans_list.append('DNS:server.' ~ hashicorp_nomad_configuration.region ~ '.nomad') -%}
+                    {%- if  (enable_consul | bool) -%}
+                      {%- set _ = sans_list.append('DNS:nomad.service.consul') -%}
+                    {%- endif -%}
+                  {%- endif -%}
+                  {%- if hashicorp_nomad_configuration.client.enabled -%}
+                    {%- set _ = sans_list.append('DNS:client.' ~ hashicorp_nomad_configuration.region ~ '.nomad') -%}
+                  {%- endif -%}
+                  {{ sans_list }}
+              community.crypto.openssl_csr_pipe:
+                privatekey_path: "{{ nomad_private_key_path }}"
+                common_name: "{{ inventory_hostname }}"
+                subject_alt_name: "{{ nomad_csr_sans }}"
+                key_usage_critical: true
+                key_usage:
+                  - Digital Signature
+                  - Key Encipherment
+                extended_key_usage:
+                  - TLS Web Server Authentication
+                  - TLS Web Client Authentication
+                organization_name: EDNZ Cloud
+                use_common_name_for_san: false
+              register: nomad_csr
+
+            - name: "Sign certificates with internal CA"
+              community.crypto.x509_certificate:
+                path: "{{ nomad_certificate_path }}"
+                csr_content: "{{ nomad_csr.csr }}"
+                provider: ownca
+                ownca_path: "{{ hashistack_ca_cert_path }}"
+                ownca_privatekey_path: "{{ hashistack_ca_key_path }}"
+                ownca_not_after: "+365d"
+                ownca_not_before: "-1d"
+
+            - name: "Concatenate CA and Child certificates"
+              block:
+                - name: "Read content of ca.crt"
+                  ansible.builtin.slurp:
+                    src: "{{ hashistack_ca_cert_path }}"
+                  register: ca_crt_content
+
+                - name: "Read content of cert.pem"
+                  ansible.builtin.slurp:
+                    src: "{{ nomad_certificate_path }}"
+                  register: cert_pem_content
+
+                - name: "Concatenate certificates"
+                  ansible.builtin.copy:
+                    content: |
+                      {{ cert_pem_content['content'] | b64decode }}{{ ca_crt_content['content'] | b64decode }}
+                    dest: "{{ nomad_certificate_path }}"
+                    owner: "{{ lookup('env', 'USER') }}"
+                    group: "{{ lookup('env', 'USER') }}"
+                    mode: "0644"

playbooks/group_vars/all/globals.yml

@@ -111,9 +111,9 @@ consul_tls_configuration:
 
 extra_consul_container_volumes: []
 
-#######################
+##############################
-# extra configuration #
+# consul extra configuration #
-#######################
+##############################
 
 consul_extra_configuration: {}
 consul_extra_files_list: []
@@ -201,9 +201,71 @@ vault_logging_configuration:
 
 extra_vault_container_volumes: []
 
-#####################
+#############################
-# extra configuration
+# vault extra configuration #
-#####################
+#############################
 
 vault_extra_configuration: {}
 vault_extra_files_list: []
+
+#####################################################
+#                                                   #
+#                Nomad                              #
+#                                                   #
+#####################################################
+
+nomad_datacenter: dc1
+nomad_region: global
+
+###########################
+# nomad ACL configuration #
+###########################
+
+nomad_acl_configuration:
+  enabled: true
+  token_ttl: 30s
+  policy_ttl: 60s
+  role_ttl: 60s
+
+############################
+# nomad consul integration #
+############################
+
+nomad_enable_consul_integration: "{{ enable_consul | bool }}"
+nomad_consul_integration_configuration:
+  address: "127.0.0.1:{{ hashicorp_consul_configuration.ports.https if consul_enable_tls else hashicorp_consul_configuration.ports.http }}"
+  auto_advertise: true
+  ssl: "{{ consul_enable_tls | bool }}"
+  token: "{{ _credentials.consul.tokens.nomad.server.secret_id if nomad_enable_server else _credentials.consul.tokens.nomad.client.secret_id}}"
+  tags: []
+
+############################
+# nomad vault integration #
+############################
+
+nomad_enable_vault_integration: false
+nomad_vault_integration_configuration: {}
+
+###############################
+# nomad drivers configuration #
+###############################
+
+######################
+# nomad internal tls #
+######################
+
+nomad_enable_tls: false
+nomad_tls_configuration:
+  http: true
+  rpc: true
+  ca_file: "/etc/ssl/certs/ca-certificates.crt"
+  cert_file: "{{ nomad_certificates_directory }}/cert.pem"
+  key_file: "{{ nomad_certificates_directory }}/key.pem"
+  verify_server_hostname: true
+
+#############################
+# nomad extra configuration #
+#############################
+
+nomad_extra_configuration: {}
+nomad_extra_files_list: []

playbooks/group_vars/all/nomad.yml

@@ -5,8 +5,6 @@
 #                                                   #
 #####################################################
 
-nomad_datacenter: dc1
-
 ####################
 # nomad api config #
 ####################
@@ -62,16 +60,6 @@ nomad_address_configuration:
     rpc: 4647
     serf: 4648
 
-###########################
-# nomad ACL configuration #
-###########################
-
-nomad_acl_configuration:
-  enabled: true
-  token_ttl: 30s
-  policy_ttl: 60s
-  role_ttl: 60s
-
 #################################
 # nomad autopilot configuration #
 #################################
@@ -82,14 +70,6 @@ nomad_autopilot_configuration: {}
 # nomad consul integration #
 ############################
 
-nomad_enable_consul_integration: "{{ enable_consul | bool }}"
-nomad_consul_integration_configuration:
-  address: "127.0.0.1:{{ hashicorp_consul_configuration.ports.https if consul_enable_tls else hashicorp_consul_configuration.ports.http }}"
-  auto_advertise: true
-  ssl: "{{ consul_enable_tls | bool }}"
-  token: "{{ _credentials.consul.tokens.nomad.server.secret_id if nomad_enable_server else _credentials.consul.tokens.nomad.client.secret_id}}"
-  tags: []
-
 nomad_consul_integration_tls_configuration:
   ca_file: "/etc/ssl/certs/ca-certificates.crt"
 
@@ -127,13 +107,6 @@ nomad_consul_integration_client_policy: |
     policy = "write"
   }
 
-############################
-# nomad vault integration #
-############################
-
-nomad_enable_vault_integration: false
-nomad_vault_integration_configuration: {}
-
 #############################
 # nomad leave configuration #
 #############################
@@ -179,27 +152,11 @@ nomad_client_configuration:
 # nomad internal tls #
 ######################
 
-nomad_enable_tls: false
-nomad_tls_configuration:
-  http: true
-  rpc: true
-  ca_file: "/etc/ssl/certs/ca-certificates.crt"
-  cert_file: "{{ nomad_certificates_directory }}/cert.pem"
-  key_file: "{{ nomad_certificates_directory }}/key.pem"
-  verify_server_hostname: true
-
 nomad_certificates_directory: "{{ hashicorp_nomad_config_dir }}/tls"
 nomad_certificates_extra_files_dir:
   - src: "{{ sub_configuration_directories['certificates'] }}/nomad/{{ inventory_hostname }}"
     dest: "{{ nomad_certificates_directory }}"
 
-#######################
-# extra configuration #
-#######################
-
-nomad_extra_configuration: {}
-nomad_extra_files_list: []
-
 ########################
 # nomad role variables #
 ########################
@@ -213,7 +170,7 @@ hashicorp_nomad_version: "{{ nomad_version }}"
 hashicorp_nomad_env_variables: {}
 hashicorp_nomad_config_dir: "/etc/nomad.d"
 hashicorp_nomad_data_dir: /opt/nomad
-hashicorp_nomad_extra_files: false
+hashicorp_nomad_extra_files: true
 hashicorp_nomad_extra_files_list: "{{ ([] +
   (nomad_certificates_extra_files_dir if nomad_enable_tls else []) +
   nomad_extra_files_list)
@@ -222,6 +179,7 @@ hashicorp_nomad_extra_files_list: "{{ ([] +
   }}"
 hashicorp_nomad_configuration:
   datacenter: "{{ nomad_datacenter }}"
+  region: "{{ nomad_region }}"
   bind_addr: "0.0.0.0"
   data_dir: "{{ hashicorp_nomad_data_dir }}"
   leave_on_interrupt: "{{ nomad_leave_on_interrupt }}"

playbooks/tasks/consul/consul_deploy.yml

@@ -31,7 +31,6 @@
 
     - name: "Create consul agents token"
       when:
-        # - _consul_init_secret.changed
         - consul_acl_configuration.enabled
       block:
         - name: "Create consul agents token" # noqa: run-once[task] no-handler

playbooks/tasks/misc/load_ca_certificates.yml

@@ -43,7 +43,7 @@
       loop: "{{ _hashistack_copied_ca.results }}"
       register: _hashistack_usr_local_share_ca_certificates
 
-    - name: "Update the trust store"
+    - name: "Update the trust store" # noqa: no-handler
       ansible.builtin.command: update-ca-certificates
       changed_when: false
       when: _hashistack_usr_local_share_ca_certificates.changed

playbooks/tasks/nomad/nomad_deploy.yml

@@ -76,6 +76,7 @@
       ednz_cloud.hashistack.nomad_acl_bootstrap:
         bootstrap_secret: "{{ _credentials.nomad.root_token.secret_id }}"
         api_url: "{{ nomad_api_addr }}"
+        tls_verify: false
       run_once: true
       delegate_to: "{{ groups['nomad_servers'] | first }}"
       register: _nomad_init_secret

plugins/modules/nomad_acl_bootstrap.py

@@ -91,18 +91,25 @@ def bootstrap_nomad_acl(
     if bootstrap_secret:
         payload["BootstrapSecret"] = bootstrap_secret
 
+    response = None
+
     try:
-        response = requests.put(
+        response = requests.post(
             f"{api_url}/v1/acl/bootstrap", json=payload, verify=tls_verify
         )
         response.raise_for_status()
         return True, response.json()
+    except requests.exceptions.HTTPError as e:
+        if response is not None and response.status_code == 400:
+            try:
+                error_message = response.json().get(
+                    "Errors", ["Nomad ACL bootstrap already done"]
+                )[0]
+            except ValueError:
+                error_message = response.text
+            return False, {"message": error_message}
+        raise ValueError(f"Nomad ACL bootstrap failed: {str(e)}")
     except requests.exceptions.RequestException as e:
-        if (
-            response.status_code == 400
-            and "ACL bootstrap already done" in response.text
-        ):
-            return False, {"message": "Nomad ACL system is already bootstrapped"}
         raise ValueError(f"Nomad ACL bootstrap failed: {str(e)}")