From bd2028c3aff7316269790f4255e63da925028224 Mon Sep 17 00:00:00 2001
From: Bertrand Lanson <bertrand.lanson@protonmail.com>
Date: Sun, 5 May 2024 02:37:29 +0200
Subject: [PATCH 1/3] feat(consul): allow enabling consul internal TLS

---
 .gitignore                               |  1 +
 playbooks/generate_certs.yml             |  2 +-
 playbooks/group_vars/all/consul.yml      | 38 ++++++++++++++++++++----
 playbooks/group_vars/all/globals.yml     | 15 +++++++---
 playbooks/group_vars/all/vault.yml       |  4 +--
 playbooks/tasks/consul/consul_deploy.yml | 15 +++++-----
 roles/hashicorp_consul                   |  2 +-
 7 files changed, 55 insertions(+), 22 deletions(-)

diff --git a/.gitignore b/.gitignore
index 02e5a4e..cb534dd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,5 +6,6 @@ vault_config.yml
 consul_config.yml
 **/certificates/**
 **/secrets/credentials.yml
+**/secrets/credentials.decrypt.yml
 **/secrets/vault.yml
 **/.ansible-vault
diff --git a/playbooks/generate_certs.yml b/playbooks/generate_certs.yml
index dc2913c..14e975d 100644
--- a/playbooks/generate_certs.yml
+++ b/playbooks/generate_certs.yml
@@ -192,7 +192,7 @@
 
         - name: "Create Consul certificates"
           when:
-            - "'consul_servers' in group_names"
+            - "('consul_servers' in group_names) or ('consul_agents' in group_names)"
           vars:
             consul_private_key_path: "{{ sub_configuration_directories['certificates'] }}/consul/{{ inventory_hostname }}/key.pem"
             consul_certificate_path: "{{ sub_configuration_directories['certificates'] }}/consul/{{ inventory_hostname }}/cert.pem"
diff --git a/playbooks/group_vars/all/consul.yml b/playbooks/group_vars/all/consul.yml
index 1914609..f65b51d 100644
--- a/playbooks/group_vars/all/consul.yml
+++ b/playbooks/group_vars/all/consul.yml
@@ -5,6 +5,16 @@
 #                                                   #
 #####################################################
 
+#####################
+# consul api config #
+#####################
+
+consul_api_addr: "{{ consul_api_scheme }}://{{ api_interface_address }}:{{ consul_api_port[consul_api_scheme] }}"
+consul_api_scheme: "{{ 'https' if consul_enable_tls else 'http' }}"
+consul_api_port:
+  http: 8500
+  https: 8501
+
 ##########################
 # consul haproxy backend #
 ##########################
@@ -27,7 +37,7 @@ consul_external_backend_options:
 consul_external_backend_servers: |
   [
     {% for host in groups['consul_servers'] %}
-      'server consul-{{ hostvars[host].api_interface_address }} {{ hostvars[host].api_interface_address }}:{{ hashi_consul_configuration.ports.http }} check inter 5s'{% if not loop.last %},{% endif %}
+      'server consul-{{ hostvars[host].api_interface_address }} {{ hostvars[host].api_interface_address }}:{{ hostvars[host].consul_api_port[consul_api_scheme] }} check {{ 'ssl verify none ' if consul_enable_tls }}inter 5s'{% if not loop.last %},{% endif %}
     {% endfor %}
   ]
 
@@ -43,6 +53,15 @@ consul_default_agent_policy: |
     policy = "read"
   }
 
+#######################
+# consul internal tls #
+#######################
+
+consul_certificates_directory: "{{ hashi_consul_config_dir }}/tls"
+consul_certificates_extra_files_dir:
+  - src: "{{ sub_configuration_directories['certificates'] }}/consul/{{ inventory_hostname }}"
+    dest: "{{ consul_certificates_directory }}"
+
 #########################
 # consul role variables #
 #########################
@@ -53,9 +72,15 @@ hashi_consul_deploy_method: "{{ deployment_method }}"
 hashi_consul_env_variables: {}
 hashi_consul_config_dir: "/etc/consul.d"
 hashi_consul_data_dir: "/opt/consul"
-hashi_consul_extra_files: false
-hashi_consul_extra_files_src: "{{ sub_configuration_directories.consul_servers }}/config"
-hashi_consul_extra_files_dst: "{{ hashi_consul_config_dir }}/config"
+hashi_consul_extra_files: true
+hashi_consul_extra_files_list: "{{ ([] +
+  (consul_certificates_extra_files_dir if consul_enable_tls else []) +
+  (vault_plugin_extra_files_dir if vault_enable_plugins else []) +
+  vault_extra_files_list)
+  | unique
+  | sort
+  }}"
+hashi_consul_extra_container_volumes: "{{ default_container_extra_volumes | union(extra_consul_container_volumes) | unique }}"
 hashi_consul_envoy_install: false
 hashi_consul_envoy_version: v1.27.2
 hashi_consul_configuration:
@@ -83,8 +108,6 @@ hashi_consul_configuration:
   dns_config: "{{ consul_dns_configuration }}"
   ports:
     dns: 8600
-    http: 8500 # "{{ ('8500'|int) if not  }}"
-    https: -1
     grpc: 8502
     grpc_tls: 8503
     server: 8300
@@ -98,3 +121,6 @@ hashi_consul_configuration:
 # this is used to circumvent jinja limitation to convert string to integer
 hashi_consul_configuration_string: |
   bootstrap_expect: {{ (groups['consul_servers'] | length) }}
+  ports:
+    http: {{ (consul_api_port.http|int) if not consul_enable_tls else ('-1' | int) }}
+    https: {{ (consul_api_port.https|int) if consul_enable_tls else ('-1' | int) }}
diff --git a/playbooks/group_vars/all/globals.yml b/playbooks/group_vars/all/globals.yml
index 7b95713..6f7a9d4 100644
--- a/playbooks/group_vars/all/globals.yml
+++ b/playbooks/group_vars/all/globals.yml
@@ -100,22 +100,29 @@ consul_mesh_configuration:
 # consul tls configuration #
 ############################
 
-consul_enable_tls: false
+consul_enable_tls: true
 consul_tls_configuration:
-  default:
+  defaults:
     ca_file: "/etc/ssl/certs/ca-certificates.crt"
-    cert_file: "{{ hashi_consul_config_dir }}/tls/cert.pem"
-    key_file: "{{ hashi_consul_config_dir }}/tls/key.pem"
+    cert_file: "{{ consul_certificates_directory }}/cert.pem"
+    key_file: "{{ consul_certificates_directory }}/key.pem"
     verify_incoming: false
     verify_outgoing: true
   internal_rpc:
     verify_server_hostname: true
 
+############################
+# consul container volumes #
+############################
+
+extra_consul_container_volumes: []
+
 #######################
 # extra configuration #
 #######################
 
 consul_extra_configuration: {}
+consul_extra_files_list: []
 
 #####################################################
 #                                                   #
diff --git a/playbooks/group_vars/all/vault.yml b/playbooks/group_vars/all/vault.yml
index 01f2ca4..35bbb7d 100644
--- a/playbooks/group_vars/all/vault.yml
+++ b/playbooks/group_vars/all/vault.yml
@@ -38,7 +38,7 @@ vault_external_backend_servers: |
 vault_certificates_directory: "{{ hashi_vault_config_dir }}/tls"
 vault_certificates_extra_files_dir:
   - src: "{{ sub_configuration_directories['certificates'] }}/vault/{{ inventory_hostname }}"
-    dest: "{{ hashi_vault_config_dir }}/tls"
+    dest: "{{ vault_certificates_directory }}"
 
 #################
 # vault plugins #
@@ -47,7 +47,7 @@ vault_certificates_extra_files_dir:
 vault_plugin_directory: "{{ hashi_vault_config_dir }}/plugin"
 vault_plugin_extra_files_dir:
   - src: "{{ sub_configuration_directories['vault_servers'] }}/plugin"
-    dest: "{{ hashi_vault_config_dir }}/plugin"
+    dest: "{{ vault_plugin_directory }}"
 
 ########################
 # vault role variables #
diff --git a/playbooks/tasks/consul/consul_deploy.yml b/playbooks/tasks/consul/consul_deploy.yml
index 7a890e4..5552f26 100644
--- a/playbooks/tasks/consul/consul_deploy.yml
+++ b/playbooks/tasks/consul/consul_deploy.yml
@@ -7,7 +7,7 @@
 
     - name: "Wait for consul cluster to initialize" # noqa: run-once[task]
       ansible.builtin.uri:
-        url: "http://{{ api_interface_address }}:8500" # TODO: this should be dynamic (http/https)
+        url: "{{ consul_api_addr }}"
         validate_certs: no
         return_content: yes
         status_code:
@@ -20,9 +20,9 @@
     - name: "Initialize consul cluster" # noqa: run-once[task]
       community.general.consul_acl_bootstrap:
         bootstrap_secret: "{{ _credentials.consul.root_token.secret_id }}"
-        host: "{{ hashi_consul_configuration['advertise_addr'] }}"
-        port: 8500
-        scheme: http
+        host: "{{ api_interface_address }}"
+        port: "{{ consul_api_port[consul_api_scheme] }}"
+        scheme: "{{ consul_api_scheme }}"
         state: present
       run_once: true
       delegate_to: "{{ groups['consul_servers'] | first }}"
@@ -63,8 +63,8 @@
               community.general.consul_policy:
                 token: "{{ _credentials.consul.root_token.secret_id }}"
                 host: "{{ api_interface_address }}"
-                port: 8500
-                scheme: http # TODO: this should be dynamic
+                port: "{{ consul_api_port[consul_api_scheme] }}"
+                scheme: "{{ consul_api_scheme }}"
                 validate_certs: false
                 state: present
                 name: agents-policy
@@ -73,14 +73,13 @@
 
             - name: "Create consul agents token"
               vars:
-                consul_full_url: "http://{{ hashi_consul_configuration['advertise_addr'] }}:8500"
                 consul_token_body:
                   Description: "Consul agents token"
                   SecretID: "{{ _credentials.consul.tokens.agent }}"
                   Policies:
                     - ID: "{{ _consul_agent_policy.policy.ID }}"
               ansible.builtin.uri:
-                url: "{{ consul_full_url }}/v1/acl/token"
+                url: "{{ consul_api_addr }}/v1/acl/token"
                 method: PUT
                 headers:
                   X-Consul-Token: "{{ _credentials.consul.root_token.secret_id  }}"
diff --git a/roles/hashicorp_consul b/roles/hashicorp_consul
index c6fbfe5..56696c3 160000
--- a/roles/hashicorp_consul
+++ b/roles/hashicorp_consul
@@ -1 +1 @@
-Subproject commit c6fbfe5b78ef8b8884af129ea84afb26da754833
+Subproject commit 56696c3552308225d4e5b91efc8e4bf75d31d2f3
-- 
2.45.2


From c53e6905df1631baa923f01c83f63ccfb3547e32 Mon Sep 17 00:00:00 2001
From: Bertrand Lanson <bertrand.lanson@protonmail.com>
Date: Sun, 5 May 2024 02:47:49 +0200
Subject: [PATCH 2/3] fix(vault/consul): ensure idempotence of extra_volumes
 list to avoid restarting on each run due to slightly different service files

---
 playbooks/group_vars/all/consul.yml | 2 +-
 playbooks/group_vars/all/vault.yml  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/playbooks/group_vars/all/consul.yml b/playbooks/group_vars/all/consul.yml
index f65b51d..4caab26 100644
--- a/playbooks/group_vars/all/consul.yml
+++ b/playbooks/group_vars/all/consul.yml
@@ -80,7 +80,7 @@ hashi_consul_extra_files_list: "{{ ([] +
   | unique
   | sort
   }}"
-hashi_consul_extra_container_volumes: "{{ default_container_extra_volumes | union(extra_consul_container_volumes) | unique }}"
+hashi_consul_extra_container_volumes: "{{ default_container_extra_volumes | union(extra_consul_container_volumes) | unique | sort }}"
 hashi_consul_envoy_install: false
 hashi_consul_envoy_version: v1.27.2
 hashi_consul_configuration:
diff --git a/playbooks/group_vars/all/vault.yml b/playbooks/group_vars/all/vault.yml
index 35bbb7d..a92cc1e 100644
--- a/playbooks/group_vars/all/vault.yml
+++ b/playbooks/group_vars/all/vault.yml
@@ -69,7 +69,7 @@ hashi_vault_extra_files_list: "{{ ([] +
   }}"
 hashi_vault_extra_files_src: "{{ sub_configuration_directories.vault_servers }}/config"
 hashi_vault_extra_files_dst: "{{ hashi_vault_config_dir }}/config"
-hashi_vault_extra_container_volumes: "{{ default_container_extra_volumes | union(extra_vault_container_volumes) | unique }}"
+hashi_vault_extra_container_volumes: "{{ default_container_extra_volumes | union(extra_vault_container_volumes) | unique | sort }}"
 hashi_vault_configuration:
   cluster_name: "{{ vault_cluster_name }}"
   cluster_addr: "{{ 'https' if vault_enable_tls else 'http'}}://{{ api_interface_address }}:8201"
-- 
2.45.2


From 2480a387c3e0c6c074301be88c85ef5faa376c4f Mon Sep 17 00:00:00 2001
From: Bertrand Lanson <bertrand.lanson@protonmail.com>
Date: Sun, 5 May 2024 19:41:29 +0200
Subject: [PATCH 3/3] fix(globals): restore default globals.yml file, move
 changes to test directory

---
 .../etc/hashistack/globals.yml                | 212 ++++++++++++++++++
 playbooks/group_vars/all/globals.yml          |  23 +-
 2 files changed, 222 insertions(+), 13 deletions(-)

diff --git a/molecule/no_tls_multi_node/etc/hashistack/globals.yml b/molecule/no_tls_multi_node/etc/hashistack/globals.yml
index e69de29..e41c895 100644
--- a/molecule/no_tls_multi_node/etc/hashistack/globals.yml
+++ b/molecule/no_tls_multi_node/etc/hashistack/globals.yml
@@ -0,0 +1,212 @@
+---
+##########################
+# General options ########
+##########################
+
+# enable_haproxy: "yes"
+# enable_vault: "yes"
+# enable_consul: "yes"
+# enable_nomad: "no"
+
+# haproxy_version: "2.8"
+# nomad_version: "1.7.7"
+# consul_version: "1.18.1"
+# vault_version: "1.16.2"
+
+# deployment_method: "docker"
+
+# consul_fqdn: consul.ednz.lab
+# vault_fqdn: vault.ednz.lab
+# nomad_fqdn: nomad.ednz.lab
+
+# hashistack_external_vip_interface: "eth0"
+# hashistack_external_vip_addr: "192.168.121.100"
+# hashistack_internal_vip_interface: "{{ hashistack_external_vip_interface }}"
+# hashistack_internal_vip_addr: "{{ hashistack_external_vip_addr }}"
+
+api_interface: "eth1"
+# api_interface_address: "{{ ansible_facts[api_interface]['ipv4']['address'] }}"
+
+########################
+# external tls options #
+########################
+
+enable_tls_external: true
+# external_tls_externally_managed_certs: false
+
+#####################################################
+#                                                   #
+#                Consul                             #
+#                                                   #
+#####################################################
+
+# consul_domain: consul
+# consul_datacenter: dc1
+# consul_primary_datacenter: dc1
+# consul_leave_on_terminate: true
+# consul_rejoin_after_leave: true
+# consul_enable_script_checks: true
+# consul_gossip_encryption_key: "{{ 'mysupersecretgossipencryptionkey'|b64encode }}"
+
+################################
+# consul address configuration #
+################################
+
+# consul_address_configuration:
+#   # The address to which Consul will bind client interfaces,
+#   # including the HTTP and DNS servers.
+#   client_addr: "0.0.0.0"
+#   # The address that should be bound to for internal cluster communications.
+#   bind_addr: "{{ api_interface_address }}"
+#   # The advertise address is used to change the address that we advertise to other nodes in the cluster.
+#   advertise_addr: "{{ api_interface_address }}"
+
+############################
+# consul ACL configuration #
+############################
+
+# consul_acl_configuration:
+#   enabled: true
+#   default_policy: "deny" # can be allow or deny
+#   enable_token_persistence: true
+
+############################
+# consul DNS configuration #
+############################
+
+# consul_dns_configuration:
+#   allow_stale: true
+#   enable_truncate: true
+#   only_passing: true
+
+###########################
+# consul ui configuration #
+###########################
+
+# consul_ui_configuration:
+#   enabled: "{{ 'consul_servers' in group_names }}"
+
+#####################################
+# consul service mesh configuration #
+#####################################
+
+# consul_mesh_configuration:
+#   enabled: true
+
+############################
+# consul tls configuration #
+############################
+
+consul_enable_tls: true
+# consul_tls_configuration:
+#   defaults:
+#     ca_file: "/etc/ssl/certs/ca-certificates.crt"
+#     cert_file: "{{ consul_certificates_directory }}/cert.pem"
+#     key_file: "{{ consul_certificates_directory }}/key.pem"
+#     verify_incoming: false
+#     verify_outgoing: true
+#   internal_rpc:
+#     verify_server_hostname: true
+
+############################
+# consul container volumes #
+############################
+
+# extra_consul_container_volumes: []
+
+#######################
+# extra configuration #
+#######################
+
+# consul_extra_configuration: {}
+# consul_extra_files_list: []
+
+#####################################################
+#                                                   #
+#                Vault                              #
+#                                                   #
+#####################################################
+
+# vault_cluster_name: vault
+# vault_enable_ui: true
+# vault_seal_configuration:
+#   key_shares: 3
+#   key_threshold: 2
+
+#################
+# vault storage #
+#################
+
+# vault_storage_configuration:
+#   raft:
+#     path: "{{ hashi_vault_data_dir }}/data"
+#     node_id: "{{ ansible_hostname }}"
+#     retry_join: |
+#       [
+#       {% for host in groups['vault_servers'] %}
+#         {
+#           'leader_api_addr': '{{ "https" if vault_enable_tls else "http"}}://{{ hostvars[host].api_interface_address }}:8200'
+#         }{% if not loop.last %},{% endif %}
+#       {% endfor %}
+#       ]
+
+##################
+# vault listener #
+##################
+
+vault_enable_tls: true
+# vault_tls_verify: false
+# vault_listener_configuration:
+#   tcp:
+#     address: "0.0.0.0:8200"
+#     tls_disable: true
+
+# vault_tls_listener_configuration:
+#   tcp:
+#     tls_disable: false
+#     tls_cert_file: "{{ vault_certificates_directory }}/cert.pem"
+#     tls_key_file: "{{ vault_certificates_directory }}/key.pem"
+#     tls_disable_client_certs: true
+
+# vault_extra_listener_configuration: {}
+
+########################
+# service registration #
+########################
+
+# vault_enable_service_registration: false
+# vault_service_registration_configuration:
+#   consul:
+#     address: "127.0.0.1:8500"
+#     scheme: "http"
+#     token: ""
+
+#################
+# vault plugins #
+#################
+
+# vault_enable_plugins: false
+
+###########
+# logging #
+###########
+
+# vault_enable_log_to_file: false
+# vault_logging_configuration:
+#   log_level: info
+#   log_format: standard
+#   log_rotate_duration: 24h
+#   log_rotate_max_files: 30
+
+###########################
+# vault container volumes #
+###########################
+
+# extra_vault_container_volumes: []
+
+#####################
+# extra configuration
+#####################
+
+# vault_extra_configuration: {}
+# vault_extra_files_list: []
diff --git a/playbooks/group_vars/all/globals.yml b/playbooks/group_vars/all/globals.yml
index 6f7a9d4..941414e 100644
--- a/playbooks/group_vars/all/globals.yml
+++ b/playbooks/group_vars/all/globals.yml
@@ -6,12 +6,12 @@
 enable_haproxy: "yes"
 enable_vault: "yes"
 enable_consul: "yes"
-enable_nomad: "no"
+enable_nomad: "yes"
 
 haproxy_version: "2.8"
-nomad_version: "latest"
-consul_version: "latest"
-vault_version: "1.15.6"
+nomad_version: "1.7.7"
+consul_version: "1.18.1"
+vault_version: "1.16.2"
 
 deployment_method: "docker"
 
@@ -21,20 +21,17 @@ nomad_fqdn: nomad.ednz.lab
 
 hashistack_external_vip_interface: "eth0"
 hashistack_external_vip_addr: "192.168.121.100"
-hashistack_internal_vip_interface: "eth1"
-# hashistack_internal_vip_interface: "{{ hashistack_external_vip_interface }}"
-hashistack_internal_vip_addr: "192.168.100.100"
-# hashistack_internal_vip_addr: "{{ hashistack_external_vip_addr }}"
+hashistack_internal_vip_interface: "{{ hashistack_external_vip_interface }}"
+hashistack_internal_vip_addr: "{{ hashistack_external_vip_addr }}"
 
-# api_interface: "eth0"
-api_interface: "eth1"
+api_interface: "eth0"
 api_interface_address: "{{ ansible_facts[api_interface]['ipv4']['address'] }}"
 
 ########################
 # external tls options #
 ########################
 
-enable_tls_external: true
+enable_tls_external: false
 external_tls_externally_managed_certs: false
 
 #####################################################
@@ -100,7 +97,7 @@ consul_mesh_configuration:
 # consul tls configuration #
 ############################
 
-consul_enable_tls: true
+consul_enable_tls: false
 consul_tls_configuration:
   defaults:
     ca_file: "/etc/ssl/certs/ca-certificates.crt"
@@ -157,7 +154,7 @@ vault_storage_configuration:
 # vault listener #
 ##################
 
-vault_enable_tls: true
+vault_enable_tls: false
 vault_tls_verify: false
 vault_listener_configuration:
   tcp:
-- 
2.45.2