From 267a23317f7cfaf699462ff7041d744fbe345e37 Mon Sep 17 00:00:00 2001
From: Bertrand Lanson <bertrand.lanson@protonmail.com>
Date: Sun, 17 Mar 2024 19:05:02 +0100
Subject: [PATCH] feat(consul): start configuring and deploying agents

---
 molecule/no_tls_multi_node/molecule.yml       |   2 +
 playbooks/deploy.yml                          |  11 +-
 playbooks/group_vars/all/all.yml              |  10 +-
 playbooks/group_vars/all/consul.yml           | 154 +++++++++---------
 playbooks/group_vars/all/haproxy.yml          |   8 +-
 playbooks/group_vars/all/vault.yml            |   2 +-
 playbooks/inventory/multinode.ini             |   3 +
 playbooks/tasks/consul/consul_deploy.yml      |   9 +-
 playbooks/tasks/consul/consul_vars.yml        |  16 +-
 .../files/keepalived/keepalived.conf.j2       |  50 ++++++
 .../files/keepalived/scripts.d/chk_haproxy.sh |  15 ++
 playbooks/tasks/load_vars.yml                 |   2 +-
 12 files changed, 185 insertions(+), 97 deletions(-)
 create mode 100644 playbooks/tasks/haproxy/files/keepalived/keepalived.conf.j2
 create mode 100644 playbooks/tasks/haproxy/files/keepalived/scripts.d/chk_haproxy.sh

diff --git a/molecule/no_tls_multi_node/molecule.yml b/molecule/no_tls_multi_node/molecule.yml
index b43b37b..5aa16bb 100644
--- a/molecule/no_tls_multi_node/molecule.yml
+++ b/molecule/no_tls_multi_node/molecule.yml
@@ -20,6 +20,7 @@ platforms:
     groups:
       - common
       - haproxy_servers
+      - consul_agents
   - name: proxy02
     box: generic/${MOLECULE_TEST_OS}
     cpus: 2
@@ -32,6 +33,7 @@ platforms:
     groups:
       - common
       - haproxy_servers
+      - consul_agents
   - name: hashistack01
     box: generic/${MOLECULE_TEST_OS}
     cpus: 4
diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml
index b45af06..d763349 100644
--- a/playbooks/deploy.yml
+++ b/playbooks/deploy.yml
@@ -19,7 +19,7 @@
       tags:
         - haproxy
 
-    - name: "Deploy Consul"
+    - name: "Deploy Consul Control Plane"
       ansible.builtin.import_tasks:
         file: tasks/consul/consul_deploy.yml
       when:
@@ -28,6 +28,15 @@
       tags:
         - consul
 
+    - name: "Deploy Consul Agents"
+      ansible.builtin.include_role:
+        name: ednz_cloud.hashistack.hashicorp_consul
+      when:
+        - enable_consul | bool
+        - "'consul_agents' in group_names"
+      tags:
+        - consul
+
     - name: "Deploy Vault"
       ansible.builtin.import_tasks:
         file: tasks/vault/vault_deploy.yml
diff --git a/playbooks/group_vars/all/all.yml b/playbooks/group_vars/all/all.yml
index 06beae1..1a01190 100644
--- a/playbooks/group_vars/all/all.yml
+++ b/playbooks/group_vars/all/all.yml
@@ -14,6 +14,10 @@ vault_version: latest
 
 deployment_method: "docker"
 
+consul_fqdn: consul.ednz.lab
+vault_fqdn: vault.ednz.lab
+nomad_fqdn: nomad.ednz.lab
+
 hashistack_external_vip_interface: "eth0"
 hashistack_external_vip_addr: "192.168.121.100"
 hashistack_internal_vip_interface: "eth1"
@@ -32,15 +36,15 @@ api_interface_address: "{{ ansible_facts[api_interface]['ipv4']['address'] }}"
 # manage_pip_packages_allow_break_system_packages: true
 
 vault_versions:
-  host: "{{ vault_version }}{% '*' if vault_version != 'latest' %}"
+  host: "{{ vault_version if vault_version != 'latest' else vault_version + '*' }}"
   docker: "{{ vault_version }}"
 
 consul_versions:
-  host: "{{ consul_version }}{% '*' if consul_version != 'latest' %}"
+  host: "{{ consul_version if consul_version != 'latest' else consul_version + '*' }}"
   docker: "{{ consul_version }}"
 
 nomad_versions:
-  host: "{{ nomad_version }}{% '*' if nomad_version != 'latest' %}"
+  host: "{{ nomad_version if nomad_version != 'latest' else nomad_version + '*' }}"
   docker: "{{ nomad_version }}"
 
 configuration_directory: "{{ lookup('env', 'PWD') }}/etc/hashistack"
diff --git a/playbooks/group_vars/all/consul.yml b/playbooks/group_vars/all/consul.yml
index b40c47f..3b60816 100644
--- a/playbooks/group_vars/all/consul.yml
+++ b/playbooks/group_vars/all/consul.yml
@@ -1,7 +1,7 @@
 ---
 #####################################################
 #                                                   #
-#                Consul Configuration               #
+#                Editable                           #
 #                                                   #
 #####################################################
 
@@ -11,41 +11,82 @@ consul_primary_datacenter: dc1
 consul_leave_on_terminate: true
 consul_rejoin_after_leave: true
 consul_enable_script_checks: true
+consul_gossip_encryption_key: "{{ 'mysupersecretgossipencryptionkey'|b64encode }}"
+
+################################
+# consul address configuration #
+################################
+
+consul_address_configuration:
+  # The address to which Consul will bind client interfaces,
+  # including the HTTP and DNS servers.
+  client_addr: "0.0.0.0"
+  # The address that should be bound to for internal cluster communications.
+  bind_addr: "{{ api_interface_address }}"
+  # The advertise address is used to change the address that we advertise to other nodes in the cluster.
+  advertise_addr: "{{ api_interface_address }}"
+
+############################
+# consul ACL configuration #
+############################
+
+consul_acl_configuration:
+  enabled: true
+  default_policy: "deny" # can be allow or deny
+  enable_token_persistence: true
+
+############################
+# consul DNS configuration #
+############################
+
+consul_dns_configuration:
+  allow_stale: true
+  enable_truncate: true
+  only_passing: true
+
+###########################
+# consul ui configuration #
+###########################
+
+consul_ui_configuration:
+  enabled: "{{ 'consul_servers' in group_names }}"
+
+#####################################
+# consul service mesh configuration #
+#####################################
+
+consul_mesh_configuration:
+  enabled: true
+
+#######################
+# extra configuration #
+#######################
+
+consul_extra_configuration: {}
+
+#####################################################
+#                                                   #
+#                Non-Editable                       #
+#                                                   #
+#####################################################
+
+##########################
+# consul haproxy backend #
+##########################
 
-########################
-# consul haproxy backend
-########################
 consul_haproxy_frontends:
-  - name: consul_internal
-    options:
-      - description consul internal gossip frontend
-      - mode tcp
-      - option tcplog
-      - bind :{{ hashi_consul_configuration.ports.serf_lan }}
-      - default_backend consul_internal
   - name: consul_external
     options:
       - description consul external http frontend
       - mode http
       - bind :80
-      - default_backend consul_external
+      - acl is_consul hdr(host) -i {{ consul_fqdn }}
+      - use_backend consul_external if is_consul
 
 consul_haproxy_backends:
-  - name: consul_internal
-    options: "{{ consul_internal_backend_options + consul_internal_backend_servers }}"
   - name: consul_external
     options: "{{ consul_external_backend_options + consul_external_backend_servers }}"
 
-consul_internal_backend_options:
-  - description consul internal gossip backend
-
-consul_internal_backend_servers: |
-  [
-    {% for host in groups['consul_servers'] %}
-      'server {{ hostvars[host].api_interface_address }} {{ hostvars[host].api_interface_address }}:{{ hashi_consul_configuration.ports.serf_lan }} check inter 3s'{% if not loop.last %},{% endif %}
-    {% endfor %}
-  ]
-
 consul_external_backend_options:
   - description consul external http backend
   - option forwardfor
@@ -59,27 +100,9 @@ consul_external_backend_servers: |
     {% endfor %}
   ]
 
-##############################
-# consul address configuration
-##############################
-
-consul_address_configuration:
-  # The address to which Consul will bind client interfaces,
-  # including the HTTP and DNS servers.
-  client_addr: "0.0.0.0"
-  # The address that should be bound to for internal cluster communications.
-  bind_addr: "{{ api_interface_address }}"
-  # The advertise address is used to change the address that we advertise to other nodes in the cluster.
-  advertise_addr: "{{ api_interface_address }}"
-
-##########################
-# consul ACL configuration
-##########################
-
-consul_acl_configuration:
-  enabled: true
-  default_policy: "deny" # can be allow or deny
-  enable_token_persistence: true
+############################
+# consul ACL configuration #
+############################
 
 consul_default_agent_policy: |
   agent_prefix "" {
@@ -92,41 +115,12 @@ consul_default_agent_policy: |
     policy = "read"
   }
 
-##########################
-# consul DNS configuration
-##########################
-
-consul_dns_configuration:
-  allow_stale: true
-  enable_truncate: true
-  only_passing: true
-
 #########################
-# consul ui configuration
+# consul role variables #
 #########################
 
-consul_ui_configuration:
-  enabled: true
-
-###################################
-# consul service mesh configuration
-###################################
-
-consul_mesh_configuration:
-  enabled: true
-
-#####################
-# extra configuration
-#####################
-
-consul_extra_configuration: {}
-
-###############
-# configuration
-###############
-
 hashi_consul_start_service: true
-hashi_consul_version: latest
+hashi_consul_version: "{{ consul_versions[deployment_method] }}"
 hashi_consul_deploy_method: "{{ deployment_method }}"
 hashi_consul_env_variables: {}
 hashi_cosul_config_dir: "/etc/consul.d"
@@ -141,7 +135,7 @@ hashi_consul_configuration:
   datacenter: "{{ consul_datacenter }}"
   primary_datacenter: "{{ consul_primary_datacenter }}"
   data_dir: "{{ hashi_consul_data_dir }}"
-  encrypt: "" # "{{ 'mysupersecretgossipencryptionkey'|b64encode }}"
+  encrypt: "{{ consul_gossip_encryption_key }}"
   server: "{{ 'consul_servers' in group_names }}"
   retry_join: "{{
     groups['consul_servers'] |
@@ -152,9 +146,9 @@ hashi_consul_configuration:
     }}"
   ui_config: "{{ consul_ui_configuration }}"
   connect: "{{ consul_mesh_configuration }}"
-  leave_on_terminate: true
-  rejoin_after_leave: true
-  enable_script_checks: true
+  leave_on_terminate: "{{ consul_leave_on_terminate }}"
+  rejoin_after_leave: "{{ consul_rejoin_after_leave }}"
+  enable_script_checks: "{{ consul_enable_script_checks }}"
   enable_syslog: "{{ deployment_method == 'host' }}"
   log_level: INFO
   acl: "{{ consul_acl_configuration }}"
diff --git a/playbooks/group_vars/all/haproxy.yml b/playbooks/group_vars/all/haproxy.yml
index 64361ec..267eba5 100644
--- a/playbooks/group_vars/all/haproxy.yml
+++ b/playbooks/group_vars/all/haproxy.yml
@@ -67,11 +67,11 @@ deploy_keepalived_unicast_peers: "{{ groups['haproxy_servers'] | difference([ans
 deploy_keepalived_auth_passwd: "password"
 deploy_keepalived_virtual_ips:
   - "{{ hashistack_external_vip_addr }}/32 dev {{ hashistack_external_vip_interface }}"
-  - "{{ hashistack_internal_vip_addr }}/32 dev {{ hashistack_internal_vip_interface }}"
+  # - "{{ hashistack_internal_vip_addr }}/32 dev {{ hashistack_internal_vip_interface }}"
 deploy_keepalived_notify_script: notify.sh
 
-deploy_keepalived_custom_scripts_src:
+deploy_keepalived_custom_scripts_src: tasks/haproxy/files/keepalived/scripts.d
 deploy_keepalived_extra_container_volumes: []
 
-deploy_keepalived_use_custom_config: false
-deploy_keepalived_custom_config_src:
+deploy_keepalived_use_custom_config: true
+deploy_keepalived_custom_config_src: tasks/haproxy/files/keepalived/keepalived.conf.j2
diff --git a/playbooks/group_vars/all/vault.yml b/playbooks/group_vars/all/vault.yml
index fa50464..27271c3 100644
--- a/playbooks/group_vars/all/vault.yml
+++ b/playbooks/group_vars/all/vault.yml
@@ -91,7 +91,7 @@ vault_extra_configuration: {}
 ###############
 
 hashi_vault_start_service: true
-hashi_vault_version: latest
+hashi_vault_version: "{{ vault_versions[deployment_method] }}"
 hashi_vault_deploy_method: "{{ deployment_method }}"
 hashi_vault_env_variables: {}
 hashi_vault_config_dir: "/etc/vault.d"
diff --git a/playbooks/inventory/multinode.ini b/playbooks/inventory/multinode.ini
index 8f68d6f..6acffb8 100644
--- a/playbooks/inventory/multinode.ini
+++ b/playbooks/inventory/multinode.ini
@@ -12,6 +12,9 @@ consul01
 consul02
 consul03
 
+[consul_agents:children]
+haproxy_servers
+
 [nomad_servers]
 nomad01
 nomad02
diff --git a/playbooks/tasks/consul/consul_deploy.yml b/playbooks/tasks/consul/consul_deploy.yml
index 4558097..197ecc3 100644
--- a/playbooks/tasks/consul/consul_deploy.yml
+++ b/playbooks/tasks/consul/consul_deploy.yml
@@ -25,8 +25,9 @@
       when: hashi_consul_configuration.acl.enabled
 
     - name: "Write consul configuration to file" # noqa: run-once[task] no-handler
-      ansible.builtin.copy:
-        content: "{{
+      ansible.builtin.blockinfile:
+        marker: "## -- {root_token} ANSIBLE MANAGED BLOCK ##"
+        block: "{{
           {
           'root_token':{
           'accessor_id': _consul_init_secret.result.AccessorID,
@@ -34,9 +35,7 @@
           }
           } | to_nice_yaml
           }}"
-        dest: "{{ sub_configuration_directories.consul_servers }}/consul_config.yml"
-        owner: "{{ lookup('env', 'USER') }}"
-        group: "{{ lookup('env', 'USER') }}"
+        path: "{{ sub_configuration_directories.consul_servers }}/consul_config.yml"
         mode: "0644"
       when: _consul_init_secret.result is defined
       run_once: true
diff --git a/playbooks/tasks/consul/consul_vars.yml b/playbooks/tasks/consul/consul_vars.yml
index cf05af2..a4f5073 100644
--- a/playbooks/tasks/consul/consul_vars.yml
+++ b/playbooks/tasks/consul/consul_vars.yml
@@ -8,7 +8,9 @@
       hashi_consul_configuration |
       combine(_config_to_merge|from_yaml)
       }}"
-  when: hashi_consul_configuration_string is defined
+  when:
+    - hashi_consul_configuration_string is defined
+    - "'consul_servers' in group_names"
 
 - name: "Consul | Merge addresses configuration"
   vars:
@@ -34,7 +36,7 @@
         name: _consul_cluster_config
       when: _consul_config_file.stat.exists
 
-    - name: "Merge token configuration"
+    - name: "Consul | Merge token configuration"
       vars:
         _config_to_merge:
           acl:
@@ -43,6 +45,16 @@
         hashi_consul_configuration: "{{ hashi_consul_configuration | default({}) | combine(_config_to_merge, recursive=true) }}"
       when: _consul_cluster_config.tokens is defined
 
+    - name: "Consul | Create secrets file if not present"
+      ansible.builtin.file:
+        path: "{{ sub_configuration_directories.consul_servers }}/consul_config.yml"
+        state: touch
+        owner: "{{ lookup('env', 'USER') }}"
+        group: "{{ lookup('env', 'USER') }}"
+        mode: "0644"
+      run_once: true
+      when: not _consul_config_file.stat.exists
+
 - name: "Consul | Merge extra configuration settings"
   vars:
     _config_to_merge: "{{ consul_extra_configuration }}"
diff --git a/playbooks/tasks/haproxy/files/keepalived/keepalived.conf.j2 b/playbooks/tasks/haproxy/files/keepalived/keepalived.conf.j2
new file mode 100644
index 0000000..e2377e2
--- /dev/null
+++ b/playbooks/tasks/haproxy/files/keepalived/keepalived.conf.j2
@@ -0,0 +1,50 @@
+# {{ ansible_managed }}
+global_defs {
+  script_user keepalived_script
+  enable_script_security
+}
+
+vrrp_script chk_haproxy{
+  script "/etc/keepalived/scripts.d/chk_haproxy.sh"
+  interval 3
+  weight 0
+  rise 6
+  fall 1
+}
+
+vrrp_instance {{ deploy_keepalived_vrrp_instance_name }} {
+  interface {{ deploy_keepalived_interface }}
+
+  state {{ deploy_keepalived_state }}
+  virtual_router_id {{ deploy_keepalived_router_id }}
+  priority {{ deploy_keepalived_priority }}
+  advert_int {{ deploy_keepalived_advert_interval }}
+
+  nopreempt
+
+{% if deploy_keepalived_unicast_peers %}
+  unicast_src_ip {{ deploy_keepalived_unicast_source }}
+  unicast_peer {
+{% for peer in deploy_keepalived_unicast_peers %}
+    {{ peer }}
+{% endfor %}
+  }
+{% endif %}
+
+  authentication {
+    auth_type PASS
+    auth_pass {{ deploy_keepalived_auth_passwd }}
+  }
+
+  virtual_ipaddress {
+{% for vip in deploy_keepalived_virtual_ips %}
+    {{ vip }}
+{% endfor %}
+  }
+
+  track_script {
+    chk_haproxy
+  }
+
+  notify {{ deploy_keepalived_scripts_dir }}/{{ deploy_keepalived_notify_script }}
+}
diff --git a/playbooks/tasks/haproxy/files/keepalived/scripts.d/chk_haproxy.sh b/playbooks/tasks/haproxy/files/keepalived/scripts.d/chk_haproxy.sh
new file mode 100644
index 0000000..7741349
--- /dev/null
+++ b/playbooks/tasks/haproxy/files/keepalived/scripts.d/chk_haproxy.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+set -e
+
+function do_ping() {
+  wget -q -O /dev/null --server-response "http://127.0.0.1:9000/health" 2>&1 | awk '/^  HTTP/{print $2}'
+}
+
+ping_result=$(do_ping)
+echo "Ping result: $ping_result"
+
+if [ "$ping_result" = "200" ]; then
+  exit 0
+else
+  exit 1
+fi
diff --git a/playbooks/tasks/load_vars.yml b/playbooks/tasks/load_vars.yml
index fbc90a4..7acd219 100644
--- a/playbooks/tasks/load_vars.yml
+++ b/playbooks/tasks/load_vars.yml
@@ -69,7 +69,7 @@
     file: "consul/consul_vars.yml"
   when:
     - enable_consul | bool
-    - "'consul_servers' in group_names"
+    - "('consul_servers' in group_names) or ('consul_agents' in group_names)"
 
 - name: "Merge vault configurations"
   ansible.builtin.import_tasks: