feat(consul): start configuring and deploying agents

molecule/no_tls_multi_node/molecule.yml

@@ -20,6 +20,7 @@ platforms:
     groups:
       - common
       - haproxy_servers
+      - consul_agents
   - name: proxy02
     box: generic/${MOLECULE_TEST_OS}
     cpus: 2
@@ -32,6 +33,7 @@ platforms:
     groups:
       - common
       - haproxy_servers
+      - consul_agents
   - name: hashistack01
     box: generic/${MOLECULE_TEST_OS}
     cpus: 4

playbooks/deploy.yml

@@ -19,7 +19,7 @@
       tags:
         - haproxy
 
-    - name: "Deploy Consul"
+    - name: "Deploy Consul Control Plane"
       ansible.builtin.import_tasks:
         file: tasks/consul/consul_deploy.yml
       when:
@@ -28,6 +28,15 @@
       tags:
         - consul
 
+    - name: "Deploy Consul Agents"
+      ansible.builtin.include_role:
+        name: ednz_cloud.hashistack.hashicorp_consul
+      when:
+        - enable_consul | bool
+        - "'consul_agents' in group_names"
+      tags:
+        - consul
+
     - name: "Deploy Vault"
       ansible.builtin.import_tasks:
         file: tasks/vault/vault_deploy.yml

playbooks/group_vars/all/all.yml

@@ -14,6 +14,10 @@ vault_version: latest
 
 deployment_method: "docker"
 
+consul_fqdn: consul.ednz.lab
+vault_fqdn: vault.ednz.lab
+nomad_fqdn: nomad.ednz.lab
+
 hashistack_external_vip_interface: "eth0"
 hashistack_external_vip_addr: "192.168.121.100"
 hashistack_internal_vip_interface: "eth1"
@@ -32,15 +36,15 @@ api_interface_address: "{{ ansible_facts[api_interface]['ipv4']['address'] }}"
 # manage_pip_packages_allow_break_system_packages: true
 
 vault_versions:
-  host: "{{ vault_version }}{% '*' if vault_version != 'latest' %}"
+  host: "{{ vault_version if vault_version != 'latest' else vault_version + '*' }}"
   docker: "{{ vault_version }}"
 
 consul_versions:
-  host: "{{ consul_version }}{% '*' if consul_version != 'latest' %}"
+  host: "{{ consul_version if consul_version != 'latest' else consul_version + '*' }}"
   docker: "{{ consul_version }}"
 
 nomad_versions:
-  host: "{{ nomad_version }}{% '*' if nomad_version != 'latest' %}"
+  host: "{{ nomad_version if nomad_version != 'latest' else nomad_version + '*' }}"
   docker: "{{ nomad_version }}"
 
 configuration_directory: "{{ lookup('env', 'PWD') }}/etc/hashistack"

playbooks/group_vars/all/consul.yml

@@ -1,7 +1,7 @@
 ---
 #####################################################
 #                                                   #
-#                Consul Configuration               #
+#                Editable                           #
 #                                                   #
 #####################################################
 
@@ -11,41 +11,82 @@ consul_primary_datacenter: dc1
 consul_leave_on_terminate: true
 consul_rejoin_after_leave: true
 consul_enable_script_checks: true
+consul_gossip_encryption_key: "{{ 'mysupersecretgossipencryptionkey'|b64encode }}"
+
+################################
+# consul address configuration #
+################################
+
+consul_address_configuration:
+  # The address to which Consul will bind client interfaces,
+  # including the HTTP and DNS servers.
+  client_addr: "0.0.0.0"
+  # The address that should be bound to for internal cluster communications.
+  bind_addr: "{{ api_interface_address }}"
+  # The advertise address is used to change the address that we advertise to other nodes in the cluster.
+  advertise_addr: "{{ api_interface_address }}"
+
+############################
+# consul ACL configuration #
+############################
+
+consul_acl_configuration:
+  enabled: true
+  default_policy: "deny" # can be allow or deny
+  enable_token_persistence: true
+
+############################
+# consul DNS configuration #
+############################
+
+consul_dns_configuration:
+  allow_stale: true
+  enable_truncate: true
+  only_passing: true
+
+###########################
+# consul ui configuration #
+###########################
+
+consul_ui_configuration:
+  enabled: "{{ 'consul_servers' in group_names }}"
+
+#####################################
+# consul service mesh configuration #
+#####################################
+
+consul_mesh_configuration:
+  enabled: true
+
+#######################
+# extra configuration #
+#######################
+
+consul_extra_configuration: {}
+
+#####################################################
+#                                                   #
+#                Non-Editable                       #
+#                                                   #
+#####################################################
+
+##########################
+# consul haproxy backend #
+##########################
 
-########################
-# consul haproxy backend
-########################
 consul_haproxy_frontends:
-  - name: consul_internal
-    options:
-      - description consul internal gossip frontend
-      - mode tcp
-      - option tcplog
-      - bind :{{ hashi_consul_configuration.ports.serf_lan }}
-      - default_backend consul_internal
   - name: consul_external
     options:
       - description consul external http frontend
       - mode http
       - bind :80
-      - default_backend consul_external
+      - acl is_consul hdr(host) -i {{ consul_fqdn }}
+      - use_backend consul_external if is_consul
 
 consul_haproxy_backends:
-  - name: consul_internal
-    options: "{{ consul_internal_backend_options + consul_internal_backend_servers }}"
   - name: consul_external
     options: "{{ consul_external_backend_options + consul_external_backend_servers }}"
 
-consul_internal_backend_options:
-  - description consul internal gossip backend
-
-consul_internal_backend_servers: |
-  [
-    {% for host in groups['consul_servers'] %}
-      'server {{ hostvars[host].api_interface_address }} {{ hostvars[host].api_interface_address }}:{{ hashi_consul_configuration.ports.serf_lan }} check inter 3s'{% if not loop.last %},{% endif %}
-    {% endfor %}
-  ]
-
 consul_external_backend_options:
   - description consul external http backend
   - option forwardfor
@@ -59,27 +100,9 @@ consul_external_backend_servers: |
     {% endfor %}
   ]
 
-##############################
+############################
-# consul address configuration
+# consul ACL configuration #
-##############################
+############################
-
-consul_address_configuration:
-  # The address to which Consul will bind client interfaces,
-  # including the HTTP and DNS servers.
-  client_addr: "0.0.0.0"
-  # The address that should be bound to for internal cluster communications.
-  bind_addr: "{{ api_interface_address }}"
-  # The advertise address is used to change the address that we advertise to other nodes in the cluster.
-  advertise_addr: "{{ api_interface_address }}"
-
-##########################
-# consul ACL configuration
-##########################
-
-consul_acl_configuration:
-  enabled: true
-  default_policy: "deny" # can be allow or deny
-  enable_token_persistence: true
 
 consul_default_agent_policy: |
   agent_prefix "" {
@@ -92,41 +115,12 @@ consul_default_agent_policy: |
     policy = "read"
   }
 
-##########################
-# consul DNS configuration
-##########################
-
-consul_dns_configuration:
-  allow_stale: true
-  enable_truncate: true
-  only_passing: true
-
 #########################
-# consul ui configuration
+# consul role variables #
 #########################
 
-consul_ui_configuration:
-  enabled: true
-
-###################################
-# consul service mesh configuration
-###################################
-
-consul_mesh_configuration:
-  enabled: true
-
-#####################
-# extra configuration
-#####################
-
-consul_extra_configuration: {}
-
-###############
-# configuration
-###############
-
 hashi_consul_start_service: true
-hashi_consul_version: latest
+hashi_consul_version: "{{ consul_versions[deployment_method] }}"
 hashi_consul_deploy_method: "{{ deployment_method }}"
 hashi_consul_env_variables: {}
 hashi_cosul_config_dir: "/etc/consul.d"
@@ -141,7 +135,7 @@ hashi_consul_configuration:
   datacenter: "{{ consul_datacenter }}"
   primary_datacenter: "{{ consul_primary_datacenter }}"
   data_dir: "{{ hashi_consul_data_dir }}"
-  encrypt: "" # "{{ 'mysupersecretgossipencryptionkey'|b64encode }}"
+  encrypt: "{{ consul_gossip_encryption_key }}"
   server: "{{ 'consul_servers' in group_names }}"
   retry_join: "{{
     groups['consul_servers'] |
@@ -152,9 +146,9 @@ hashi_consul_configuration:
     }}"
   ui_config: "{{ consul_ui_configuration }}"
   connect: "{{ consul_mesh_configuration }}"
-  leave_on_terminate: true
+  leave_on_terminate: "{{ consul_leave_on_terminate }}"
-  rejoin_after_leave: true
+  rejoin_after_leave: "{{ consul_rejoin_after_leave }}"
-  enable_script_checks: true
+  enable_script_checks: "{{ consul_enable_script_checks }}"
   enable_syslog: "{{ deployment_method == 'host' }}"
   log_level: INFO
   acl: "{{ consul_acl_configuration }}"

playbooks/group_vars/all/haproxy.yml

@@ -67,11 +67,11 @@ deploy_keepalived_unicast_peers: "{{ groups['haproxy_servers'] | difference([ans
 deploy_keepalived_auth_passwd: "password"
 deploy_keepalived_virtual_ips:
   - "{{ hashistack_external_vip_addr }}/32 dev {{ hashistack_external_vip_interface }}"
-  - "{{ hashistack_internal_vip_addr }}/32 dev {{ hashistack_internal_vip_interface }}"
+  # - "{{ hashistack_internal_vip_addr }}/32 dev {{ hashistack_internal_vip_interface }}"
 deploy_keepalived_notify_script: notify.sh
 
-deploy_keepalived_custom_scripts_src:
+deploy_keepalived_custom_scripts_src: tasks/haproxy/files/keepalived/scripts.d
 deploy_keepalived_extra_container_volumes: []
 
-deploy_keepalived_use_custom_config: false
+deploy_keepalived_use_custom_config: true
-deploy_keepalived_custom_config_src:
+deploy_keepalived_custom_config_src: tasks/haproxy/files/keepalived/keepalived.conf.j2

playbooks/group_vars/all/vault.yml

@@ -91,7 +91,7 @@ vault_extra_configuration: {}
 ###############
 
 hashi_vault_start_service: true
-hashi_vault_version: latest
+hashi_vault_version: "{{ vault_versions[deployment_method] }}"
 hashi_vault_deploy_method: "{{ deployment_method }}"
 hashi_vault_env_variables: {}
 hashi_vault_config_dir: "/etc/vault.d"

playbooks/inventory/multinode.ini

@@ -12,6 +12,9 @@ consul01
 consul02
 consul03
 
+[consul_agents:children]
+haproxy_servers
+
 [nomad_servers]
 nomad01
 nomad02

playbooks/tasks/consul/consul_deploy.yml

@@ -25,8 +25,9 @@
       when: hashi_consul_configuration.acl.enabled
 
     - name: "Write consul configuration to file" # noqa: run-once[task] no-handler
-      ansible.builtin.copy:
+      ansible.builtin.blockinfile:
-        content: "{{
+        marker: "## -- {root_token} ANSIBLE MANAGED BLOCK ##"
+        block: "{{
           {
           'root_token':{
           'accessor_id': _consul_init_secret.result.AccessorID,
@@ -34,9 +35,7 @@
           }
           } | to_nice_yaml
           }}"
-        dest: "{{ sub_configuration_directories.consul_servers }}/consul_config.yml"
+        path: "{{ sub_configuration_directories.consul_servers }}/consul_config.yml"
-        owner: "{{ lookup('env', 'USER') }}"
-        group: "{{ lookup('env', 'USER') }}"
         mode: "0644"
       when: _consul_init_secret.result is defined
       run_once: true

playbooks/tasks/consul/consul_vars.yml

@@ -8,7 +8,9 @@
       hashi_consul_configuration |
       combine(_config_to_merge|from_yaml)
       }}"
-  when: hashi_consul_configuration_string is defined
+  when:
+    - hashi_consul_configuration_string is defined
+    - "'consul_servers' in group_names"
 
 - name: "Consul | Merge addresses configuration"
   vars:
@@ -34,7 +36,7 @@
         name: _consul_cluster_config
       when: _consul_config_file.stat.exists
 
-    - name: "Merge token configuration"
+    - name: "Consul | Merge token configuration"
       vars:
         _config_to_merge:
           acl:
@@ -43,6 +45,16 @@
         hashi_consul_configuration: "{{ hashi_consul_configuration | default({}) | combine(_config_to_merge, recursive=true) }}"
       when: _consul_cluster_config.tokens is defined
 
+    - name: "Consul | Create secrets file if not present"
+      ansible.builtin.file:
+        path: "{{ sub_configuration_directories.consul_servers }}/consul_config.yml"
+        state: touch
+        owner: "{{ lookup('env', 'USER') }}"
+        group: "{{ lookup('env', 'USER') }}"
+        mode: "0644"
+      run_once: true
+      when: not _consul_config_file.stat.exists
+
 - name: "Consul | Merge extra configuration settings"
   vars:
     _config_to_merge: "{{ consul_extra_configuration }}"

playbooks/tasks/haproxy/files/keepalived/keepalived.conf.j2

@@ -0,0 +1,50 @@
+# {{ ansible_managed }}
+global_defs {
+  script_user keepalived_script
+  enable_script_security
+}
+
+vrrp_script chk_haproxy{
+  script "/etc/keepalived/scripts.d/chk_haproxy.sh"
+  interval 3
+  weight 0
+  rise 6
+  fall 1
+}
+
+vrrp_instance {{ deploy_keepalived_vrrp_instance_name }} {
+  interface {{ deploy_keepalived_interface }}
+
+  state {{ deploy_keepalived_state }}
+  virtual_router_id {{ deploy_keepalived_router_id }}
+  priority {{ deploy_keepalived_priority }}
+  advert_int {{ deploy_keepalived_advert_interval }}
+
+  nopreempt
+
+{% if deploy_keepalived_unicast_peers %}
+  unicast_src_ip {{ deploy_keepalived_unicast_source }}
+  unicast_peer {
+{% for peer in deploy_keepalived_unicast_peers %}
+    {{ peer }}
+{% endfor %}
+  }
+{% endif %}
+
+  authentication {
+    auth_type PASS
+    auth_pass {{ deploy_keepalived_auth_passwd }}
+  }
+
+  virtual_ipaddress {
+{% for vip in deploy_keepalived_virtual_ips %}
+    {{ vip }}
+{% endfor %}
+  }
+
+  track_script {
+    chk_haproxy
+  }
+
+  notify {{ deploy_keepalived_scripts_dir }}/{{ deploy_keepalived_notify_script }}
+}

playbooks/tasks/haproxy/files/keepalived/scripts.d/chk_haproxy.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+set -e
+
+function do_ping() {
+  wget -q -O /dev/null --server-response "http://127.0.0.1:9000/health" 2>&1 | awk '/^  HTTP/{print $2}'
+}
+
+ping_result=$(do_ping)
+echo "Ping result: $ping_result"
+
+if [ "$ping_result" = "200" ]; then
+  exit 0
+else
+  exit 1
+fi

playbooks/tasks/load_vars.yml

@@ -69,7 +69,7 @@
     file: "consul/consul_vars.yml"
   when:
     - enable_consul | bool
-    - "'consul_servers' in group_names"
+    - "('consul_servers' in group_names) or ('consul_agents' in group_names)"
 
 - name: "Merge vault configurations"
   ansible.builtin.import_tasks: