From b32815066ff14312ee2ebd91f20c87169a532f71 Mon Sep 17 00:00:00 2001 From: Bertrand Lanson Date: Thu, 20 Jun 2024 00:32:33 +0200 Subject: [PATCH] feat: use new vault and consul roles, and only allow for host deployment after docker support drop --- docs/vault_clusters.md | 2 +- .../etc/hashistack/globals.yml | 4 +-- molecule/no_tls_multi_node/requirements.yml | 4 +-- playbooks/bootstrap.yml | 6 ++-- playbooks/generate_certs.yml | 4 +-- playbooks/group_vars/all/all.yml | 12 ------- playbooks/group_vars/all/consul.yml | 31 +++++++++---------- playbooks/group_vars/all/globals.yml | 4 +-- playbooks/group_vars/all/haproxy.yml | 4 +-- playbooks/group_vars/all/nomad.yml | 3 +- playbooks/group_vars/all/vault.yml | 28 ++++++++--------- playbooks/preflight.yml | 17 ---------- playbooks/tasks/consul/consul_deploy.yml | 6 ++-- playbooks/tasks/consul/consul_vars.yml | 22 ++++++------- .../files/keepalived/scripts.d/chk_haproxy.sh | 2 +- playbooks/tasks/vault/vault_deploy.yml | 6 ++-- playbooks/tasks/vault/vault_vars.yml | 16 +++++----- roles/requirements.yml | 6 ++-- 18 files changed, 71 insertions(+), 106 deletions(-) diff --git a/docs/vault_clusters.md b/docs/vault_clusters.md index ad91216..089178e 100644 --- a/docs/vault_clusters.md +++ b/docs/vault_clusters.md @@ -53,7 +53,7 @@ The storage configuration for vault can be edited as well. By default, vault wil ```yaml vault_storage_configuration: raft: - path: "{{ hashi_vault_data_dir }}/data" + path: "{{ hashicorp_vault_data_dir }}/data" node_id: "{{ ansible_hostname }}" retry_join: | [ diff --git a/molecule/no_tls_multi_node/etc/hashistack/globals.yml b/molecule/no_tls_multi_node/etc/hashistack/globals.yml index f35190a..ca8da97 100644 --- a/molecule/no_tls_multi_node/etc/hashistack/globals.yml +++ b/molecule/no_tls_multi_node/etc/hashistack/globals.yml @@ -13,8 +13,6 @@ enable_nomad: "no" # consul_version: "1.18.1" # vault_version: "1.16.2" -# deployment_method: "docker" - # consul_fqdn: consul.ednz.lab # vault_fqdn: vault.ednz.lab # nomad_fqdn: nomad.ednz.lab @@ -139,7 +137,7 @@ consul_enable_tls: true # vault_storage_configuration: # raft: -# path: "{{ hashi_vault_data_dir }}/data" +# path: "{{ hashicorp_vault_data_dir }}/data" # node_id: "{{ ansible_hostname }}" # retry_join: | # [ diff --git a/molecule/no_tls_multi_node/requirements.yml b/molecule/no_tls_multi_node/requirements.yml index b025327..ae8019d 100644 --- a/molecule/no_tls_multi_node/requirements.yml +++ b/molecule/no_tls_multi_node/requirements.yml @@ -6,8 +6,8 @@ roles: - name: ednz_cloud.manage_pip_packages - name: ednz_cloud.install_docker - name: ednz_cloud.docker_systemd_service - - name: ednz_cloud.deploy_haproxy - - name: ednz_cloud.deploy_keepalived +# - name: ednz_cloud.deploy_haproxy +# - name: ednz_cloud.deploy_keepalived collections: - name: ednz_cloud.hashistack diff --git a/playbooks/bootstrap.yml b/playbooks/bootstrap.yml index d167da2..b190b9c 100644 --- a/playbooks/bootstrap.yml +++ b/playbooks/bootstrap.yml @@ -35,13 +35,11 @@ install_docker_auto_update: false install_docker_start_service: true install_docker_compose: false - install_docker_compose_version: latest - install_docker_python_packages: true - install_docker_python_packages_version: latest + install_docker_python_packages: false install_docker_users: - "{{ ansible_user }}" install_docker_daemon_options: {} - when: deployment_method == 'docker' + #! when: "'nomad_agents' in group_names" - name: "Ensure /etc/localtime exists" ansible.builtin.file: diff --git a/playbooks/generate_certs.yml b/playbooks/generate_certs.yml index 14e975d..ede97b2 100644 --- a/playbooks/generate_certs.yml +++ b/playbooks/generate_certs.yml @@ -221,8 +221,8 @@ 'IP:' + api_interface_address, 'IP:127.0.0.1' ] -%} - {%- if hashi_consul_configuration.server -%} - {%- set _ = sans_list.append('DNS:server.' ~ hashi_consul_configuration.datacenter ~ '.' ~ hashi_consul_configuration.domain) -%} + {%- if hashicorp_consul_configuration.server -%} + {%- set _ = sans_list.append('DNS:server.' ~ hashicorp_consul_configuration.datacenter ~ '.' ~ hashicorp_consul_configuration.domain) -%} {%- endif -%} {{ sans_list }} community.crypto.openssl_csr_pipe: diff --git a/playbooks/group_vars/all/all.yml b/playbooks/group_vars/all/all.yml index d5b9469..8e95eb1 100644 --- a/playbooks/group_vars/all/all.yml +++ b/playbooks/group_vars/all/all.yml @@ -5,18 +5,6 @@ manage_pip_packages_allow_break_system_packages: "{{ ansible_distribution == 'Debian' and ansible_distribution_version == '12' }}" -vault_versions: - host: "{{ vault_version if vault_version != 'latest' else vault_version + '*' }}" - docker: "{{ vault_version }}" - -consul_versions: - host: "{{ consul_version if consul_version != 'latest' else consul_version + '*' }}" - docker: "{{ consul_version }}" - -nomad_versions: - host: "{{ nomad_version if nomad_version != 'latest' else nomad_version + '*' }}" - docker: "{{ nomad_version }}" - configuration_directory: "{{ lookup('env', 'PWD') }}/etc/hashistack" sub_configuration_directories: secrets: "{{ configuration_directory }}/secrets" diff --git a/playbooks/group_vars/all/consul.yml b/playbooks/group_vars/all/consul.yml index bbfbd59..4768cac 100644 --- a/playbooks/group_vars/all/consul.yml +++ b/playbooks/group_vars/all/consul.yml @@ -57,7 +57,7 @@ consul_default_agent_policy: | # consul internal tls # ####################### -consul_certificates_directory: "{{ hashi_consul_config_dir }}/tls" +consul_certificates_directory: "{{ hashicorp_consul_config_dir }}/tls" consul_certificates_extra_files_dir: - src: "{{ sub_configuration_directories['certificates'] }}/consul/{{ inventory_hostname }}" dest: "{{ consul_certificates_directory }}" @@ -66,28 +66,27 @@ consul_certificates_extra_files_dir: # consul role variables # ######################### -hashi_consul_start_service: true -hashi_consul_version: "{{ consul_versions[deployment_method] }}" -hashi_consul_deploy_method: "{{ deployment_method }}" -hashi_consul_env_variables: {} -hashi_consul_config_dir: "/etc/consul.d" -hashi_consul_data_dir: "/opt/consul" -hashi_consul_extra_files: true -hashi_consul_extra_files_list: "{{ ([] + +hashicorp_consul_start_service: true +hashicorp_consul_service_name: "consul" +hashicorp_consul_version: "{{ consul_version }}" +hashicorp_consul_env_variables: {} +hashicorp_consul_config_dir: "/etc/consul.d" +hashicorp_consul_data_dir: "/opt/consul" +hashicorp_consul_extra_files: true +hashicorp_consul_extra_files_list: "{{ ([] + (consul_certificates_extra_files_dir if consul_enable_tls else []) + (vault_plugin_extra_files_dir if vault_enable_plugins else []) + vault_extra_files_list) | unique | sort }}" -hashi_consul_extra_container_volumes: "{{ default_container_extra_volumes | union(extra_consul_container_volumes) | unique | sort }}" -hashi_consul_envoy_install: false -hashi_consul_envoy_version: v1.27.2 -hashi_consul_configuration: +hashicorp_consul_envoy_install: false +hashicorp_consul_envoy_version: v1.27.2 +hashicorp_consul_configuration: domain: "{{ consul_domain }}" datacenter: "{{ consul_datacenter }}" primary_datacenter: "{{ consul_primary_datacenter }}" - data_dir: "{{ hashi_consul_data_dir }}" + data_dir: "{{ hashicorp_consul_data_dir }}" encrypt: "{{ _credentials.consul.gossip_encryption_key }}" server: "{{ 'consul_servers' in group_names }}" retry_join: "{{ @@ -102,7 +101,7 @@ hashi_consul_configuration: leave_on_terminate: "{{ consul_leave_on_terminate }}" rejoin_after_leave: "{{ consul_rejoin_after_leave }}" enable_script_checks: "{{ consul_enable_script_checks }}" - enable_syslog: "{{ deployment_method == 'host' }}" + enable_syslog: true log_level: INFO acl: "{{ consul_acl_configuration }}" dns_config: "{{ consul_dns_configuration }}" @@ -119,7 +118,7 @@ hashi_consul_configuration: expose_max_port: 21755 # this is used to circumvent jinja limitation to convert string to integer -hashi_consul_configuration_string: | +hashicorp_consul_configuration_string: | bootstrap_expect: {{ (groups['consul_servers'] | length) }} ports: http: {{ (consul_api_port.http|int) if not consul_enable_tls else ('-1' | int) }} diff --git a/playbooks/group_vars/all/globals.yml b/playbooks/group_vars/all/globals.yml index f72cf7b..7331c23 100644 --- a/playbooks/group_vars/all/globals.yml +++ b/playbooks/group_vars/all/globals.yml @@ -13,8 +13,6 @@ nomad_version: "1.7.7" consul_version: "1.18.1" vault_version: "1.16.2" -deployment_method: "docker" - consul_fqdn: consul.ednz.lab vault_fqdn: vault.ednz.lab nomad_fqdn: nomad.ednz.lab @@ -138,7 +136,7 @@ vault_seal_configuration: vault_storage_configuration: raft: - path: "{{ hashi_vault_data_dir }}/data" + path: "{{ hashicorp_vault_data_dir }}" node_id: "{{ ansible_hostname }}" retry_join: | [ diff --git a/playbooks/group_vars/all/haproxy.yml b/playbooks/group_vars/all/haproxy.yml index 7ae85cc..e41f632 100644 --- a/playbooks/group_vars/all/haproxy.yml +++ b/playbooks/group_vars/all/haproxy.yml @@ -5,7 +5,7 @@ # # ##################################################### -deploy_haproxy_deploy_method: "{{ deployment_method }}" +deploy_haproxy_deploy_method: "host" deploy_haproxy_version: "{{ haproxy_version }}" deploy_haproxy_env_variables: {} @@ -73,7 +73,7 @@ deploy_haproxy_listen: - monitor-uri /health - http-request use-service prometheus-exporter if { path /metrics } -deploy_keepalived_deploy_method: "{{ deployment_method }}" +deploy_keepalived_deploy_method: "host" deploy_keepalived_version: "latest" deploy_keepalived_start_service: true deploy_keepalived_env_variables: {} diff --git a/playbooks/group_vars/all/nomad.yml b/playbooks/group_vars/all/nomad.yml index 5572039..561de18 100644 --- a/playbooks/group_vars/all/nomad.yml +++ b/playbooks/group_vars/all/nomad.yml @@ -63,8 +63,9 @@ nomad_client_configuration: enabled: "{{ 'nomad_clients' in group_names | bool }}" state_dir: "{{ hashicorp_nomad_data_dir }}/client" -hashicorp_nomad_cni_plugins_install: true hashicorp_nomad_start_service: true +hashicorp_nomad_service_name: "nomad" +hashicorp_nomad_cni_plugins_install: true hashicorp_nomad_cni_plugins_version: latest hashicorp_nomad_cni_plugins_install_path: /opt/cni/bin hashicorp_nomad_version: latest diff --git a/playbooks/group_vars/all/vault.yml b/playbooks/group_vars/all/vault.yml index d6094b8..02c0e57 100644 --- a/playbooks/group_vars/all/vault.yml +++ b/playbooks/group_vars/all/vault.yml @@ -35,7 +35,7 @@ vault_external_backend_servers: | # vault internal tls # ###################### -vault_certificates_directory: "{{ hashi_vault_config_dir }}/tls" +vault_certificates_directory: "{{ hashicorp_vault_config_dir }}/tls" vault_certificates_extra_files_dir: - src: "{{ sub_configuration_directories['certificates'] }}/vault/{{ inventory_hostname }}" dest: "{{ vault_certificates_directory }}" @@ -44,7 +44,7 @@ vault_certificates_extra_files_dir: # vault plugins # ################# -vault_plugin_directory: "{{ hashi_vault_config_dir }}/plugin" +vault_plugin_directory: "{{ hashicorp_vault_config_dir }}/plugin" vault_plugin_extra_files_dir: - src: "{{ sub_configuration_directories['vault_servers'] }}/plugin" dest: "{{ vault_plugin_directory }}" @@ -62,24 +62,24 @@ vault_service_registration_policy: | # vault role variables # ######################## -hashi_vault_start_service: true -hashi_vault_version: "{{ vault_versions[deployment_method] }}" -hashi_vault_deploy_method: "{{ deployment_method }}" -hashi_vault_env_variables: {} -hashi_vault_config_dir: "/etc/vault.d" -hashi_vault_data_dir: "/opt/vault" -hashi_vault_extra_files: true -hashi_vault_extra_files_list: "{{ ([] + +hashicorp_vault_start_service: true +hashicorp_vault_service_name: "vault" +hashicorp_vault_version: "{{ vault_version }}" +hashicorp_vault_env_variables: {} +hashicorp_vault_config_dir: "/etc/vault.d" +hashicorp_vault_data_dir: "/opt/vault" +hashicorp_vault_extra_files: true +hashicorp_vault_extra_files_list: "{{ ([] + (vault_certificates_extra_files_dir if vault_enable_tls else []) + (vault_plugin_extra_files_dir if vault_enable_plugins else []) + vault_extra_files_list) | unique | sort }}" -hashi_vault_extra_files_src: "{{ sub_configuration_directories.vault_servers }}/config" -hashi_vault_extra_files_dst: "{{ hashi_vault_config_dir }}/config" -hashi_vault_extra_container_volumes: "{{ default_container_extra_volumes | union(extra_vault_container_volumes) | unique | sort }}" -hashi_vault_configuration: +hashicorp_vault_extra_files_src: "{{ sub_configuration_directories.vault_servers }}/config" +hashicorp_vault_extra_files_dst: "{{ hashicorp_vault_config_dir }}/config" +hashicorp_vault_extra_container_volumes: "{{ default_container_extra_volumes | union(extra_vault_container_volumes) | unique | sort }}" +hashicorp_vault_configuration: cluster_name: "{{ vault_cluster_name }}" cluster_addr: "{{ 'https' if vault_enable_tls else 'http'}}://{{ api_interface_address }}:8201" api_addr: "{{ 'https' if vault_enable_tls else 'http'}}://{{ api_interface_address }}:8200" diff --git a/playbooks/preflight.yml b/playbooks/preflight.yml index 0ec807e..e4216c2 100644 --- a/playbooks/preflight.yml +++ b/playbooks/preflight.yml @@ -288,20 +288,3 @@ that: - "ansible_facts.service_mgr == 'systemd'" when: inventory_hostname in groups['common'] - - - name: "Checking that python SDK for docker is installed" - when: deployment_method == 'docker' - vars: - wanted_docker_sdk_package: "python3-docker" - block: - - name: "Get packages facts" - ansible.builtin.package_facts: - manager: auto - - - name: "Checking that python SDK for docker is installed" - ansible.builtin.assert: - that: - - "wanted_docker_sdk_package in ansible_facts.packages" - fail_msg: >- - The python sdk for docker is really out of date, you need to install - a more recent version of it in order to use this tool. diff --git a/playbooks/tasks/consul/consul_deploy.yml b/playbooks/tasks/consul/consul_deploy.yml index 5937d93..7c54178 100644 --- a/playbooks/tasks/consul/consul_deploy.yml +++ b/playbooks/tasks/consul/consul_deploy.yml @@ -27,7 +27,7 @@ run_once: true delegate_to: "{{ groups['consul_servers'] | first }}" register: _consul_init_secret - when: hashi_consul_configuration.acl.enabled + when: hashicorp_consul_configuration.acl.enabled - name: "Create consul agents token" when: @@ -63,9 +63,9 @@ state: present register: _consul_agent_token - - name: "Restart consul service" + - name: "Restart consul service" # noqa: no-handler ansible.builtin.service: - name: "consul_container" + name: "{{ hashicorp_consul_service_name }}" state: restarted throttle: 1 when: _consul_agent_token.changed diff --git a/playbooks/tasks/consul/consul_vars.yml b/playbooks/tasks/consul/consul_vars.yml index a6b9e01..5e2fdf0 100644 --- a/playbooks/tasks/consul/consul_vars.yml +++ b/playbooks/tasks/consul/consul_vars.yml @@ -2,22 +2,22 @@ # hashistack configuration merging for consul - name: "Consul | Merge stringified configuration" vars: - _config_to_merge: "{{ hashi_consul_configuration_string }}" + _config_to_merge: "{{ hashicorp_consul_configuration_string }}" ansible.builtin.set_fact: - hashi_consul_configuration: "{{ - hashi_consul_configuration | + hashicorp_consul_configuration: "{{ + hashicorp_consul_configuration | combine(_config_to_merge|from_yaml, recursive=true) }}" when: - - hashi_consul_configuration_string is defined + - hashicorp_consul_configuration_string is defined - "'consul_servers' in group_names" - name: "Consul | Merge addresses configuration" vars: _config_to_merge: "{{ consul_address_configuration }}" ansible.builtin.set_fact: - hashi_consul_configuration: "{{ - hashi_consul_configuration | + hashicorp_consul_configuration: "{{ + hashicorp_consul_configuration | combine(_config_to_merge, recursive=true) }}" when: consul_address_configuration is defined @@ -27,8 +27,8 @@ _config_to_merge: tls: "{{ consul_tls_configuration }}" ansible.builtin.set_fact: - hashi_consul_configuration: "{{ - hashi_consul_configuration | + hashicorp_consul_configuration: "{{ + hashicorp_consul_configuration | combine(_config_to_merge, recursive=true) }}" when: consul_enable_tls @@ -43,14 +43,14 @@ tokens: agent: "{{ _credentials.consul.tokens.agent.secret_id }}" ansible.builtin.set_fact: - hashi_consul_configuration: "{{ hashi_consul_configuration | default({}) | combine(_config_to_merge, recursive=true) }}" + hashicorp_consul_configuration: "{{ hashicorp_consul_configuration | default({}) | combine(_config_to_merge, recursive=true) }}" - name: "Consul | Merge extra configuration settings" vars: _config_to_merge: "{{ consul_extra_configuration }}" ansible.builtin.set_fact: - hashi_consul_configuration: "{{ - hashi_consul_configuration | + hashicorp_consul_configuration: "{{ + hashicorp_consul_configuration | combine(_config_to_merge, recursive=true) }}" when: consul_extra_configuration is defined diff --git a/playbooks/tasks/haproxy/files/keepalived/scripts.d/chk_haproxy.sh b/playbooks/tasks/haproxy/files/keepalived/scripts.d/chk_haproxy.sh index 7741349..78141b6 100644 --- a/playbooks/tasks/haproxy/files/keepalived/scripts.d/chk_haproxy.sh +++ b/playbooks/tasks/haproxy/files/keepalived/scripts.d/chk_haproxy.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -e function do_ping() { diff --git a/playbooks/tasks/vault/vault_deploy.yml b/playbooks/tasks/vault/vault_deploy.yml index f3e3e33..844118a 100644 --- a/playbooks/tasks/vault/vault_deploy.yml +++ b/playbooks/tasks/vault/vault_deploy.yml @@ -44,7 +44,7 @@ - name: "Initialize vault cluster" # noqa: run-once[task] ednz_cloud.hashistack.vault_init: - api_url: "{{ hashi_vault_configuration['api_addr'] }}" + api_url: "{{ hashicorp_vault_configuration['api_addr'] }}" tls_verify: "{{ vault_tls_verify }}" key_shares: "{{ vault_seal_configuration['key_shares'] }}" key_threshold: "{{ vault_seal_configuration['key_threshold'] }}" @@ -77,7 +77,7 @@ - name: "Unseal the bootstrap node" # noqa: run-once[task] no-handler ednz_cloud.hashistack.vault_unseal: - api_url: "{{ hashi_vault_configuration['api_addr'] }}" + api_url: "{{ hashicorp_vault_configuration['api_addr'] }}" tls_verify: "{{ vault_tls_verify }}" key_shares: "{{ _credentials.vault['keys'] }}" run_once: true @@ -87,7 +87,7 @@ - name: "Unseal all vault nodes" ednz_cloud.hashistack.vault_unseal: - api_url: "{{ hashi_vault_configuration['api_addr'] }}" + api_url: "{{ hashicorp_vault_configuration['api_addr'] }}" tls_verify: "{{ vault_tls_verify }}" key_shares: "{{ _credentials.vault['keys'] }}" retries: 5 diff --git a/playbooks/tasks/vault/vault_vars.yml b/playbooks/tasks/vault/vault_vars.yml index c95c205..6a5758b 100644 --- a/playbooks/tasks/vault/vault_vars.yml +++ b/playbooks/tasks/vault/vault_vars.yml @@ -13,8 +13,8 @@ _config_to_merge: service_registration: "{{ vault_service_registration_configuration }}" ansible.builtin.set_fact: - hashi_vault_configuration: "{{ - hashi_vault_configuration | + hashicorp_vault_configuration: "{{ + hashicorp_vault_configuration | combine(_config_to_merge) }}" when: vault_enable_service_registration @@ -24,8 +24,8 @@ _config_to_merge: plugin_directory: "{{ vault_plugin_directory }}" ansible.builtin.set_fact: - hashi_vault_configuration: "{{ - hashi_vault_configuration | + hashicorp_vault_configuration: "{{ + hashicorp_vault_configuration | combine(_config_to_merge) }}" when: vault_enable_plugins @@ -34,8 +34,8 @@ vars: _config_to_merge: "{{ vault_logging_configuration }}" ansible.builtin.set_fact: - hashi_vault_configuration: "{{ - hashi_vault_configuration | + hashicorp_vault_configuration: "{{ + hashicorp_vault_configuration | combine(_config_to_merge) }}" when: vault_enable_log_to_file @@ -44,8 +44,8 @@ vars: _config_to_merge: "{{ vault_extra_configuration }}" ansible.builtin.set_fact: - hashi_vault_configuration: "{{ - hashi_vault_configuration | + hashicorp_vault_configuration: "{{ + hashicorp_vault_configuration | combine(_config_to_merge) }}" when: vault_extra_configuration is defined diff --git a/roles/requirements.yml b/roles/requirements.yml index ee2798a..15c4e7e 100644 --- a/roles/requirements.yml +++ b/roles/requirements.yml @@ -24,10 +24,10 @@ roles: version: main - name: ednz_cloud.hashicorp_nomad src: https://github.com/ednz-cloud/hashicorp_nomad.git - version: v0.1.0 + version: v0.4.0 - name: ednz_cloud.hashicorp_consul src: https://github.com/ednz-cloud/hashicorp_consul.git - version: main + version: v0.2.0 - name: ednz_cloud.hashicorp_vault src: https://github.com/ednz-cloud/hashicorp_vault.git - version: main + version: v0.2.0