diff --git a/.dockerignore b/.dockerignore index e3394ae..f08f439 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,3 +8,4 @@ !ansible.cfg !hosts.ini !playbooks/ +!requirements.yml diff --git a/Dockerfile b/Dockerfile index f03a09f..d2306b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,8 +7,14 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ apt update apt install -y \ build-essential \ + curl \ git \ + unzip \ ; + curl -fsSL "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_64bit/session-manager-plugin.deb" \ + -o /tmp/session-manager-plugin.deb + dpkg -i /tmp/session-manager-plugin.deb + rm /tmp/session-manager-plugin.deb EOCMD FROM minimal AS dependencies @@ -41,10 +47,16 @@ RUN < AWS_SSM_BUCKET= ./scripts/ssm-smoke-test.sh INSTANCE_ID + +[test_hosts] +example-db-node ansible_host=INSTANCE_ID ansible_connection=aws_ssm ansible_aws_ssm_region=us-west-2 ansible_remote_tmp=/tmp/ansible-ssm + +[mongo_node_2] +example-db-node + +[mongo_servers] +mongo_node_2 diff --git a/playbooks/clickhouse_backup.yml b/playbooks/clickhouse_backup.yml new file mode 100644 index 0000000..cdbfcf3 --- /dev/null +++ b/playbooks/clickhouse_backup.yml @@ -0,0 +1,10 @@ +--- +# Description: Playbook to launch ClickHouse backups +- name: Launch backups + hosts: clickhouse_servers + become: true + gather_facts: true + roles: + - role: clickhouse_backup + tags: + - clickhouse_backup diff --git a/pyproject.toml b/pyproject.toml index f0df8eb..7dc7392 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "atlas-ansible-utils" -version = "21.1.1" +version = "21.1.2" requires-python = ">=3.12" dependencies = [ "ansible", diff --git a/requirements.yml b/requirements.yml new file mode 100644 index 0000000..bde903a --- /dev/null +++ b/requirements.yml @@ -0,0 +1,6 @@ +--- +collections: + - name: amazon.aws + version: ">=9.0.0" + - name: community.general + version: ">=10.0.0" diff --git a/roles/clickhouse_backup/defaults/main.yml b/roles/clickhouse_backup/defaults/main.yml new file mode 100644 index 0000000..d66ffab --- /dev/null +++ b/roles/clickhouse_backup/defaults/main.yml @@ -0,0 +1,35 @@ +--- +CLICKHOUSE_BACKUP_USER: "{{ CLICKHOUSE_ADMIN_USER | default('ch_admin') }}" +CLICKHOUSE_BACKUP_PASSWORD: "{{ CLICKHOUSE_ADMIN_PASSWORD | default('') }}" +CLICKHOUSE_BACKUP_ALL_DATABASES: true +CLICKHOUSE_BACKUP_DATABASES: + - prod_xapi +# Docker layout defaults; override in inventory when paths differ. +CLICKHOUSE_BASE_DIR: /opt/clickhouse +CLICKHOUSE_DATA_DIR: /data/clickhouse/data +CLICKHOUSE_CONTAINER_NAME: clickhouse + +CLICKHOUSE_BACKUP_ROOT: /var/edunext_tmp/clickhouse +CLICKHOUSE_BACKUP_DATE: "{{ ansible_date_time.date }}_{{ '%02d' | format(ansible_date_time.hour | int) }}-{{ '%02d' | format(ansible_date_time.minute | int) }}" +CLICKHOUSE_BACKUP_LOCATION: "{{ CLICKHOUSE_BACKUP_ROOT }}/{{ CLICKHOUSE_BACKUP_DATE }}.d" +clickhouse_artifact_path: "{{ CLICKHOUSE_BACKUP_ROOT }}" +CLICKHOUSE_BACKUP_PRE_CLEAN_ROOT: false + +CLICKHOUSE_BACKUP_DISK_NAME: backups +CLICKHOUSE_BACKUP_CONFIGURE_DISK: true +CLICKHOUSE_BACKUP_CONFIG_FILE: backup_disk.xml + +# Override deployment mode explicitly, or inherit from clickhouse_docker inventory. +CLICKHOUSE_BACKUP_USE_DOCKER: false +CLICKHOUSE_BACKUP_CONTAINER_NAME: "{{ CLICKHOUSE_CONTAINER_NAME | default('clickhouse') }}" +CLICKHOUSE_BACKUP_CLIENT_COMMAND: "" +CLICKHOUSE_BACKUP_RESTART_ON_CONFIG_CHANGE: true + +# ClickHouse backup storage options +CLICKHOUSE_BACKUP_STORAGE_OPTIONS: + EXTERNAL_STORAGE_TYPE: aws + EXTERNAL_STORAGE_OPTIONS: + S3_BUCKET_NAME: s3_bucket_name + S3_BUCKET_PATH: clickhouse + AWS_ACCESS_KEY_ID: s3_access_key + AWS_SECRET_ACCESS_KEY: s3_secret_key diff --git a/roles/clickhouse_backup/handlers/main.yml b/roles/clickhouse_backup/handlers/main.yml new file mode 100644 index 0000000..13656c2 --- /dev/null +++ b/roles/clickhouse_backup/handlers/main.yml @@ -0,0 +1,26 @@ +--- +- name: Restart native ClickHouse server + listen: Restart ClickHouse for backup disk + ansible.builtin.service: + name: clickhouse-server + state: restarted + when: + - CLICKHOUSE_BACKUP_RESTART_ON_CONFIG_CHANGE | bool + - not clickhouse_backup_use_docker | bool + +- name: Recreate ClickHouse docker container + listen: Restart ClickHouse for backup disk + ansible.builtin.command: + argv: + - docker + - compose + - -f + - "{{ clickhouse_backup_compose_file }}" + - up + - -d + - --force-recreate + - --remove-orphans + when: + - CLICKHOUSE_BACKUP_RESTART_ON_CONFIG_CHANGE | bool + - clickhouse_backup_use_docker | bool + changed_when: false diff --git a/roles/clickhouse_backup/tasks/backup.yml b/roles/clickhouse_backup/tasks/backup.yml new file mode 100644 index 0000000..3836711 --- /dev/null +++ b/roles/clickhouse_backup/tasks/backup.yml @@ -0,0 +1,58 @@ +--- +- name: Create backup directory on host + ansible.builtin.file: + path: "{{ clickhouse_backup_location }}" + state: directory + mode: "0755" + +- name: Convert ClickHouse backup databases to list for backwards compatibility + ansible.builtin.set_fact: + CLICKHOUSE_BACKUP_DATABASES: "{{ CLICKHOUSE_BACKUP_DATABASES | split }}" + when: CLICKHOUSE_BACKUP_DATABASES is string + +- name: Discover ClickHouse databases to backup + ansible.builtin.shell: | + {{ clickhouse_backup_client_base }} \ + --user {{ CLICKHOUSE_BACKUP_USER | quote }} \ + {% if CLICKHOUSE_BACKUP_PASSWORD | length > 0 %}--password {{ CLICKHOUSE_BACKUP_PASSWORD | quote }}{% endif %} \ + --query "SELECT name FROM system.databases WHERE name NOT IN ('system', 'INFORMATION_SCHEMA', 'information_schema') FORMAT TabSeparated" + register: clickhouse_databases_raw + when: CLICKHOUSE_BACKUP_ALL_DATABASES + changed_when: false + +- name: Set ClickHouse databases list from discovery + ansible.builtin.set_fact: + clickhouse_backup_databases: "{{ clickhouse_databases_raw.stdout_lines | map('trim') | select('ne', '') | list }}" + when: CLICKHOUSE_BACKUP_ALL_DATABASES + +- name: Set ClickHouse databases list from inventory + ansible.builtin.set_fact: + clickhouse_backup_databases: "{{ CLICKHOUSE_BACKUP_DATABASES }}" + when: not CLICKHOUSE_BACKUP_ALL_DATABASES + +- name: Create ClickHouse database backup + ansible.builtin.shell: | + {{ clickhouse_backup_client_base }} \ + --user {{ CLICKHOUSE_BACKUP_USER | quote }} \ + {% if CLICKHOUSE_BACKUP_PASSWORD | length > 0 %}--password {{ CLICKHOUSE_BACKUP_PASSWORD | quote }}{% endif %} \ + --query "BACKUP DATABASE {{ item }} TO Disk('{{ CLICKHOUSE_BACKUP_DISK_NAME }}', '{{ CLICKHOUSE_BACKUP_DATE }}_{{ item }}_clickhouse.zip')" + loop: "{{ clickhouse_backup_databases }}" + register: clickhouse_backup_results + changed_when: clickhouse_backup_results.rc == 0 + +- name: Build ClickHouse backup files list for upload + ansible.builtin.set_fact: + clickhouse_backup_files_to_upload: "{{ clickhouse_backup_files_to_upload | default([]) + [clickhouse_backup_host_path ~ '/' ~ CLICKHOUSE_BACKUP_DATE ~ '_' ~ item ~ '_clickhouse.zip'] }}" + loop: "{{ clickhouse_backup_databases }}" + +- name: Give the server time to recover + ansible.builtin.pause: + minutes: 1 + prompt: Pausing to give the server time to recover + +- name: Upload the backup to a remote storage + ansible.builtin.include_role: + name: storage_backups + vars: + STORAGE_BACKUPS_OPTIONS: "{{ CLICKHOUSE_BACKUP_STORAGE_OPTIONS }}" + STORAGE_BACKUPS_FILES_TO_UPLOAD: "{{ clickhouse_backup_files_to_upload }}" diff --git a/roles/clickhouse_backup/tasks/configure_disk.yml b/roles/clickhouse_backup/tasks/configure_disk.yml new file mode 100644 index 0000000..ede1c67 --- /dev/null +++ b/roles/clickhouse_backup/tasks/configure_disk.yml @@ -0,0 +1,38 @@ +--- +- name: Ensure ClickHouse backup host path exists + ansible.builtin.file: + path: "{{ clickhouse_backup_host_path }}" + state: directory + owner: "{{ CLICKHOUSE_CONTAINER_UID | default('101') }}" + group: "{{ CLICKHOUSE_CONTAINER_GID | default('101') }}" + mode: "0755" + when: clickhouse_backup_use_docker | bool + +- name: Ensure ClickHouse backup host path exists for native deployments + ansible.builtin.file: + path: "{{ clickhouse_backup_host_path }}" + state: directory + owner: clickhouse + group: clickhouse + mode: "0755" + when: not clickhouse_backup_use_docker | bool + +- name: Ensure ClickHouse backup config directory exists + ansible.builtin.file: + path: "{{ clickhouse_backup_config_path }}" + state: directory + owner: root + group: root + mode: "0755" + +- name: Configure ClickHouse backup disk + ansible.builtin.template: + src: backup_disk.xml.j2 + dest: "{{ clickhouse_backup_config_path }}/{{ CLICKHOUSE_BACKUP_CONFIG_FILE }}" + owner: root + group: root + mode: "0644" + notify: Restart ClickHouse for backup disk + +- name: Flush handlers to apply backup disk configuration + ansible.builtin.meta: flush_handlers diff --git a/roles/clickhouse_backup/tasks/main.yml b/roles/clickhouse_backup/tasks/main.yml new file mode 100644 index 0000000..7adc293 --- /dev/null +++ b/roles/clickhouse_backup/tasks/main.yml @@ -0,0 +1,77 @@ +--- +- name: Resolve ClickHouse backup deployment mode + ansible.builtin.set_fact: + clickhouse_backup_use_docker: "{{ CLICKHOUSE_BACKUP_USE_DOCKER | default(CLICKHOUSE_INSTALL_DOCKER | default(false)) | bool }}" + +- name: Resolve ClickHouse backup paths for Docker deployments + ansible.builtin.set_fact: + clickhouse_backup_host_path: "{{ CLICKHOUSE_DATA_DIR }}/backups" + clickhouse_backup_server_path: /var/lib/clickhouse/backups + clickhouse_backup_config_path: "{{ CLICKHOUSE_BASE_DIR }}/config.d" + clickhouse_backup_container_name: "{{ CLICKHOUSE_BACKUP_CONTAINER_NAME | default(CLICKHOUSE_CONTAINER_NAME | default('clickhouse')) }}" + clickhouse_backup_compose_file: "{{ CLICKHOUSE_BASE_DIR }}/docker-compose.yml" + when: clickhouse_backup_use_docker | bool + +- name: Resolve ClickHouse backup paths for native deployments + ansible.builtin.set_fact: + clickhouse_backup_host_path: "{{ CLICKHOUSE_BACKUP_ROOT }}" + clickhouse_backup_server_path: "{{ CLICKHOUSE_BACKUP_ROOT }}" + clickhouse_backup_config_path: /etc/clickhouse-server/config.d + clickhouse_backup_container_name: "{{ CLICKHOUSE_BACKUP_CONTAINER_NAME | default('clickhouse') }}" + clickhouse_backup_compose_file: "" + when: not clickhouse_backup_use_docker | bool + +- name: Set ClickHouse backup artifact paths + ansible.builtin.set_fact: + clickhouse_artifact_path: "{{ clickhouse_backup_host_path }}" + clickhouse_backup_location: "{{ clickhouse_backup_host_path }}/{{ CLICKHOUSE_BACKUP_DATE }}.d" + +- name: Set ClickHouse client command + ansible.builtin.set_fact: + clickhouse_backup_client_base: >- + {{ + CLICKHOUSE_BACKUP_CLIENT_COMMAND + if CLICKHOUSE_BACKUP_CLIENT_COMMAND | length > 0 + else ( + 'docker exec ' ~ clickhouse_backup_container_name ~ ' clickhouse-client' + if clickhouse_backup_use_docker | bool + else 'clickhouse-client' + ) + }} + +- name: Clean the backup root path before starting the routine + ansible.builtin.file: + state: absent + path: "{{ clickhouse_artifact_path }}/" + when: | + CLICKHOUSE_BACKUP_PRE_CLEAN_ROOT + and clickhouse_artifact_path is defined + and clickhouse_artifact_path != "" + and CLICKHOUSE_BACKUP_STORAGE_OPTIONS.EXTERNAL_STORAGE_TYPE != "" + +# Since ubuntu 24/Debian 12 have problems with the backup playbook because of PEP 668 when updating the libraries, we proceed +# to create a task to ignore EXTERNALLY-MANAGED +- name: Get python system info + community.general.python_requirements_info: + register: python_info + +- name: Ignore PEP 668 (Ubuntu/Debian) + ansible.builtin.file: + path: "/usr/lib/python{{ python_info.python_version_info.major }}.{{ python_info.python_version_info.minor }}/EXTERNALLY-MANAGED" + state: absent + when: + - (ansible_distribution == 'Ubuntu' and ansible_distribution_version is version('24', '>=')) or + (ansible_distribution == 'Debian' and ansible_distribution_major_version | int >= 12) + +- name: Configure ClickHouse backup disk + ansible.builtin.include_tasks: configure_disk.yml + when: CLICKHOUSE_BACKUP_CONFIGURE_DISK | bool + +- name: Launch ClickHouse backups + ansible.builtin.include_tasks: backup.yml + +- name: Clean artifact path + ansible.builtin.file: + state: absent + path: "{{ clickhouse_artifact_path }}/" + when: clickhouse_artifact_path is defined and clickhouse_artifact_path != "" and CLICKHOUSE_BACKUP_STORAGE_OPTIONS.EXTERNAL_STORAGE_TYPE != "" diff --git a/roles/clickhouse_backup/templates/backup_disk.xml.j2 b/roles/clickhouse_backup/templates/backup_disk.xml.j2 new file mode 100644 index 0000000..55dd725 --- /dev/null +++ b/roles/clickhouse_backup/templates/backup_disk.xml.j2 @@ -0,0 +1,14 @@ + + + + <{{ CLICKHOUSE_BACKUP_DISK_NAME }}> + local + {{ clickhouse_backup_server_path }}/ + + + + + {{ CLICKHOUSE_BACKUP_DISK_NAME }} + {{ clickhouse_backup_server_path }}/ + + diff --git a/scripts/ssm-smoke-test.sh b/scripts/ssm-smoke-test.sh new file mode 100755 index 0000000..584ced1 --- /dev/null +++ b/scripts/ssm-smoke-test.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Smoke test for AWS SSM connectivity using test_os_info.yml. +# Usage: +# AWS_PROFILE= AWS_SSM_BUCKET= ./scripts/ssm-smoke-test.sh [inventory-path] +set -euo pipefail + +INSTANCE_ID="${1:?Usage: AWS_PROFILE= AWS_SSM_BUCKET= $0 [inventory-path]}" +INVENTORY_PATH="${2:-inventories/examples/ssm/hosts.ini}" +AWS_REGION="${AWS_REGION:-us-west-2}" +AWS_PROFILE_NAME="${AWS_PROFILE:-${AWS_DEFAULT_PROFILE:-}}" +AWS_SSM_BUCKET="${AWS_SSM_BUCKET:-}" +ANSIBLE_REMOTE_TEMP="${ANSIBLE_REMOTE_TEMP:-/tmp/ansible-ssm}" +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +if [[ -z "${AWS_SSM_BUCKET}" ]]; then + echo "ERROR: AWS_SSM_BUCKET is required by the amazon.aws.aws_ssm connection plugin." >&2 + echo "Set it to an S3 bucket that the controller profile can read/write/delete." >&2 + exit 1 +fi + +TMP_INVENTORY="$(mktemp)" +trap 'rm -f "${TMP_INVENTORY}"' EXIT + +sed "s/INSTANCE_ID/${INSTANCE_ID}/g" "${REPO_ROOT}/${INVENTORY_PATH}" > "${TMP_INVENTORY}" + +cd "${REPO_ROOT}" +make requirements + +ANSIBLE_EXTRA_VARS=( + -e "ansible_aws_ssm_region=${AWS_REGION}" + -e "ansible_aws_ssm_bucket_name=${AWS_SSM_BUCKET}" + -e "ansible_remote_tmp=${ANSIBLE_REMOTE_TEMP}" +) +if [[ -n "${AWS_PROFILE_NAME}" ]]; then + ANSIBLE_EXTRA_VARS+=(-e "ansible_aws_ssm_profile=${AWS_PROFILE_NAME}") + echo "Using AWS profile: ${AWS_PROFILE_NAME}" +else + echo "Using default AWS credential provider chain" +fi +echo "Using SSM transfer bucket: ${AWS_SSM_BUCKET}" +echo "Using remote temp path: ${ANSIBLE_REMOTE_TEMP}" + +uv run ansible-playbook playbooks/test_os_info.yml \ + -i "${TMP_INVENTORY}" \ + --limit test_hosts \ + "${ANSIBLE_EXTRA_VARS[@]}" \ + -v diff --git a/uv.lock b/uv.lock index 59d094e..a31a891 100644 --- a/uv.lock +++ b/uv.lock @@ -84,7 +84,7 @@ wheels = [ [[package]] name = "atlas-ansible-utils" -version = "21.1.1" +version = "21.1.2" source = { virtual = "." } dependencies = [ { name = "ansible" },