feat: clickhouse backups

.dockerignore

@@ -8,3 +8,4 @@
 !ansible.cfg
 !hosts.ini
 !playbooks/
+!requirements.yml

Dockerfile

@@ -7,8 +7,14 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     apt update
     apt install -y \
         build-essential \
+        curl \
         git \
+        unzip \
     ;
+    curl -fsSL "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_64bit/session-manager-plugin.deb" \
+        -o /tmp/session-manager-plugin.deb
+    dpkg -i /tmp/session-manager-plugin.deb
+    rm /tmp/session-manager-plugin.deb
 EOCMD
 
 FROM minimal AS dependencies
@@ -41,10 +47,16 @@ RUN <<EOCMD
     chown -R 1000:1000 /runner;
 EOCMD
 
+COPY requirements.yml .
+RUN mkdir -p /usr/share/ansible/collections \
+    && ansible-galaxy collection install -r requirements.yml -p /usr/share/ansible/collections \
+    && chown -R runner:runner /usr/share/ansible/collections
+
 COPY . .
 
 USER 1000:1000
 
+ENV ANSIBLE_COLLECTIONS_PATHS="/usr/share/ansible/collections"
 # For now, we prefer skipping the host key checking
 ENV ANSIBLE_HOST_KEY_CHECKING="no"
 ENV ATLAS_ANSIBLE_PLAYBOOK="test_os_info.yml"

Makefile

@@ -1,5 +1,9 @@
 .DEFAULT_GOAL := help
-.PHONY: help
+.PHONY: help requirements
+
+requirements: ## Install Python dependencies and Ansible collections
+	uv sync --no-dev
+	uv run ansible-galaxy collection install -r requirements.yml
 
 quality: ## Run linters
 	uv run ansible-lint --exclude charts -c .ansible-lint.yml

README.md

@@ -104,6 +104,80 @@ In this case, we can override the default variables using:
 
 ---
 
+## Using AWS SSM instead of SSH
+
+When target instances are reachable only through AWS Systems Manager, configure the inventory with the `aws_ssm` connection plugin (provided by the `amazon.aws` collection, installed via `requirements.yml`).
+
+Prerequisites on the controller (CNC, Docker image, or Kubernetes runner):
+
+- AWS credentials with `ssm:StartSession` and related permissions on the target instances.
+- The [Session Manager plugin](https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager-working-with-install-plugin.html) installed (included in the Docker image).
+- Target instances registered in SSM with an agent and instance profile.
+
+Example inventory entry:
+
+```ini
+[test_hosts]
+example-db-node ansible_host=i-0123456789abcdef0 ansible_connection=aws_ssm ansible_aws_ssm_region=us-west-2 ansible_remote_tmp=/tmp/ansible-ssm
+```
+
+The `aws_ssm` connection plugin requires an S3 bucket to transfer Ansible modules to the target instance, even for fact gathering. Set a staging bucket in group vars or pass it as an extra var:
+
+```yaml
+ansible_aws_ssm_bucket_name: my-ssm-staging-bucket
+```
+
+Install dependencies locally:
+
+```bash
+make requirements
+```
+
+Run a connectivity smoke test:
+
+```bash
+AWS_PROFILE=my-aws-profile AWS_SSM_BUCKET=my-ssm-staging-bucket \
+  ./scripts/ssm-smoke-test.sh i-0123456789abcdef0
+```
+
+Or run the playbook directly:
+
+```bash
+uv run ansible-playbook playbooks/test_os_info.yml \
+  -i inventories/examples/ssm/hosts.ini \
+  --limit test_hosts \
+  -e "ansible_aws_ssm_profile=my-aws-profile" \
+  -e "ansible_aws_ssm_bucket_name=my-ssm-staging-bucket" \
+  -e "ansible_remote_tmp=/tmp/ansible-ssm" \
+  -v
+```
+
+Run Mongo backups over SSM:
+
+```bash
+ansible-playbook playbooks/mongo_backup.yml \
+  -i /path/to/inventory/hosts.ini \
+  --limit mongo_node_2 \
+  -v
+```
+
+When uploading backups to S3 from the target host, you can omit `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` in `MONGO_BACKUP_STORAGE_OPTIONS` if the EC2 instance profile already has access to the backup bucket.
+
+Run ClickHouse backups over SSM:
+
+```bash
+ansible-playbook playbooks/clickhouse_backup.yml \
+  -i /path/to/inventory/hosts.ini \
+  --limit clickhouse_replica_2 \
+  -v
+```
+
+The `clickhouse_backup` role uses ClickHouse native `BACKUP DATABASE ... TO Disk(...)` to write artifacts on the target host, then uploads them with the shared `storage_backups` role. When uploading to S3, you can omit `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` in `CLICKHOUSE_BACKUP_STORAGE_OPTIONS` if the EC2 instance profile already has access to the backup bucket.
+
+For Docker deployments managed by the `clickhouse_docker` role, the backup role writes the allowed backup disk configuration under `{{ CLICKHOUSE_BASE_DIR }}/config.d`, stores artifacts in `{{ CLICKHOUSE_DATA_DIR }}/backups` on the host (mounted as `/var/lib/clickhouse/backups` inside the container), and recreates the container with `docker compose` when the backup disk configuration changes. Set `CLICKHOUSE_BACKUP_USE_DOCKER: true` or rely on `CLICKHOUSE_INSTALL_DOCKER: true` from inventory when backups run against a Docker-based ClickHouse cluster.
+
+---
+
 ## Using in Kubernetes
 
 To run the atlas-ansible-utils playbooks to provision external servers from Kubernetes, jobs or cronjobs can be implemented using the atlas-ansible-utils Docker image. Let's look at the following example:

changelog.d/20260617_clickhouse_backup.md

@@ -0,0 +1,7 @@
+### Added
+
+- A new `clickhouse_backup` role and playbook that creates ClickHouse native
+  database backups on local disk and uploads them to S3 or Azure using the
+  existing `storage_backups` role. The role integrates with the Docker-based
+  `clickhouse_docker` deployment layout and also supports native package
+  installations.

inventories/examples/ssm/hosts.ini

@@ -0,0 +1,13 @@
+# Example inventory for AWS SSM connectivity.
+# Replace INSTANCE_ID with the target EC2 instance id.
+# For local runs with a named AWS profile, prefer:
+# AWS_PROFILE=<profile> AWS_SSM_BUCKET=<bucket> ./scripts/ssm-smoke-test.sh INSTANCE_ID
+
+[test_hosts]
+example-db-node ansible_host=INSTANCE_ID ansible_connection=aws_ssm ansible_aws_ssm_region=us-west-2 ansible_remote_tmp=/tmp/ansible-ssm
+
+[mongo_node_2]
+example-db-node
+
+[mongo_servers]
+mongo_node_2

playbooks/clickhouse_backup.yml

@@ -0,0 +1,10 @@
+---
+# Description: Playbook to launch ClickHouse backups
+- name: Launch backups
+  hosts: clickhouse_servers
+  become: true
+  gather_facts: true
+  roles:
+    - role: clickhouse_backup
+      tags:
+        - clickhouse_backup

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "atlas-ansible-utils"
-version = "21.1.1"
+version = "21.1.2"
 requires-python = ">=3.12"
 dependencies = [
 	"ansible",

requirements.yml

@@ -0,0 +1,6 @@
+---
+collections:
+  - name: amazon.aws
+    version: ">=9.0.0"
+  - name: community.general
+    version: ">=10.0.0"

roles/clickhouse_backup/defaults/main.yml

@@ -0,0 +1,35 @@
+---
+CLICKHOUSE_BACKUP_USER: "{{ CLICKHOUSE_ADMIN_USER | default('ch_admin') }}"
+CLICKHOUSE_BACKUP_PASSWORD: "{{ CLICKHOUSE_ADMIN_PASSWORD | default('') }}"
+CLICKHOUSE_BACKUP_ALL_DATABASES: true
+CLICKHOUSE_BACKUP_DATABASES:
+  - prod_xapi
+# Docker layout defaults; override in inventory when paths differ.
+CLICKHOUSE_BASE_DIR: /opt/clickhouse
+CLICKHOUSE_DATA_DIR: /data/clickhouse/data
+CLICKHOUSE_CONTAINER_NAME: clickhouse
+
+CLICKHOUSE_BACKUP_ROOT: /var/edunext_tmp/clickhouse
+CLICKHOUSE_BACKUP_DATE: "{{ ansible_date_time.date }}_{{ '%02d' | format(ansible_date_time.hour | int) }}-{{ '%02d' | format(ansible_date_time.minute | int) }}"
+CLICKHOUSE_BACKUP_LOCATION: "{{ CLICKHOUSE_BACKUP_ROOT }}/{{ CLICKHOUSE_BACKUP_DATE }}.d"
+clickhouse_artifact_path: "{{ CLICKHOUSE_BACKUP_ROOT }}"
+CLICKHOUSE_BACKUP_PRE_CLEAN_ROOT: false
+
+CLICKHOUSE_BACKUP_DISK_NAME: backups
+CLICKHOUSE_BACKUP_CONFIGURE_DISK: true
+CLICKHOUSE_BACKUP_CONFIG_FILE: backup_disk.xml
+
+# Override deployment mode explicitly, or inherit from clickhouse_docker inventory.
+CLICKHOUSE_BACKUP_USE_DOCKER: false
+CLICKHOUSE_BACKUP_CONTAINER_NAME: "{{ CLICKHOUSE_CONTAINER_NAME | default('clickhouse') }}"
+CLICKHOUSE_BACKUP_CLIENT_COMMAND: ""
+CLICKHOUSE_BACKUP_RESTART_ON_CONFIG_CHANGE: true
+
+# ClickHouse backup storage options
+CLICKHOUSE_BACKUP_STORAGE_OPTIONS:
+  EXTERNAL_STORAGE_TYPE: aws
+  EXTERNAL_STORAGE_OPTIONS:
+    S3_BUCKET_NAME: s3_bucket_name
+    S3_BUCKET_PATH: clickhouse
+    AWS_ACCESS_KEY_ID: s3_access_key
+    AWS_SECRET_ACCESS_KEY: s3_secret_key

roles/clickhouse_backup/handlers/main.yml

@@ -0,0 +1,26 @@
+---
+- name: Restart native ClickHouse server
+  listen: Restart ClickHouse for backup disk
+  ansible.builtin.service:
+    name: clickhouse-server
+    state: restarted
+  when:
+    - CLICKHOUSE_BACKUP_RESTART_ON_CONFIG_CHANGE | bool
+    - not clickhouse_backup_use_docker | bool
+
+- name: Recreate ClickHouse docker container
+  listen: Restart ClickHouse for backup disk
+  ansible.builtin.command:
+    argv:
+      - docker
+      - compose
+      - -f
+      - "{{ clickhouse_backup_compose_file }}"
+      - up
+      - -d
+      - --force-recreate
+      - --remove-orphans
+  when:
+    - CLICKHOUSE_BACKUP_RESTART_ON_CONFIG_CHANGE | bool
+    - clickhouse_backup_use_docker | bool
+  changed_when: false

roles/clickhouse_backup/tasks/backup.yml

@@ -0,0 +1,58 @@
+---
+- name: Create backup directory on host
+  ansible.builtin.file:
+    path: "{{ clickhouse_backup_location }}"
+    state: directory
+    mode: "0755"
+
+- name: Convert ClickHouse backup databases to list for backwards compatibility
+  ansible.builtin.set_fact:
+    CLICKHOUSE_BACKUP_DATABASES: "{{ CLICKHOUSE_BACKUP_DATABASES | split }}"
+  when: CLICKHOUSE_BACKUP_DATABASES is string
+
+- name: Discover ClickHouse databases to backup
+  ansible.builtin.shell: |
+    {{ clickhouse_backup_client_base }} \
+      --user {{ CLICKHOUSE_BACKUP_USER | quote }} \
+      {% if CLICKHOUSE_BACKUP_PASSWORD | length > 0 %}--password {{ CLICKHOUSE_BACKUP_PASSWORD | quote }}{% endif %} \
+      --query "SELECT name FROM system.databases WHERE name NOT IN ('system', 'INFORMATION_SCHEMA', 'information_schema') FORMAT TabSeparated"
+  register: clickhouse_databases_raw
+  when: CLICKHOUSE_BACKUP_ALL_DATABASES
+  changed_when: false
+
+- name: Set ClickHouse databases list from discovery
+  ansible.builtin.set_fact:
+    clickhouse_backup_databases: "{{ clickhouse_databases_raw.stdout_lines | map('trim') | select('ne', '') | list }}"
+  when: CLICKHOUSE_BACKUP_ALL_DATABASES
+
+- name: Set ClickHouse databases list from inventory
+  ansible.builtin.set_fact:
+    clickhouse_backup_databases: "{{ CLICKHOUSE_BACKUP_DATABASES }}"
+  when: not CLICKHOUSE_BACKUP_ALL_DATABASES
+
+- name: Create ClickHouse database backup
+  ansible.builtin.shell: |
+    {{ clickhouse_backup_client_base }} \
+      --user {{ CLICKHOUSE_BACKUP_USER | quote }} \
+      {% if CLICKHOUSE_BACKUP_PASSWORD | length > 0 %}--password {{ CLICKHOUSE_BACKUP_PASSWORD | quote }}{% endif %} \
+      --query "BACKUP DATABASE {{ item }} TO Disk('{{ CLICKHOUSE_BACKUP_DISK_NAME }}', '{{ CLICKHOUSE_BACKUP_DATE }}_{{ item }}_clickhouse.zip')"
+  loop: "{{ clickhouse_backup_databases }}"
+  register: clickhouse_backup_results
+  changed_when: clickhouse_backup_results.rc == 0
+
+- name: Build ClickHouse backup files list for upload
+  ansible.builtin.set_fact:
+    clickhouse_backup_files_to_upload: "{{ clickhouse_backup_files_to_upload | default([]) + [clickhouse_backup_host_path ~ '/' ~ CLICKHOUSE_BACKUP_DATE ~ '_' ~ item ~ '_clickhouse.zip'] }}"
+  loop: "{{ clickhouse_backup_databases }}"
+
+- name: Give the server time to recover
+  ansible.builtin.pause:
+    minutes: 1
+    prompt: Pausing to give the server time to recover
+
+- name: Upload the backup to a remote storage
+  ansible.builtin.include_role:
+    name: storage_backups
+  vars:
+    STORAGE_BACKUPS_OPTIONS: "{{ CLICKHOUSE_BACKUP_STORAGE_OPTIONS }}"
+    STORAGE_BACKUPS_FILES_TO_UPLOAD: "{{ clickhouse_backup_files_to_upload }}"

roles/clickhouse_backup/tasks/configure_disk.yml

@@ -0,0 +1,38 @@
+---
+- name: Ensure ClickHouse backup host path exists
+  ansible.builtin.file:
+    path: "{{ clickhouse_backup_host_path }}"
+    state: directory
+    owner: "{{ CLICKHOUSE_CONTAINER_UID | default('101') }}"
+    group: "{{ CLICKHOUSE_CONTAINER_GID | default('101') }}"
+    mode: "0755"
+  when: clickhouse_backup_use_docker | bool
+
+- name: Ensure ClickHouse backup host path exists for native deployments
+  ansible.builtin.file:
+    path: "{{ clickhouse_backup_host_path }}"
+    state: directory
+    owner: clickhouse
+    group: clickhouse
+    mode: "0755"
+  when: not clickhouse_backup_use_docker | bool
+
+- name: Ensure ClickHouse backup config directory exists
+  ansible.builtin.file:
+    path: "{{ clickhouse_backup_config_path }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Configure ClickHouse backup disk
+  ansible.builtin.template:
+    src: backup_disk.xml.j2
+    dest: "{{ clickhouse_backup_config_path }}/{{ CLICKHOUSE_BACKUP_CONFIG_FILE }}"
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Restart ClickHouse for backup disk
+
+- name: Flush handlers to apply backup disk configuration
+  ansible.builtin.meta: flush_handlers