From 70172a299574aa6b8501157e0b0609ee2839b5da Mon Sep 17 00:00:00 2001 From: Alik Kurdyukov Date: Fri, 13 Jun 2025 22:49:55 +0400 Subject: [PATCH 01/19] Fixing role for Rocky 8/9 --- molecule/default/molecule.yml | 4 ++++ tasks/noauto_install_pip.yml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml index 5469447..80087a7 100644 --- a/molecule/default/molecule.yml +++ b/molecule/default/molecule.yml @@ -14,6 +14,10 @@ platforms: image: debian:bookworm - name: ubuntu-latest image: ubuntu:latest + - name: rockylinux-8 + image: rockylinux/rockylinux:8 + - name: rockylinux-9 + image: rockylinux/rockylinux:9 provisioner: name: ansible verifier: diff --git a/tasks/noauto_install_pip.yml b/tasks/noauto_install_pip.yml index 770cf09..3a1fcd2 100644 --- a/tasks/noauto_install_pip.yml +++ b/tasks/noauto_install_pip.yml @@ -5,7 +5,7 @@ community.general.dnf_config_manager: name: crb state: enabled - when: ansible_os_family == 'RedHat' and ansible_distribution != 'Fedora' + when: ansible_os_family == 'RedHat' and ansible_distribution != 'Fedora' and ansible_distribution != 'Rocky' - name: Install build dependencies ansible.builtin.package: From b206824a38a8a11bb5296bb7cefa7dfc5801f7f3 Mon Sep 17 00:00:00 2001 From: Alik Kurdyukov Date: Fri, 13 Jun 2025 23:18:54 +0400 Subject: [PATCH 02/19] Fix for Rocky 9 --- molecule/default/molecule.yml | 5 +++-- tasks/noauto_install_pip.yml | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml index 80087a7..0333e5d 100644 --- a/molecule/default/molecule.yml +++ b/molecule/default/molecule.yml @@ -14,8 +14,9 @@ platforms: image: debian:bookworm - name: ubuntu-latest image: ubuntu:latest - - name: rockylinux-8 - image: rockylinux/rockylinux:8 + # TODO: setup ansible<10 for this + # - name: rockylinux-8 + # image: rockylinux/rockylinux:8 - name: rockylinux-9 image: rockylinux/rockylinux:9 provisioner: diff --git a/tasks/noauto_install_pip.yml b/tasks/noauto_install_pip.yml index 3a1fcd2..0482ee7 100644 --- a/tasks/noauto_install_pip.yml +++ b/tasks/noauto_install_pip.yml @@ -5,7 +5,10 @@ community.general.dnf_config_manager: name: crb state: enabled - when: ansible_os_family == 'RedHat' and ansible_distribution != 'Fedora' and ansible_distribution != 'Rocky' + when: + - ansible_os_family == 'RedHat' + - ansible_distribution != 'Fedora' + - ansible_distribution != 'Rocky' or ansible_distribution_major_version == "9" - name: Install build dependencies ansible.builtin.package: From 2daac5a8161c32eb0bdff4a21dfa57c10a029d85 Mon Sep 17 00:00:00 2001 From: Alik Kurdyukov Date: Fri, 13 Jun 2025 23:39:33 +0400 Subject: [PATCH 03/19] rocky8 specifics added --- .github/workflows/main.yml | 16 ++++++++++ molecule/rocky8/Dockerfile.j2 | 23 +++++++++++++++ molecule/rocky8/INSTALL.rst | 22 ++++++++++++++ molecule/rocky8/converge.yml | 55 +++++++++++++++++++++++++++++++++++ molecule/rocky8/molecule.yml | 16 ++++++++++ molecule/rocky8/verify.yml | 13 +++++++++ requirements-dev-rocky8.txt | 4 +++ 7 files changed, 149 insertions(+) create mode 100644 molecule/rocky8/Dockerfile.j2 create mode 100644 molecule/rocky8/INSTALL.rst create mode 100644 molecule/rocky8/converge.yml create mode 100644 molecule/rocky8/molecule.yml create mode 100644 molecule/rocky8/verify.yml create mode 100644 requirements-dev-rocky8.txt diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ef664fc..6af84de 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,3 +19,19 @@ jobs: # uses: mxschmitt/action-tmate@v3.5 - name: Test using Molecule run: molecule test + test-rocky8: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Install Molecule + run: | + pip install -U pip setuptools wheel + pip install -r requirements-dev-rocky8.txt + # - name: Debugging with tmate + # uses: mxschmitt/action-tmate@v3.5 + - name: Test using Molecule + run: molecule test -s rocky8 diff --git a/molecule/rocky8/Dockerfile.j2 b/molecule/rocky8/Dockerfile.j2 new file mode 100644 index 0000000..0b331c5 --- /dev/null +++ b/molecule/rocky8/Dockerfile.j2 @@ -0,0 +1,23 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +{% if item.env is defined %} +{% for var, value in item.env.items() %} +{% if value %} +ENV {{ var }} {{ value }} +{% endif %} +{% endfor %} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python3 python3-pip sudo bash ca-certificates iproute2 python3-apt aptitude && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install /usr/bin/python3 /usr/bin/python3-config /usr/bin/dnf-3 sudo bash iproute && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y /usr/bin/python /usr/bin/python2-config sudo yum-plugin-ovl bash iproute && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml iproute2 && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ + elif [ $(command -v pacman) ]; then pacman --noconfirm -Suy python python-pip sudo openssh; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates iproute2 && xbps-remove -O; fi diff --git a/molecule/rocky8/INSTALL.rst b/molecule/rocky8/INSTALL.rst new file mode 100644 index 0000000..6a44bde --- /dev/null +++ b/molecule/rocky8/INSTALL.rst @@ -0,0 +1,22 @@ +******* +Docker driver installation guide +******* + +Requirements +============ + +* Docker Engine + +Install +======= + +Please refer to the `Virtual environment`_ documentation for installation best +practices. If not using a virtual environment, please consider passing the +widely recommended `'--user' flag`_ when invoking ``pip``. + +.. _Virtual environment: https://virtualenv.pypa.io/en/latest/ +.. _'--user' flag: https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site + +.. code-block:: bash + + $ pip install 'molecule[docker]' diff --git a/molecule/rocky8/converge.yml b/molecule/rocky8/converge.yml new file mode 100644 index 0000000..e0108e3 --- /dev/null +++ b/molecule/rocky8/converge.yml @@ -0,0 +1,55 @@ +--- +- name: Converge + hosts: all + pre_tasks: + - name: Set ssh server package name for non-Archlinux ansible_os_family + set_fact: + openssh_package: "openssh-server" + pip3_extra_args: "" + when: ansible_os_family != "Archlinux" + + - name: Set ssh server package name and pip3 argument for Archlinux ansible_os_family + set_fact: + openssh_package: "openssh" + pip3_extra_args: "--break-system-packages" + when: ansible_os_family == "Archlinux" + + - name: Install openssh + package: + name: "{{ openssh_package }}" + state: present + + - name: Enable EPEL for yamllint + package: + name: epel-release + state: present + when: ansible_os_family == 'RedHat' and ansible_distribution != 'Fedora' + + - name: Install yamllint + package: + name: yamllint + state: present + + roles: + - role: borgbase.ansible_role_borgbackup + borg_install_method: pip + borgmatic_timer: cron + borg_repository: m5vz9gp4@m5vz9gp4.repo.borgbase.com:repo + borg_encryption_passphrase: CHANGEME + borg_source_directories: + - /srv/www + - /var/lib/automysqlbackup + borg_exclude_patterns: + - /srv/www/old-sites + borg_retention_policy: + keep_hourly: 3 + keep_daily: 7 + keep_weekly: 4 + keep_monthly: 6 + borgmatic_hooks: + before_backup: + - echo "`date` - Starting backup." + postgresql_databases: + - name: users + hostname: database1.example.org + port: 5433 diff --git a/molecule/rocky8/molecule.yml b/molecule/rocky8/molecule.yml new file mode 100644 index 0000000..bfb867e --- /dev/null +++ b/molecule/rocky8/molecule.yml @@ -0,0 +1,16 @@ +--- +dependency: + name: galaxy +driver: + name: docker +platforms: + - name: rockylinux-8 + image: rockylinux/rockylinux:8 +provisioner: + name: ansible +verifier: + name: ansible +lint: | + set -e + yamllint . + ansible-lint . diff --git a/molecule/rocky8/verify.yml b/molecule/rocky8/verify.yml new file mode 100644 index 0000000..f816e3f --- /dev/null +++ b/molecule/rocky8/verify.yml @@ -0,0 +1,13 @@ +--- +- name: Verify + hosts: all + tasks: + - name: Ensure Borgmatic is installed correctly + command: borgmatic --version + + - name: Ensure Borg is installed correctly + command: borgmatic borg --version + + - name: Ensure produced YAML is valid + command: | + yamllint -d "{extends: relaxed, rules: {line-length: {max: 120}}}" /etc/borgmatic/config.yaml diff --git a/requirements-dev-rocky8.txt b/requirements-dev-rocky8.txt new file mode 100644 index 0000000..50d35e5 --- /dev/null +++ b/requirements-dev-rocky8.txt @@ -0,0 +1,4 @@ +ansible<10 +ansible-lint +molecule +molecule-plugins[docker] From fa6acbeaa86b2ae034a33480a8079019c5001f46 Mon Sep 17 00:00:00 2001 From: Alik Kurdyukov Date: Sat, 14 Jun 2025 00:38:38 +0400 Subject: [PATCH 04/19] Local running Rocky8 --- tasks/noauto_install_pip.yml | 6 ++++++ vars/Rocky-8.yml | 27 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 vars/Rocky-8.yml diff --git a/tasks/noauto_install_pip.yml b/tasks/noauto_install_pip.yml index 0482ee7..d97a387 100644 --- a/tasks/noauto_install_pip.yml +++ b/tasks/noauto_install_pip.yml @@ -10,6 +10,12 @@ - ansible_distribution != 'Fedora' - ansible_distribution != 'Rocky' or ansible_distribution_major_version == "9" + - name: Ensure that devel repository in enabled for Rocky 8 (need xxxhash-devel) + community.general.dnf_config_manager: + name: devel + state: enabled + when: ansible_distribution == 'Rocky' and ansible_distribution_major_version == "8" + - name: Install build dependencies ansible.builtin.package: name: "{{ borg_pip_packages }}" diff --git a/vars/Rocky-8.yml b/vars/Rocky-8.yml new file mode 100644 index 0000000..48e492a --- /dev/null +++ b/vars/Rocky-8.yml @@ -0,0 +1,27 @@ +borg_dep_packages: + - openssh-clients + - python3.9 + +borg_cron_package: cronie + +borg_pip_packages: + - libacl-devel + - libacl + - gcc + - gcc-c++ + - openssl-devel + - lz4-devel + - libzstd-devel + - xxhash-devel + - python39-pip + - python39-wheel + - python39-devel + - python39-setuptools + - python3-virtualenv + +borg_distro_packages: + - borgbackup + - borgmatic + +python_bin: python3.9 +pip_bin: pip3.9 From 5d0d42d2d27c18b332d15f8b74560cc916c0011d Mon Sep 17 00:00:00 2001 From: Alik Kurdyukov Date: Sat, 14 Jun 2025 13:37:09 +0400 Subject: [PATCH 05/19] Use absolute path for borgmatic --- tasks/noauto_install_package.yml | 4 ++++ tasks/noauto_install_pip.yml | 5 +++++ templates/borgmatic.service.j2 | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tasks/noauto_install_package.yml b/tasks/noauto_install_package.yml index 5b9346a..7c284c9 100644 --- a/tasks/noauto_install_package.yml +++ b/tasks/noauto_install_package.yml @@ -18,4 +18,8 @@ name: "{{ item }}" state: present loop: "{{ borg_distro_packages }}" + + - name: Set absolute path to /usr/bin + ansible.builtin.set_fact: + borg_abs_path: "/usr/bin" ... diff --git a/tasks/noauto_install_pip.yml b/tasks/noauto_install_pip.yml index d97a387..c2e3ddf 100644 --- a/tasks/noauto_install_pip.yml +++ b/tasks/noauto_install_pip.yml @@ -63,4 +63,9 @@ borg "$@" dest: /usr/local/bin/borg mode: "0755" + + - name: Set absolute path to /usr/bin + ansible.builtin.set_fact: + borg_abs_path: "/usr/local/bin" + ... diff --git a/templates/borgmatic.service.j2 b/templates/borgmatic.service.j2 index 51da9f4..0994e9d 100644 --- a/templates/borgmatic.service.j2 +++ b/templates/borgmatic.service.j2 @@ -12,7 +12,7 @@ ConditionACPower=true [Service] Type=oneshot User={{ borg_user }} -ExecStart=borgmatic -c /etc/borgmatic/{{ borgmatic_config_name }} {{ borgmatic_timer_flags }} +ExecStart={{ borg_abs_path }}/borgmatic -c /etc/borgmatic/{{ borgmatic_config_name }} {{ borgmatic_timer_flags }} # Source: https://projects.torsion.org/borgmatic-collective/borgmatic/raw/branch/master/sample/systemd/borgmatic.service # Security settings for systemd running as root, optional but recommended to improve security. You From 31fdbfb89ac6fdb25bda2887a3bdc1f712b35c73 Mon Sep 17 00:00:00 2001 From: Alik Kurdyukov Date: Sat, 14 Jun 2025 13:44:50 +0400 Subject: [PATCH 06/19] Fix legacy config --- templates/config.yaml.j2 | 182 +++++++++++++++++++-------------------- 1 file changed, 89 insertions(+), 93 deletions(-) diff --git a/templates/config.yaml.j2 b/templates/config.yaml.j2 index accbc87..d8bcc9a 100644 --- a/templates/config.yaml.j2 +++ b/templates/config.yaml.j2 @@ -3,180 +3,176 @@ # Managed by Ansible, please don't edit manually # Full config: https://torsion.org/borgmatic/docs/reference/config.yaml -location: {% if borg_source_directories is not defined or borg_source_directories | length == 0 %} - source_directories: - - /etc/hostname # prevent empty backupconfig +source_directories: + - /etc/hostname # prevent empty backupconfig {% else %} - source_directories: - {% for dir in borg_source_directories %} - - {{ dir }} - {% endfor %} +source_directories: +{% for dir in borg_source_directories %} + - {{ dir }} +{% endfor %} {% endif %} - # Stay in same file system (do not cross mount points). - one_file_system: {{ borg_one_file_system }} - repositories: +# Stay in same file system (do not cross mount points). +one_file_system: {{ borg_one_file_system }} +repositories: {% if borg_repository is iterable and (borg_repository is not string and borg_repository is not mapping) %} {% for repo in borg_repository %} - - {{ repo }} + - {{ repo }} {% endfor %} {% elif borg_repository is defined and borg_repository is string %} - - {{ borg_repository }} + - {{ borg_repository }} {% endif %} - # Store atime into archive. - atime: {{ borgmatic_store_atime }} +# Store atime into archive. +atime: {{ borgmatic_store_atime }} - # Store ctime into archive. - ctime: {{ borgmatic_store_ctime }} +# Store ctime into archive. +ctime: {{ borgmatic_store_ctime }} {% if borg_exclude_patterns %} # Any paths matching these patterns are excluded from backups. Globs and tildes # are expanded. See the output of "borg help patterns" for more details. - exclude_patterns: +exclude_patterns: {% for dir in borg_exclude_patterns %} - - '{{ dir }}' + - '{{ dir }}' {% endfor %} {% endif %} {% if borg_exclude_from %} - # Read exclude patterns from one or more separate named files, one pattern per - # line. See the output of "borg help patterns" for more details. - exclude_from: +# Read exclude patterns from one or more separate named files, one pattern per +# line. See the output of "borg help patterns" for more details. +exclude_from: {% for dir in borg_exclude_from %} - - {{ dir }} + - {{ dir }} {% endfor %} {% endif %} - # Exclude directories that contain a CACHEDIR.TAG file. See - # http://www.brynosaurus.com/cachedir/spec.html for details. - exclude_caches: true +# Exclude directories that contain a CACHEDIR.TAG file. See +# http://www.brynosaurus.com/cachedir/spec.html for details. +exclude_caches: true - # Exclude directories that contain a file with the given filename. - exclude_if_present: .nobackup +# Exclude directories that contain a file with the given filename. +exclude_if_present: .nobackup - # Alternate Borg remote executable. Defaults to "borg". - # remote_path: borg1 +# Alternate Borg remote executable. Defaults to "borg". +# remote_path: borg1 {% if borg_remote_path %} - remote_path: {{ borg_remote_path }} +remote_path: {{ borg_remote_path }} {% endif %} # Repository storage options. See # https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create and # https://borgbackup.readthedocs.io/en/stable/usage/general.html#environment-variables for # details. -storage: {% if borg_encryption_passphrase %} - encryption_passphrase: {{ borg_encryption_passphrase }} +encryption_passphrase: {{ borg_encryption_passphrase }} {% endif %} - # The standard output of this command is used to unlock the encryption key. Only - # use on repositories that were initialized with passcommand/repokey encryption. - # Note that if both encryption_passcommand and encryption_passphrase are set, - # then encryption_passphrase takes precedence. - # encryption_passcommand: secret-tool lookup borg-repository repo-name +# The standard output of this command is used to unlock the encryption key. Only +# use on repositories that were initialized with passcommand/repokey encryption. +# Note that if both encryption_passcommand and encryption_passphrase are set, +# then encryption_passphrase takes precedence. +# encryption_passcommand: secret-tool lookup borg-repository repo-name {% if borg_encryption_passcommand %} - encryption_passcommand: {{ borg_encryption_passcommand }} +encryption_passcommand: {{ borg_encryption_passcommand }} {% endif %} - # Type of compression to use when creating archives. See - # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create for details. - # Defaults to no compression. - compression: {{ borg_compression|default('auto,zstd') }} +# Type of compression to use when creating archives. See +# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create for details. +# Defaults to no compression. +compression: {{ borg_compression|default('auto,zstd') }} - # Remote network upload rate limit in kiBytes/second. +# Remote network upload rate limit in kiBytes/second. {% if borg_remote_rate_limit %} - remote_rate_limit: {{ borg_remote_rate_limit }} +remote_rate_limit: {{ borg_remote_rate_limit }} {% endif %} - # Command to use instead of just "ssh". This can be used to specify ssh options. - # ssh_command: ssh -i ~/.ssh/id_ed25519 +# Command to use instead of just "ssh". This can be used to specify ssh options. +# ssh_command: ssh -i ~/.ssh/id_ed25519 {% if borg_ssh_command %} - ssh_command: {{ borg_ssh_command }} +ssh_command: {{ borg_ssh_command }} {% endif %} - # Umask to be used for borg create. - umask: 0077 +# Umask to be used for borg create. +umask: 0077 - # Maximum seconds to wait for acquiring a repository/cache lock. - lock_wait: {{ borg_lock_wait_time }} +# Maximum seconds to wait for acquiring a repository/cache lock. +lock_wait: {{ borg_lock_wait_time }} - # Name of the archive. Borg placeholders can be used. See the output of - # "borg help placeholders" for details. Default is - # "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". If you specify this option, you must - # also specify a prefix in the retention section to avoid accidental pruning of - # archives with a different archive name format. And you should also specify a - # prefix in the consistency section as well. - archive_name_format: '{hostname}-{now:%Y-%m-%d-%H%M%S}' +# Name of the archive. Borg placeholders can be used. See the output of +# "borg help placeholders" for details. Default is +# "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". If you specify this option, you must +# also specify a prefix in the retention section to avoid accidental pruning of +# archives with a different archive name format. And you should also specify a +# prefix in the consistency section as well. +archive_name_format: '{hostname}-{now:%Y-%m-%d-%H%M%S}' - # Bypass Borg error about a repository that has been moved. - relocated_repo_access_is_ok: {{ borgmatic_relocated_repo_access_is_ok }} +# Bypass Borg error about a repository that has been moved. +relocated_repo_access_is_ok: {{ borgmatic_relocated_repo_access_is_ok }} # Retention policy for how many backups to keep in each category. See # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details. # At least one of the "keep" options is required for pruning to work. -retention: {% if borg_retention_policy.keep_secondly is defined %} - # Number of secondly archives to keep. - keep_secondly: {{ borg_retention_policy.keep_secondly }} +# Number of secondly archives to keep. +keep_secondly: {{ borg_retention_policy.keep_secondly }} {% endif %} {% if borg_retention_policy.keep_minutely is defined %} - # Number of minutely archives to keep. - keep_minutely: {{ borg_retention_policy.keep_minutely }} +# Number of minutely archives to keep. +keep_minutely: {{ borg_retention_policy.keep_minutely }} {% endif %} {% if borg_retention_policy.keep_hourly is defined %} - # Number of hourly archives to keep. - keep_hourly: {{ borg_retention_policy.keep_hourly }} +# Number of hourly archives to keep. +keep_hourly: {{ borg_retention_policy.keep_hourly }} {% endif %} {% if borg_retention_policy.keep_daily is defined %} - # Number of daily archives to keep. - keep_daily: {{ borg_retention_policy.keep_daily }} +# Number of daily archives to keep. +keep_daily: {{ borg_retention_policy.keep_daily }} {% endif %} {% if borg_retention_policy.keep_weekly is defined %} - # Number of weekly archives to keep. - keep_weekly: {{ borg_retention_policy.keep_weekly }} +# Number of weekly archives to keep. +keep_weekly: {{ borg_retention_policy.keep_weekly }} {% endif %} {% if borg_retention_policy.keep_monthly is defined %} - # Number of monthly archives to keep. - keep_monthly: {{ borg_retention_policy.keep_monthly }} +# Number of monthly archives to keep. +keep_monthly: {{ borg_retention_policy.keep_monthly }} {% endif %} {% if borg_retention_policy.keep_yearly is defined %} - # Number of yearly archives to keep. - keep_yearly: {{ borg_retention_policy.keep_yearly }} +# Number of yearly archives to keep. +keep_yearly: {{ borg_retention_policy.keep_yearly }} {% endif %} # Consistency checks to run after backups. See # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check and # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-extract for details. -consistency: - # List of one or more consistency checks to run: "repository", - # "archives", "data", and/or "extract". Defaults to - # "repository" and "archives". Set to "disabled" to disable - # all consistency checks. "repository" checks the consistency - # of the repository, "archives" checks all of the archives, - # "data" verifies the integrity of the data within the - # archives, and "extract" does an extraction dry-run of the - # most recent archive. Note that "data" implies "archives". - checks: - {% for checks in borgmatic_checks %} - - {{ checks }} - {% endfor %} - # Restrict the number of checked archives to the last n. Applies only to the "archives" check. - check_last: {{ borgmatic_check_last }} +# List of one or more consistency checks to run: "repository", +# "archives", "data", and/or "extract". Defaults to +# "repository" and "archives". Set to "disabled" to disable +# all consistency checks. "repository" checks the consistency +# of the repository, "archives" checks all of the archives, +# "data" verifies the integrity of the data within the +# archives, and "extract" does an extraction dry-run of the +# most recent archive. Note that "data" implies "archives". +checks: +{% for checks in borgmatic_checks %} + - {{ checks }} +{% endfor %} + +# Restrict the number of checked archives to the last n. Applies only to the "archives" check. +check_last: {{ borgmatic_check_last }} # Shell commands or scripts to execute before and after a backup or if an error has occurred. # IMPORTANT: All provided commands and scripts are executed with user permissions of borgmatic. # Do not forget to set secure permissions on this file as well as on any script listed (chmod 0700) to # prevent potential shell injection or privilege escalation. -hooks: {% for hook in borgmatic_hooks %} - {{ hook }}: -{{ borgmatic_hooks[hook] | to_nice_yaml(indent=4) | indent(8, first=true) }} +{{ hook }}: +{{ borgmatic_hooks[hook] | to_nice_yaml(indent=4) }} {% endfor %} From 982fb1cab37b5075cd98fc10a9fe23945bcacd5c Mon Sep 17 00:00:00 2001 From: Alik Kurdyukov Date: Sat, 14 Jun 2025 13:58:42 +0400 Subject: [PATCH 07/19] fix for indent --- templates/config.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/config.yaml.j2 b/templates/config.yaml.j2 index d8bcc9a..b896911 100644 --- a/templates/config.yaml.j2 +++ b/templates/config.yaml.j2 @@ -174,5 +174,5 @@ check_last: {{ borgmatic_check_last }} # prevent potential shell injection or privilege escalation. {% for hook in borgmatic_hooks %} {{ hook }}: -{{ borgmatic_hooks[hook] | to_nice_yaml(indent=4) }} +{{ borgmatic_hooks[hook] | to_nice_yaml(indent=4) | indent(4, first=true) }} {% endfor %} From d26654f3414c17f94d1f4ca6b881b3b98cc5db15 Mon Sep 17 00:00:00 2001 From: Alik Kurdyukov Date: Sun, 15 Jun 2025 12:39:09 +0400 Subject: [PATCH 08/19] Fix legacy config (2) --- templates/config.yaml.j2 | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/templates/config.yaml.j2 b/templates/config.yaml.j2 index b896911..930e6fb 100644 --- a/templates/config.yaml.j2 +++ b/templates/config.yaml.j2 @@ -18,10 +18,10 @@ one_file_system: {{ borg_one_file_system }} repositories: {% if borg_repository is iterable and (borg_repository is not string and borg_repository is not mapping) %} {% for repo in borg_repository %} - - {{ repo }} + - path: {{ repo }} {% endfor %} {% elif borg_repository is defined and borg_repository is string %} - - {{ borg_repository }} + - path: {{ borg_repository }} {% endif %} # Store atime into archive. @@ -52,7 +52,8 @@ exclude_from: exclude_caches: true # Exclude directories that contain a file with the given filename. -exclude_if_present: .nobackup +exclude_if_present: + - .nobackup # Alternate Borg remote executable. Defaults to "borg". # remote_path: borg1 From e334994e1965b89867e29ef12d1075361b4ffae6 Mon Sep 17 00:00:00 2001 From: Alik Kurdyukov Date: Mon, 16 Jun 2025 22:02:51 +0400 Subject: [PATCH 09/19] hooks removed in favor of commands, databases minimal support added --- EXAMPLES.md | 8 +++++--- README.md | 24 ++++++++++++++++-------- defaults/main.yml | 22 +++++++++++++++------- meta/argument_specs.yml | 12 +++++++++--- molecule/default/converge.yml | 17 ++++++++++------- molecule/rocky8/converge.yml | 17 ++++++++++------- tasks/00_assert.yml | 2 +- templates/config.yaml.j2 | 16 ++++++++++++---- 8 files changed, 78 insertions(+), 40 deletions(-) diff --git a/EXAMPLES.md b/EXAMPLES.md index 5957818..046b37f 100644 --- a/EXAMPLES.md +++ b/EXAMPLES.md @@ -55,9 +55,11 @@ remote backup server. (not tested) keep_daily: 7 keep_weekly: 4 keep_monthly: 6 - borgmatic_hooks: - before_backup: - - echo "`date` - Starting backup." + borgmatic_commands: + - before: action + when: [create] + run: + - echo "`date` - Before backup" tasks: - name: Configure Borg Backup and Backupmatic tags: diff --git a/README.md b/README.md index edb3336..bcadd17 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,11 @@ Works great with [BorgBase.com](https://www.borgbase.com) - Simple and Secure Ho Systemd timers, be sure to remove the Cron job in `/etc/cron.d/borgmatic` first. The role will also alert you when trying to use both timers. +## TODO + +- [ ] Support database backup (https://torsion.org/borgmatic/docs/how-to/backup-your-databases/) +- [ ] Support healthchecks (https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/) + ## Example playbook with root as backup user, using the distro package and Cron timer ``` @@ -31,13 +36,16 @@ Works great with [BorgBase.com](https://www.borgbase.com) - Simple and Secure Ho - ssh://xxxxxx@xxxxxx.repo.borgbase.com/./repo borg_source_directories: - /var/www - borgmatic_hooks: - before_backup: - - echo "`date` - Starting backup." - postgresql_databases: - - name: users - hostname: database1.example.org - port: 5433 + borgmatic_commands: + - before: action + when: [create] + run: + - echo "Before create!" + borgmatic_databases: + postgresql: + - name: users + hostname: database1.example.org + port: 5433 ``` ## Example playbook with service user and Systemd timer @@ -111,7 +119,7 @@ $ git clone https://github.com/borgbase/ansible-role-borgbackup.git roles/ansibl - `borgmatic_timer_hour`: Hour when regular create and prune cron/systemd-timer job will run. Defaults to `{{ 6 | random }}` - `borgmatic_timer_minute`: Minute when regular create and prune cron/systemd-timer job will run. Defaults to `{{ 59 | random }}` - `borgmatic_timer_flags`: Flags to pass to borgmatic cron/systemd-timer job, like "--log-file /path/to/file.log --log-file-verbosity 2" -- `borgmatic_hooks`: Hooks to monitor your backups e.g. with [Healthchecks](https://healthchecks.io/). See [official documentation](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/) for more. +- `borgmatic_commands`: Invoke script before/after actions. See [How to add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/) for more. - `borgmatic_timer`: If the variable is set, a timer is installed. A choice must be made between `cron` and `systemd`. - `borgmatic_relocated_repo_access_is_ok`: Bypass Borg error about a repository that has been moved. Defaults to `false` - `borgmatic_store_atime`: Store atime into archive. Defaults to `true` diff --git a/defaults/main.yml b/defaults/main.yml index b3e5872..c456b31 100755 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -27,13 +27,21 @@ borg_install_method: "pip" borg_require_epel: "{{ ansible_os_family == 'RedHat' and ansible_distribution != 'Fedora' }}" borgmatic_config_name: config.yaml -borgmatic_hooks: - on_error: - - echo "`date` - Error while creating a backup." - before_backup: - - echo "`date` - Starting backup." - after_backup: - - echo "`date` - Finished backup." +borgmatic_commands: + - before: action + when: [create] + run: + - echo "Before create!" + - after: action + when: + - create + - prune + run: + - echo "After create or prune!" + - after: error + run: + - echo "Something went wrong!" + borgmatic_checks: - name: repository frequency: "4 weeks" diff --git a/meta/argument_specs.yml b/meta/argument_specs.yml index b668fe9..cf4ca8a 100644 --- a/meta/argument_specs.yml +++ b/meta/argument_specs.yml @@ -163,10 +163,11 @@ argument_specs: type: int required: false description: Restrict the number of checked archives to the last n. Applies only to the "archives" check. - borgmatic_hooks: - type: dict + borgmatic_commands: + type: list + elements: dict required: false - description: Shell commands or scripts to execute before and after a backup or if an error has occurred. + description: Shell commands or scripts to execute before and after a backup or if an error has occurred. See https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ borgmatic_timer_cron_name: type: str required: false @@ -203,3 +204,8 @@ argument_specs: type: str required: false description: Comment added to the SSH public key. + borgmatic_databases: + type: dict + required: false + description: Database server name to list of databases to backup, see EXAMPLES.md + diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index e0108e3..90b3224 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -46,10 +46,13 @@ keep_daily: 7 keep_weekly: 4 keep_monthly: 6 - borgmatic_hooks: - before_backup: - - echo "`date` - Starting backup." - postgresql_databases: - - name: users - hostname: database1.example.org - port: 5433 + borgmatic_commands: + - before: action + when: [create] + run: + - echo "Before create!" + borgmatic_databases: + postgresql: + - name: users + hostname: database1.example.org + port: 5433 diff --git a/molecule/rocky8/converge.yml b/molecule/rocky8/converge.yml index e0108e3..90b3224 100644 --- a/molecule/rocky8/converge.yml +++ b/molecule/rocky8/converge.yml @@ -46,10 +46,13 @@ keep_daily: 7 keep_weekly: 4 keep_monthly: 6 - borgmatic_hooks: - before_backup: - - echo "`date` - Starting backup." - postgresql_databases: - - name: users - hostname: database1.example.org - port: 5433 + borgmatic_commands: + - before: action + when: [create] + run: + - echo "Before create!" + borgmatic_databases: + postgresql: + - name: users + hostname: database1.example.org + port: 5433 diff --git a/tasks/00_assert.yml b/tasks/00_assert.yml index 4cbabd3..17e22d9 100644 --- a/tasks/00_assert.yml +++ b/tasks/00_assert.yml @@ -5,5 +5,5 @@ - borgmatic_failure_command is undefined - borgmatic_before_backup_command is undefined - borgmatic_after_backup_command is undefined - msg: Please use the new borgmatic_hooks variable instead of individual before/after/failure hooks. + msg: Please use the new borgmatic_commands variable instead of individual before/after/failure hooks. ... diff --git a/templates/config.yaml.j2 b/templates/config.yaml.j2 index 930e6fb..41e56e4 100644 --- a/templates/config.yaml.j2 +++ b/templates/config.yaml.j2 @@ -19,9 +19,11 @@ repositories: {% if borg_repository is iterable and (borg_repository is not string and borg_repository is not mapping) %} {% for repo in borg_repository %} - path: {{ repo }} + encryption: repokey {% endfor %} {% elif borg_repository is defined and borg_repository is string %} - path: {{ borg_repository }} + encryption: repokey {% endif %} # Store atime into archive. @@ -173,7 +175,13 @@ check_last: {{ borgmatic_check_last }} # IMPORTANT: All provided commands and scripts are executed with user permissions of borgmatic. # Do not forget to set secure permissions on this file as well as on any script listed (chmod 0700) to # prevent potential shell injection or privilege escalation. -{% for hook in borgmatic_hooks %} -{{ hook }}: -{{ borgmatic_hooks[hook] | to_nice_yaml(indent=4) | indent(4, first=true) }} -{% endfor %} +{% if borgmatic_commands is defined %} +commands: +{{ borgmatic_commands | to_nice_yaml(indent=4) | indent(4, first=true) }} +{% endif %} + +# Databases specific backup +{% for database in borgmatic_databases %} +{{ database }}_databases: +{{ borgmatic_databases[database] | to_nice_yaml(indent=4) | indent(4, first=true) }} +{% endfor %} \ No newline at end of file From 60115dd0b75993cd7e324156e33aeb821aa79747 Mon Sep 17 00:00:00 2001 From: Alik Kurdyukov Date: Tue, 17 Jun 2025 11:46:57 +0400 Subject: [PATCH 10/19] fix for undefined databases --- templates/config.yaml.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/templates/config.yaml.j2 b/templates/config.yaml.j2 index 41e56e4..7923f78 100644 --- a/templates/config.yaml.j2 +++ b/templates/config.yaml.j2 @@ -181,7 +181,9 @@ commands: {% endif %} # Databases specific backup +{% if borgmatic_databases is defined %} {% for database in borgmatic_databases %} {{ database }}_databases: {{ borgmatic_databases[database] | to_nice_yaml(indent=4) | indent(4, first=true) }} -{% endfor %} \ No newline at end of file +{% endfor %} +{% endif %} From 61470c98ac1e029bdc56465ca1b20a59875c482d Mon Sep 17 00:00:00 2001 From: Alik Kurdyukov Date: Tue, 17 Jun 2025 12:16:45 +0400 Subject: [PATCH 11/19] cosmetic config fixes --- templates/config.yaml.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/config.yaml.j2 b/templates/config.yaml.j2 index 7923f78..a37f6f1 100644 --- a/templates/config.yaml.j2 +++ b/templates/config.yaml.j2 @@ -171,17 +171,17 @@ checks: # Restrict the number of checked archives to the last n. Applies only to the "archives" check. check_last: {{ borgmatic_check_last }} +{% if borgmatic_commands is defined %} # Shell commands or scripts to execute before and after a backup or if an error has occurred. # IMPORTANT: All provided commands and scripts are executed with user permissions of borgmatic. # Do not forget to set secure permissions on this file as well as on any script listed (chmod 0700) to # prevent potential shell injection or privilege escalation. -{% if borgmatic_commands is defined %} commands: -{{ borgmatic_commands | to_nice_yaml(indent=4) | indent(4, first=true) }} +{{ borgmatic_commands | to_nice_yaml(indent=4) | indent(4, first=False) }} {% endif %} -# Databases specific backup {% if borgmatic_databases is defined %} +# Databases specific backup {% for database in borgmatic_databases %} {{ database }}_databases: {{ borgmatic_databases[database] | to_nice_yaml(indent=4) | indent(4, first=true) }} From c7ef79ad3fad43cffa4a16acaaeeda1bb0cfc934 Mon Sep 17 00:00:00 2001 From: Alik Kurdyukov Date: Tue, 17 Jun 2025 13:50:53 +0400 Subject: [PATCH 12/19] fix config yaml formatting --- defaults/main.yml | 2 ++ templates/config.yaml.j2 | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index c456b31..63045fb 100755 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -42,6 +42,8 @@ borgmatic_commands: run: - echo "Something went wrong!" +borgmatic_databases: [] + borgmatic_checks: - name: repository frequency: "4 weeks" diff --git a/templates/config.yaml.j2 b/templates/config.yaml.j2 index a37f6f1..23fd8eb 100644 --- a/templates/config.yaml.j2 +++ b/templates/config.yaml.j2 @@ -177,13 +177,13 @@ check_last: {{ borgmatic_check_last }} # Do not forget to set secure permissions on this file as well as on any script listed (chmod 0700) to # prevent potential shell injection or privilege escalation. commands: -{{ borgmatic_commands | to_nice_yaml(indent=4) | indent(4, first=False) }} +{{ borgmatic_commands | to_nice_yaml(indent=4) }} {% endif %} {% if borgmatic_databases is defined %} # Databases specific backup {% for database in borgmatic_databases %} {{ database }}_databases: -{{ borgmatic_databases[database] | to_nice_yaml(indent=4) | indent(4, first=true) }} +{{ borgmatic_databases[database] | to_nice_yaml(indent=4) }} {% endfor %} {% endif %} From 865da8d58bcd3b1b830ef526a6d9c81070e57513 Mon Sep 17 00:00:00 2001 From: Alik Kurdyukov Date: Tue, 17 Jun 2025 14:03:14 +0400 Subject: [PATCH 13/19] fix default value --- defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/defaults/main.yml b/defaults/main.yml index 63045fb..95db049 100755 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -42,7 +42,7 @@ borgmatic_commands: run: - echo "Something went wrong!" -borgmatic_databases: [] +borgmatic_databases: {} borgmatic_checks: - name: repository From d3fadbaffe41ff505ff594ca51d1adbc6e0c8f0f Mon Sep 17 00:00:00 2001 From: Alik Kurdyukov Date: Tue, 22 Jul 2025 18:06:11 +0400 Subject: [PATCH 14/19] yaml format fix --- templates/config.yaml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/config.yaml.j2 b/templates/config.yaml.j2 index e9ba1fc..f2c9133 100644 --- a/templates/config.yaml.j2 +++ b/templates/config.yaml.j2 @@ -18,10 +18,10 @@ one_file_system: {{ borg_one_file_system }} repositories: {% if borg_repository is iterable and (borg_repository is not string and borg_repository is not mapping) %} {% for repo in borg_repository %} - - path: {{ repo }} + - path: {{ repo }} {% endfor %} {% elif borg_repository is defined and borg_repository is string %} - - path: {{ borg_repository }} + - path: {{ borg_repository }} {% endif %} # Store atime into archive. From 31fa56c9e9e76a400e8b1e4bceeb27b97deccfd2 Mon Sep 17 00:00:00 2001 From: akul <0xfabc@gmail.com> Date: Mon, 28 Jul 2025 09:09:27 +0400 Subject: [PATCH 15/19] fix config template: wrong indent --- templates/config.yaml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/config.yaml.j2 b/templates/config.yaml.j2 index f2c9133..65d310e 100644 --- a/templates/config.yaml.j2 +++ b/templates/config.yaml.j2 @@ -111,8 +111,8 @@ archive_name_format: '{hostname}-{now:%Y-%m-%d-%H%M%S}' # Bypass Borg error about a repository that has been moved. relocated_repo_access_is_ok: {{ borgmatic_relocated_repo_access_is_ok }} - # Bypass Borg error about a previously unknown unencrypted repository. - unknown_unencrypted_repo_access_is_ok: {{ borgmatic_unknown_unencrypted_repo_access_is_ok }} +# Bypass Borg error about a previously unknown unencrypted repository. +unknown_unencrypted_repo_access_is_ok: {{ borgmatic_unknown_unencrypted_repo_access_is_ok }} # Retention policy for how many backups to keep in each category. See # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details. From 82c85d9a564fd6dfedfd6ca7217f4abb317467e8 Mon Sep 17 00:00:00 2001 From: akul <0xfabc@gmail.com> Date: Thu, 31 Jul 2025 13:27:44 +0400 Subject: [PATCH 16/19] cosmetic change to the service template --- templates/borgmatic.service.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/borgmatic.service.j2 b/templates/borgmatic.service.j2 index 0994e9d..4e62ca2 100644 --- a/templates/borgmatic.service.j2 +++ b/templates/borgmatic.service.j2 @@ -1,4 +1,4 @@ -# Managed by Ansible, please don't edit manually +{{ ansible_managed }} [Unit] Description=borgmatic backup From 80720c23b238769f6ed041057f0dae34e67e23da Mon Sep 17 00:00:00 2001 From: akul <0xfabc@gmail.com> Date: Fri, 8 Aug 2025 10:51:37 +0400 Subject: [PATCH 17/19] fix service template --- templates/borgmatic.service.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/borgmatic.service.j2 b/templates/borgmatic.service.j2 index 4e62ca2..daf229a 100644 --- a/templates/borgmatic.service.j2 +++ b/templates/borgmatic.service.j2 @@ -1,4 +1,4 @@ -{{ ansible_managed }} +#{{ ansible_managed }} [Unit] Description=borgmatic backup From 525bcda24970455f4e2cc6b4ce58ae15f945f037 Mon Sep 17 00:00:00 2001 From: akul <0xfabc@gmail.com> Date: Mon, 11 Aug 2025 09:34:35 +0400 Subject: [PATCH 18/19] Add borg repo label --- meta/argument_specs.yml | 4 ++++ templates/config.yaml.j2 | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/meta/argument_specs.yml b/meta/argument_specs.yml index 55e224f..8d14e28 100644 --- a/meta/argument_specs.yml +++ b/meta/argument_specs.yml @@ -67,6 +67,10 @@ argument_specs: Your own server or [BorgBase.com](https://www.borgbase.com) repo. Not required when using auto creation of repositories. Can be a list if you want to backup to multiple repositories. + borg_repository_label: + type: str + required: false + description: Label for the repository. borgmatic_store_atime: type: bool required: false diff --git a/templates/config.yaml.j2 b/templates/config.yaml.j2 index 65d310e..6e33def 100644 --- a/templates/config.yaml.j2 +++ b/templates/config.yaml.j2 @@ -19,9 +19,15 @@ repositories: {% if borg_repository is iterable and (borg_repository is not string and borg_repository is not mapping) %} {% for repo in borg_repository %} - path: {{ repo }} + {% if borg_repository_label is defined and borg_repository_label is string %} + label: {{ borg_repository_label }} + {% endif %} {% endfor %} {% elif borg_repository is defined and borg_repository is string %} - path: {{ borg_repository }} +{% if borg_repository_label is defined and borg_repository_label is string %} + label: {{ borg_repository_label }} +{% endif %} {% endif %} # Store atime into archive. From 714cb2dd6fcc8a0c7d853d7fab18e791cdea546c Mon Sep 17 00:00:00 2001 From: akul <0xfabc@gmail.com> Date: Mon, 11 Aug 2025 09:35:16 +0400 Subject: [PATCH 19/19] cosmetic change: ansible_managed instead of pure text --- templates/config.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/config.yaml.j2 b/templates/config.yaml.j2 index 6e33def..7a76492 100644 --- a/templates/config.yaml.j2 +++ b/templates/config.yaml.j2 @@ -1,6 +1,6 @@ #jinja2: lstrip_blocks: True, trim_blocks: True --- -# Managed by Ansible, please don't edit manually +# {{ ansible_managed }} # Full config: https://torsion.org/borgmatic/docs/reference/config.yaml {% if borg_source_directories is not defined or borg_source_directories | length == 0 %}