From 28dde00c89975bdc2e444f7b7a037172749ea705 Mon Sep 17 00:00:00 2001 From: "jerome.gagnon" Date: Mon, 9 Oct 2023 00:22:30 +0000 Subject: [PATCH] implements docker install method --- .github/workflows/main.yml | 2 + defaults/main.yml | 4 + molecule/docker/Dockerfile.j2 | 23 +++++ molecule/docker/INSTALL.rst | 22 ++++ molecule/docker/converge.yml | 96 ++++++++++++++++++ molecule/docker/molecule.yml | 21 ++++ molecule/docker/vault.pw | 1 + molecule/docker/verify.yml | 33 ++++++ tasks/noauto_install_docker.yml | 172 ++++++++++++++++++++++++++++++++ templates/Dockerfile.j2 | 15 +++ templates/ansible_entry.sh.j2 | 9 ++ vars/Alpine.yml | 6 ++ vars/Archlinux.yml | 3 + vars/Debian.yml | 3 + vars/Fedora.yml | 3 + vars/ManjaroLinux.yml | 3 + vars/RedHat-8.yml | 3 + vars/RedHat-9.yml | 3 + vars/RedHat.yml | 3 + 19 files changed, 425 insertions(+) create mode 100644 molecule/docker/Dockerfile.j2 create mode 100644 molecule/docker/INSTALL.rst create mode 100644 molecule/docker/converge.yml create mode 100644 molecule/docker/molecule.yml create mode 100644 molecule/docker/vault.pw create mode 100644 molecule/docker/verify.yml create mode 100644 tasks/noauto_install_docker.yml create mode 100644 templates/Dockerfile.j2 create mode 100644 templates/ansible_entry.sh.j2 create mode 100644 vars/Alpine.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ef664fc..d86146b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,3 +19,5 @@ jobs: # uses: mxschmitt/action-tmate@v3.5 - name: Test using Molecule run: molecule test + - name: Test Docker scenario using Molecule + run: molecule test -s docker \ No newline at end of file diff --git a/defaults/main.yml b/defaults/main.yml index 02e0a65..e53f23e 100755 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -46,4 +46,8 @@ borgmatic_version: ">=1.7.11" borg_venv_path: "/opt/borgmatic" borg_user: "root" borg_group: "root" + +borgmatic_docker_image_name: "ansible_borgmatic" +borgmatic_docker_container_name: "ansible_borgmatic" +borgmatic_docker_timezone: "UTC" ... diff --git a/molecule/docker/Dockerfile.j2 b/molecule/docker/Dockerfile.j2 new file mode 100644 index 0000000..5c80668 --- /dev/null +++ b/molecule/docker/Dockerfile.j2 @@ -0,0 +1,23 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +{% if item.env is defined %} +{% for var, value in item.env.items() %} +{% if value %} +ENV {{ var }} {{ value }} +{% endif %} +{% endfor %} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python3 python3-pip sudo bash ca-certificates iproute2 python3-apt aptitude && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install /usr/bin/python3 /usr/bin/python3-config /usr/bin/dnf-3 sudo bash iproute && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y /usr/bin/python /usr/bin/python2-config sudo yum-plugin-ovl bash iproute && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml iproute2 && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python3 sudo bash ca-certificates; \ + elif [ $(command -v pacman) ]; then pacman --noconfirm -Suy python python-pip sudo openssh; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates iproute2 && xbps-remove -O; fi diff --git a/molecule/docker/INSTALL.rst b/molecule/docker/INSTALL.rst new file mode 100644 index 0000000..6a44bde --- /dev/null +++ b/molecule/docker/INSTALL.rst @@ -0,0 +1,22 @@ +******* +Docker driver installation guide +******* + +Requirements +============ + +* Docker Engine + +Install +======= + +Please refer to the `Virtual environment`_ documentation for installation best +practices. If not using a virtual environment, please consider passing the +widely recommended `'--user' flag`_ when invoking ``pip``. + +.. _Virtual environment: https://virtualenv.pypa.io/en/latest/ +.. _'--user' flag: https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site + +.. code-block:: bash + + $ pip install 'molecule[docker]' diff --git a/molecule/docker/converge.yml b/molecule/docker/converge.yml new file mode 100644 index 0000000..f1596e8 --- /dev/null +++ b/molecule/docker/converge.yml @@ -0,0 +1,96 @@ +--- +- name: Converge + hosts: all + pre_tasks: + - name: Set ssh server package name for non-Archlinux ansible_os_family + set_fact: + openssh_package: "openssh-server" + pip3_extra_args: "" + when: ansible_os_family != "Archlinux" + + - name: Set ssh server package name and pip3 argument for Archlinux ansible_os_family + set_fact: + openssh_package: "openssh" + pip3_extra_args: "--break-system-packages" + when: ansible_os_family == "Archlinux" + + - name: Install openssh + package: + name: "{{ openssh_package }}" + state: present + + - name: Define borg_source_directories + ansible.builtin.set_fact: + borg_source_directories_tmp: + - /srv/www + - /var/lib/automysqlbackup + + - name: Create backup source folders on the docker host + ansible.builtin.file: + path: "{{ item }}" + mode: "0777" + state: directory + with_items: "{{ borg_source_directories_tmp }}" + + - name: Define borg_repository + ansible.builtin.set_fact: + borg_repository_tmp: + - m5vz9gp4@m5vz9gp4.repo.borgbase.com:repo + - /local_borg_repo + + - name: Create local repository folders on the docker host + ansible.builtin.file: + path: "{{ item }}" + mode: "0777" + state: directory + with_items: "{{ borg_repository_tmp }}" + when: item[0] == "/" + + roles: + - role: borgbase.ansible_role_borgbackup + borg_install_method: docker + borgmatic_timer: cron + borg_repository: "{{ borg_repository_tmp }}" + borg_encryption_passphrase: CHANGEME + borg_source_directories: "{{ borg_source_directories_tmp }}" + borg_exclude_patterns: + - /srv/www/old-sites + borg_retention_policy: + keep_hourly: 3 + keep_daily: 7 + keep_weekly: 4 + keep_monthly: 6 + borgmatic_hooks: + before_backup: + - echo "`date` - Starting backup." + postgresql_databases: + - name: users + hostname: database1.example.org + port: 5433 + borg_ssh_private_key: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 65373636303732303236313234666230386333636233313631663135323734626265616532633064 + 3163346333616539663732636366626535326238623761320a336130633135643735613433636538 + 33333336656238306163303431393562303863633137646337633861346265353131396434393531 + 6564386438356330380a373138353364316535653338396164383861396538333336666436663832 + 32613439616136313331333336636232323231623363633661656632316237653633363466313734 + 35316262653366373137393761393835643166666436333635383334643636616436623030376234 + 33343565363863613161373561616237313138633765376263656536303565363838376163313963 + 37656431316335663030336236633663313937353362653639303836366436383334373132666334 + 39313562316330613131383738613136616631336461626362313764313637356233373437613962 + 31363564643266353737656261613232366336386230333963393935353763343236333564376462 + 36653538363131616133653463613633343036363931316334613136653265636262313235366434 + 31306562363034336431373535393364346435323130386265346431343836613135353430366534 + 61323861653464313763303261656430393930623664396630666133383038313939303030396362 + 34363435316434656462366339346637396134623337633133386638646463633063363133656164 + 35396237366363383637333662366437633361356466616137623362623439323433656562636238 + 66633964323831386435306163343566666533663363343262346332373764366635643961333130 + 63346431326432313234653132383664396165313538346161316264653235616161353833633234 + 31343663346434633863393934653631376334346666346437366639613032343632356635613932 + 62306361343336386435653939386339343066366531356632643730643330353931663239326130 + 39346364363263363332363637616133323761636437313138633630363237383363393432386362 + 33633330323536346430636234373032346663336630623334363363393661376531376337313066 + 64626434356535346461326339376435643738353463343035306433343630653335643635613939 + 37323564323130356338643237383966313539663132656533656434626166373839653435343835 + 62373131393235333934356133643963613665626532643164343063666632626561666330373930 + 6132 diff --git a/molecule/docker/molecule.yml b/molecule/docker/molecule.yml new file mode 100644 index 0000000..c726e3e --- /dev/null +++ b/molecule/docker/molecule.yml @@ -0,0 +1,21 @@ +--- +dependency: + name: galaxy +driver: + name: docker +platforms: + - name: docker-dind # Based on Alpine + image: docker:dind + privileged: True # required to have access to the dind + command: /usr/local/bin/dockerd-entrypoint.sh # Starts docker daemon +provisioner: + name: ansible + config_options: + defaults: + vault_password_file: "${MOLECULE_SCENARIO_DIRECTORY}/vault.pw" +verifier: + name: ansible +lint: | + set -e + yamllint . + ansible-lint . \ No newline at end of file diff --git a/molecule/docker/vault.pw b/molecule/docker/vault.pw new file mode 100644 index 0000000..7aa311a --- /dev/null +++ b/molecule/docker/vault.pw @@ -0,0 +1 @@ +password \ No newline at end of file diff --git a/molecule/docker/verify.yml b/molecule/docker/verify.yml new file mode 100644 index 0000000..7d6a26a --- /dev/null +++ b/molecule/docker/verify.yml @@ -0,0 +1,33 @@ +--- +- name: Verify + hosts: all + tasks: + - name: Set docker_cmd + ansible.builtin.set_fact: + docker_cmd: docker exec -i ansible_borgmatic + + - name: Ensure Borgmatic is installed correctly + command: "{{ docker_cmd }} borgmatic --version" + changed_when: false + + - name: Ensure Borg is installed correctly + command: "{{ docker_cmd }} borgmatic borg --version" + changed_when: false + + - name: Ensure produced YAML is valid + ansible.builtin.shell: | + {{ docker_cmd }} pip3 install yamllint && \ + {{ docker_cmd }} yamllint --list-files -d "{extends: relaxed, rules: {line-length: {max: 120}}}" /etc/borgmatic/config.yaml + changed_when: false + + - name: Ensure modified source path name is present in the config instead of the original + command: "{{ docker_cmd }} grep /sources/var/lib/automysqlbackup /etc/borgmatic/config.yaml" + changed_when: false + + - name: Ensure modified local repo path name is present in the config instead of the original + command: "{{ docker_cmd }} grep /repositories/local_borg_repo /etc/borgmatic/config.yaml" + changed_when: false + + - name: Ensure supercronic is running + command: "{{ docker_cmd }} pgrep supercronic" + changed_when: false diff --git a/tasks/noauto_install_docker.yml b/tasks/noauto_install_docker.yml new file mode 100644 index 0000000..e69eeef --- /dev/null +++ b/tasks/noauto_install_docker.yml @@ -0,0 +1,172 @@ +--- +- name: Install borgbackup with Docker + block: + + - name: Create temp directory for Docker build + ansible.builtin.tempfile: + state: directory + register: build_dir + changed_when: false + + - name: Install build dependencies + ansible.builtin.package: + name: "{{ borg_docker_packages }}" + state: present + + - name: Define Docker image tag based on borgmatic --version + ansible.builtin.set_fact: + borgmatic_docker_tag: "{{ borgmatic_version | regex_search('\\d+\\.\\d+(\\.\\d+){0,1}') }}" + + - name: Debug borgmatic_docker_tag + debug: + var: borgmatic_docker_tag + + - name: Assert source path exists + ansible.builtin.stat: + path: "{{ item }}" + register: source_path + failed_when: not source_path.stat.exists or (source_path.stat.exists and not source_path.stat.isdir) + with_items: "{{ borg_source_directories }}" + + - name: Assert local repository path exists + ansible.builtin.stat: + path: "{{ item }}" + register: source_path + failed_when: not source_path.stat.exists or (source_path.stat.exists and not source_path.stat.isdir) + when: item[0] == "/" + with_items: "{{ borg_repository }}" + + - name: Assert user and group are repositories + ansible.builtin.assert: + that: + - borg_user == "root" + - borg_group == "root" + fail_msg: For docker deployment, only "root" is supported for borg_user and borg_group + + - name: Build volume list from borg_source_directories and borg_repository + ansible.builtin.set_fact: + volumes: >- + {%- set volumes = [] -%} + {%- for dir in borg_source_directories -%} + {%- set _ = volumes.append(dir + ":/sources" + dir + ":ro") -%} + {%- endfor -%} + {%- for dir in borg_repository -%} + {%- if dir[0] == "/" -%} + {%- set _ = volumes.append(dir + ":/repositories" + dir) -%} + {%- endif -%} + {%- endfor -%} + {{ volumes }} + + - name: Debug volume list + ansible.builtin.debug: + var: volumes + verbosity: 1 + + - name: Modify borg_source_directories to reflect path in container + ansible.builtin.set_fact: + borg_source_directories: >- + {%- set sources = [] -%} + {%- for source in borg_source_directories -%} + {%- set _ = sources.append("/sources" + source) -%} + {%- endfor -%} + {{ sources }} + + - name: Debug borg_source_directories + ansible.builtin.debug: + var: borg_source_directories + verbosity: 1 + + - name: Modify borg_repository to reflect path in container + ansible.builtin.set_fact: + borg_repository: >- + {%- set repositories = [] -%} + {%- for repo in borg_repository -%} + {%- if repo[0] == "/" -%} + {%- set _ = repositories.append("/repositories" + repo) -%} + {%- else -%} + {%- set _ = repositories.append(repo) -%} + {%- endif -%} + {%- endfor -%} + {{ repositories }} + borg_repository_flat: "{{ borg_repository | join('|') }}" + + - name: Debug borg_repository + ansible.builtin.debug: + var: borg_repository + verbosity: 1 + + - name: Check if ssh repo in the list + when: + - not borg_ssh_private_key + - borg_repository_flat is match('|[^/]') + ansible.builtin.set_fact: + has_ssh_repo: true + + - name: Test if private key was provided + when: + - not borg_ssh_private_key + - has_ssh_repo + ansible.builtin.fail: + msg: "Private key content must be provided when using docker" + + - name: Copy private key + when: borg_ssh_private_key + changed_when: false + ansible.builtin.copy: + dest: "{{ build_dir.path }}/{{ borg_ssh_key_name }}" + mode: 0600 + content: "{{ borg_ssh_private_key }}" + validate: ssh-keygen -yf %s # Also ensure priv key content is sound + + - name: Generate public key from private key + when: borg_ssh_private_key + changed_when: false + failed_when: not public_key.stdout.startswith("ssh") + register: public_key + ansible.builtin.command: "ssh-keygen -yf {{ build_dir.path }}/{{ borg_ssh_key_name }}" + + - name: Copy other files to build folder for docker build + changed_when: false + ansible.builtin.template: + dest: "{{ build_dir.path }}/{{ item | basename | regex_replace('\\.j2$', '') }}" + src: "{{ item }}" + mode: 0600 + with_items: + - Dockerfile.j2 + - config.yaml.j2 + - ansible_entry.sh.j2 + + - name: Build docker image + changed_when: false # will make the idempotency test fail otherwise + community.docker.docker_image: + name: "{{ borgmatic_docker_image_name }}:{{ borgmatic_docker_tag }}" + source: build + state: present + force_source: true + build: + path: "{{ build_dir.path }}" + pull: true + rm: false + args: + PUBLIC_KEY: "{{ public_key }}" + PRIVATE_KEY: "{{ borg_ssh_private_key }}" + + - name: Start container + changed_when: false # will make the idempotency test fail otherwise + community.docker.docker_container: + name: "{{ borgmatic_docker_container_name }}" + image: "{{ borgmatic_docker_image_name }}:{{ borgmatic_docker_tag }}" + volumes: "{{ volumes }}" + restart_policy: unless-stopped + labels: + ansible_borgmatic_managed: "1" + env: + BACKUP_CRON: "{{ borgmatic_timer_minute }} {{ borgmatic_timer_hour }} * * * borgmatic -c /etc/borgmatic/{{ borgmatic_config_name }}" + TZ: "{{ borgmatic_docker_timezone }}" + + always: + - name: Delete build folder + ansible.builtin.file: + path: "{{ build_dir.path }}" + state: absent + changed_when: false \ No newline at end of file diff --git a/templates/Dockerfile.j2 b/templates/Dockerfile.j2 new file mode 100644 index 0000000..b15f60f --- /dev/null +++ b/templates/Dockerfile.j2 @@ -0,0 +1,15 @@ +FROM ghcr.io/borgmatic-collective/borgmatic:{{ borgmatic_docker_tag }} + +LABEL "ansible_borgmatic_managed"="1" + +COPY config.yaml /etc/borgmatic/{{ borgmatic_config_name }} + +# Those keys will be copied at /root/.ssh at runtime. This is required because of the anom volumes defined in the upstream image +ARG PUBLIC_KEY="" +ARG PRIVATE_KEY="" +RUN if [ ! -z "$PUBLIC_KEY" ]; then echo "$PUBLIC_KEY" > /{{ borg_ssh_key_name}}.pub; fi +RUN if [ ! -z "$PRIVATE_KEY" ]; then echo "$PRIVATE_KEY" > /{{ borg_ssh_key_name}}; fi +COPY ansible_entry.sh / +RUN chmod 700 /ansible_entry.sh + +ENTRYPOINT [ "/ansible_entry.sh" ] \ No newline at end of file diff --git a/templates/ansible_entry.sh.j2 b/templates/ansible_entry.sh.j2 new file mode 100644 index 0000000..c54b49b --- /dev/null +++ b/templates/ansible_entry.sh.j2 @@ -0,0 +1,9 @@ +#!/bin/sh + +# We need to copy ssh keys at runtime because of the built-in volumes in the upstream Docker image definition +if [ -f "/{{ borg_ssh_key_name }}.pub" ]; then mv /{{ borg_ssh_key_name }}.pub /root/.ssh; fi +if [ -f "/{{ borg_ssh_key_name }}" ]; then mv /{{ borg_ssh_key_name }} /root/.ssh; fi + +echo "$BACKUP_CRON" > /etc/borgmatic.d/crontab.txt + +exec env SUPERCRONIC_EXTRA_FLAGS=-debug /entry.sh "$@" \ No newline at end of file diff --git a/vars/Alpine.yml b/vars/Alpine.yml new file mode 100644 index 0000000..31969fb --- /dev/null +++ b/vars/Alpine.yml @@ -0,0 +1,6 @@ +--- +borg_docker_packages: + - py3-docker-py + +python_bin: python3 +pip_bin: pip3 diff --git a/vars/Archlinux.yml b/vars/Archlinux.yml index 48c3688..ebc1d54 100644 --- a/vars/Archlinux.yml +++ b/vars/Archlinux.yml @@ -10,6 +10,9 @@ borg_pip_packages: - python-pip - python-setuptools +borg_docker_packages: + - python-docker + borg_distro_packages: - borg - borgmatic diff --git a/vars/Debian.yml b/vars/Debian.yml index c5fb44b..44ee54d 100644 --- a/vars/Debian.yml +++ b/vars/Debian.yml @@ -16,6 +16,9 @@ borg_pip_packages: - python3-msgpack - python3-venv +borg_docker_packages: + - python3-docker + borg_distro_packages: - borgbackup - borgmatic diff --git a/vars/Fedora.yml b/vars/Fedora.yml index a5583ec..977d5ee 100644 --- a/vars/Fedora.yml +++ b/vars/Fedora.yml @@ -16,6 +16,9 @@ borg_pip_packages: - python3-setuptools - python3-Cython +borg_docker_packages: + - python3-docker + borg_distro_packages: - borgbackup - borgmatic diff --git a/vars/ManjaroLinux.yml b/vars/ManjaroLinux.yml index fd4a65b..7e5b179 100644 --- a/vars/ManjaroLinux.yml +++ b/vars/ManjaroLinux.yml @@ -16,6 +16,9 @@ borg_pip_packages: # untested - python3-msgpack - python3-venv +borg_docker_packages: + - python3-docker + borg_distro_packages: - borg - borgmatic diff --git a/vars/RedHat-8.yml b/vars/RedHat-8.yml index 4497b8d..6936ec6 100644 --- a/vars/RedHat-8.yml +++ b/vars/RedHat-8.yml @@ -16,6 +16,9 @@ borg_pip_packages: - python3-setuptools - python3-virtualenv +borg_docker_packages: + - python3-docker + borg_distro_packages: - borgbackup - borgmatic diff --git a/vars/RedHat-9.yml b/vars/RedHat-9.yml index 2b900f4..e0faa41 100644 --- a/vars/RedHat-9.yml +++ b/vars/RedHat-9.yml @@ -16,6 +16,9 @@ borg_pip_packages: - python3-setuptools # - python3-virtualenv +borg_docker_packages: + - python3-docker + borg_distro_packages: - borgbackup - borgmatic diff --git a/vars/RedHat.yml b/vars/RedHat.yml index 3115a75..ebf6423 100644 --- a/vars/RedHat.yml +++ b/vars/RedHat.yml @@ -15,6 +15,9 @@ borg_pip_packages: - python36-devel - python-setuptools +borg_docker_packages: + - python36-docker + borg_distro_packages: - borgbackup - borgmatic