diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..02b9af6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.pytest_cache/ +__pycache__/ +pytestdebug.log diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..29b9480 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,24 @@ +--- +sudo: required + +language: python + +services: + - docker + +before_install: + - sudo apt-get -qq update + +install: + - pip install molecule + - pip install ansible + - pip install docker + +env: + - SCENARIO=lamp + - SCENARIO=extra_opts + - SCENARIO=mgt + - SCENARIO=multiple + - SCENARIO=clients + +script: "molecule test --scenario-name $SCENARIO" diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000..e8b5eb7 --- /dev/null +++ b/.yamllint @@ -0,0 +1,12 @@ +extends: default + +ignore: | + *test_client.yml + *test_client_restore.yml + *test_server.yml + *test_mgt.yml + +rules: + line-length: + max: 1024 + level: warning diff --git a/README.md b/README.md index da7f5f4..5e60556 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![Build Status](https://travis-ci.com/FiaasCo/borgbackup.svg?branch=master)](https://travis-ci.com/FiaasCo/borgbackup) + # Borg backup role This role installs Borg backup on borgbackup\_servers and clients. The role contains a wrapper-script 'borg-backup' to ease the usage on the client. Supported options include borg-backup info | init | list | backup | mount. Automysqlbackup will run as pre-backup command if it's installed. The role supports both self hosted and offsite backup-storage such as rsync.net and hetzner storage box as Borg server. diff --git a/backup.yml b/backup.yml index 3a02331..8ec6fae 100644 --- a/backup.yml +++ b/backup.yml @@ -2,7 +2,7 @@ - name: Configure Borg backup hosts: all - become: True + become: true roles: - role: borgbackup diff --git a/defaults/main.yml b/defaults/main.yml index 5a174f7..ac3e6ba 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,5 +1,5 @@ --- -borgbackup_required: True +borgbackup_required: true borgbackup_client_user: root borgbackup_ssh_key: "~{{ borgbackup_client_user }}/.ssh/id_borg_rsa" @@ -36,8 +36,8 @@ borgbackup_cron_day: "*" borgbackup_cron_hour: "{{ 5|random(seed=inventory_hostname) }}" borgbackup_cron_minute: "{{ 59|random(seed=inventory_hostname) }}" -borgbackup_appendonly: False -borgbackup_appendonly_repoconfig: False +borgbackup_appendonly: false +borgbackup_appendonly_repoconfig: false borgbackup_management_station: '' borgbackup_management_user: '' borgbackup_management_ssh_pubkey: '' diff --git a/meta/main.yml b/meta/main.yml index a79e23f..baaf749 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -5,15 +5,15 @@ galaxy_info: license: MIT min_ansible_version: 1.9 platforms: - - name: EL - versions: - - all - - name: Debian - versions: - - all - - name: Ubuntu - versions: - - all + - name: EL + versions: + - all + - name: Debian + versions: + - all + - name: Ubuntu + versions: + - all galaxy_tags: - borg - borgbackup diff --git a/molecule/README.md b/molecule/README.md new file mode 100644 index 0000000..b673254 --- /dev/null +++ b/molecule/README.md @@ -0,0 +1,104 @@ +# Borg backup role - Molecule testing + +## Requirements + + +Ansible 2.4 or higher +Molecule 2.17.0 or higher + +Docker host (local or remote through shell environment) + +## Available tests run on all scenarios + +All scenarios run the same tests by default defined in generic_tests. If a test applies to a certain group only, group filtering is done through testinfra with the ansible:// url. +Eg: +``` +testinfra_hosts = ["ansible://borgbackup_servers"] +``` + +### all + +[generic\_files/tests/test_all.py](generic_files/tests/test_all.py) + +Verify the borg-binary is present + + +### client + +Testinfra: [generic\_files/tests/test_client.py](generic_files/tests/test_client.py) +Verifies if all the required parameters are present on the client to perform a backup. It verifies an already existing backup has run and if it has succeeded. + +Testinfra: [generic\_files/tests/test_client_restore.py](generic_files/tests/test_client_restore.py) +Verifies if the restore functionality works correctly. + +### server + +Testinfra: [generic\_files/tests/test_server.py](generic_files/tests/test_server.py) +Verifies if all server-related configurations are in place and if the backup is working from a server perspective. + + +## Available test-scenarios + +### clients + +This tests spins up supported platforms and verifies the basic functionality of both server and client with the generic_tests on a number of platforms and linux distributions. + +``` +borgbackup_appendonly: +borgbackup_servers: +borgbackup_include: +borgbackup_passphrase: +``` + +### multiple + +Testing whether backing up to multiple targets works properly. + +``` +borgbackup_servers: +``` + +### extra_opts + +Testinfra: [generic\_files/tests/test_server.py](generic_files/tests/test_server.py) + +Verify if both pre and post commands are configured at backup time and are run. + + +``` +borgbackup_pre_commands: +borgbackup_post_commands: +``` + +Verify if both inclusion and exclusion of folders is working as expected + +``` +borgbackup_include: +borgbackup_exclude: +``` + +### lamp + +Testinfra: [generic\_files/tests/test_lamp.py](generic_files/tests/test_lamp.py) +Verify a basic lamp setup meaning making sure /var/www/ and automysqlbackup is configured properly and backed up. +As an extra preparation, apache2 and automysqlbackup are installed for verification. + +``` +backup_pre_commands: needs to be extended with automysqlbackup +borgbackup_include: need to contain both /var/www and /var/lib/automysqlbackup +``` + +### mgt + +Testinfra: [generic\_files/tests/test_mgt.py](generic_files/tests/test_mgt.py) +Testing whether the management-station functionality works and all clients are listed. + +``` +borgbackup_management: +borgbackup_management_user: +borgbackup_management_sshkey: +``` + +## restore + +A restore is tested by default in every scenario by the use of the side-effect playbook. [generic\_files/side_effect.yml](generic_files/side_effect.yml) diff --git a/molecule/clients/Dockerfile.j2 b/molecule/clients/Dockerfile.j2 new file mode 120000 index 0000000..0373a45 --- /dev/null +++ b/molecule/clients/Dockerfile.j2 @@ -0,0 +1 @@ +../generic_files/Dockerfile.j2 \ No newline at end of file diff --git a/molecule/clients/INSTALL.rst b/molecule/clients/INSTALL.rst new file mode 100644 index 0000000..3904805 --- /dev/null +++ b/molecule/clients/INSTALL.rst @@ -0,0 +1,26 @@ +******* +Docker driver installation guide +******* + +Requirements +============ + +* General molecule dependencies (see https://molecule.readthedocs.io/en/latest/installation.html) +* Docker Engine +* docker-py +* docker + +Install +======= + +Ansible < 2.6 + +.. code-block:: bash + + $ sudo pip install docker-py + +Ansible >= 2.6 + +.. code-block:: bash + + $ sudo pip install docker diff --git a/molecule/clients/molecule.yml b/molecule/clients/molecule.yml new file mode 100644 index 0000000..5e81832 --- /dev/null +++ b/molecule/clients/molecule.yml @@ -0,0 +1,88 @@ +--- +driver: + name: docker +lint: + name: yamllint +platforms: + - name: ${MOLECULE_SCENARIO_NAME}-cub + hostname: ${MOLECULE_SCENARIO_NAME}-cub + image: ubuntu:bionic + networks: + - name: ${MOLECULE_SCENARIO_NAME}-borg + - name: ${MOLECULE_SCENARIO_NAME}-cux + hostname: ${MOLECULE_SCENARIO_NAME}-cux + image: ubuntu:xenial + networks: + - name: ${MOLECULE_SCENARIO_NAME}-borg + - name: ${MOLECULE_SCENARIO_NAME}-cds + hostname: ${MOLECULE_SCENARIO_NAME}-cds + image: debian:stretch + networks: + - name: ${MOLECULE_SCENARIO_NAME}-borg + - name: ${MOLECULE_SCENARIO_NAME}-cdj + hostname: ${MOLECULE_SCENARIO_NAME}-cdj + image: debian:jessie + networks: + - name: ${MOLECULE_SCENARIO_NAME}-borg + - name: ${MOLECULE_SCENARIO_NAME}-cc7 + hostname: ${MOLECULE_SCENARIO_NAME}-cc7 + image: centos:7 + networks: + - name: ${MOLECULE_SCENARIO_NAME}-borg + groups: + - centos + - name: ${MOLECULE_SCENARIO_NAME}-cc6 + hostname: ${MOLECULE_SCENARIO_NAME}-cc6 + image: centos:6 + networks: + - name: ${MOLECULE_SCENARIO_NAME}-borg + groups: + - centos + - name: ${MOLECULE_SCENARIO_NAME}-sds + hostname: ${MOLECULE_SCENARIO_NAME}-sds + image: debian:stretch + networks: + - name: ${MOLECULE_SCENARIO_NAME}-borg + groups: + - borgbackup_servers +provisioner: + name: ansible + inventory: + group_vars: + all: + borgbackup_appendonly: true + borgbackup_servers: + - fqdn: ${MOLECULE_SCENARIO_NAME}-sds + user: borgds + type: normal + home: /var/backup/ + pool: repos + options: "" + borgbackup_include: + - "/etc" + - "/root" + - "/var/log" + - "/home" + host_vars: + ${MOLECULE_SCENARIO_NAME}-cub: + borgbackup_passphrase: soo6Uabiex + ${MOLECULE_SCENARIO_NAME}-cux: + borgbackup_passphrase: ooFeila1ie + ${MOLECULE_SCENARIO_NAME}-cds: + borgbackup_passphrase: eewoo7paf6J + ${MOLECULE_SCENARIO_NAME}-cdj: + borgbackup_passphrase: aef1Iekahpi + ${MOLECULE_SCENARIO_NAME}-cc7: + borgbackup_passphrase: ASFQiejwasd + ${MOLECULE_SCENARIO_NAME}-cc6: + borgbackup_passphrase: Ansdkfqooia + lint: + name: ansible-lint +scenario: + name: clients +verifier: + name: testinfra + options: + verbose: true + lint: + name: flake8 diff --git a/molecule/clients/playbook.yml b/molecule/clients/playbook.yml new file mode 120000 index 0000000..ace0a3d --- /dev/null +++ b/molecule/clients/playbook.yml @@ -0,0 +1 @@ +../generic_files/playbook.yml \ No newline at end of file diff --git a/molecule/clients/prepare.yml b/molecule/clients/prepare.yml new file mode 120000 index 0000000..28e9479 --- /dev/null +++ b/molecule/clients/prepare.yml @@ -0,0 +1 @@ +../generic_files/prepare.yml \ No newline at end of file diff --git a/molecule/clients/restore.sh.j2 b/molecule/clients/restore.sh.j2 new file mode 120000 index 0000000..07356a7 --- /dev/null +++ b/molecule/clients/restore.sh.j2 @@ -0,0 +1 @@ +../generic_files/restore.sh.j2 \ No newline at end of file diff --git a/molecule/clients/side_effect.yml b/molecule/clients/side_effect.yml new file mode 120000 index 0000000..9bfbda0 --- /dev/null +++ b/molecule/clients/side_effect.yml @@ -0,0 +1 @@ +../generic_files/side_effect.yml \ No newline at end of file diff --git a/molecule/clients/tests b/molecule/clients/tests new file mode 120000 index 0000000..630b75e --- /dev/null +++ b/molecule/clients/tests @@ -0,0 +1 @@ +../generic_files/tests \ No newline at end of file diff --git a/molecule/extra_opts/Dockerfile.j2 b/molecule/extra_opts/Dockerfile.j2 new file mode 120000 index 0000000..0373a45 --- /dev/null +++ b/molecule/extra_opts/Dockerfile.j2 @@ -0,0 +1 @@ +../generic_files/Dockerfile.j2 \ No newline at end of file diff --git a/molecule/extra_opts/INSTALL.rst b/molecule/extra_opts/INSTALL.rst new file mode 100644 index 0000000..3904805 --- /dev/null +++ b/molecule/extra_opts/INSTALL.rst @@ -0,0 +1,26 @@ +******* +Docker driver installation guide +******* + +Requirements +============ + +* General molecule dependencies (see https://molecule.readthedocs.io/en/latest/installation.html) +* Docker Engine +* docker-py +* docker + +Install +======= + +Ansible < 2.6 + +.. code-block:: bash + + $ sudo pip install docker-py + +Ansible >= 2.6 + +.. code-block:: bash + + $ sudo pip install docker diff --git a/molecule/extra_opts/molecule.yml b/molecule/extra_opts/molecule.yml new file mode 100644 index 0000000..dc1c80b --- /dev/null +++ b/molecule/extra_opts/molecule.yml @@ -0,0 +1,58 @@ +--- +driver: + name: docker +lint: + name: yamllint +platforms: + - name: ${MOLECULE_SCENARIO_NAME}-cds + hostname: ${MOLECULE_SCENARIO_NAME}-cds + image: debian:stretch + networks: + - name: ${MOLECULE_SCENARIO_NAME}-borg + groups: + - extra_opts + - name: ${MOLECULE_SCENARIO_NAME}-sds + hostname: ${MOLECULE_SCENARIO_NAME}-sds + image: debian:stretch + networks: + - name: ${MOLECULE_SCENARIO_NAME}-borg + groups: + - borgbackup_servers +provisioner: + name: ansible + inventory: + group_vars: + all: + borgbackup_appendonly: true + borgbackup_servers: + - fqdn: ${MOLECULE_SCENARIO_NAME}-sds + user: borgds + type: normal + home: /var/backup/ + pool: repos + options: "" + host_vars: + ${MOLECULE_SCENARIO_NAME}-cds: + borgbackup_passphrase: soo6Uabiex + borgbackup_include: + - "/etc" + - "/root" + - "/var/log" + - "/home" + - "/var/cache" + borgbackup_exclude: + - "/var/cache/apt" + borgbackup_pre_commands: + - "dpkg --get-selections" + borgbackup_post_commands: + - "apt list" + lint: + name: ansible-lint +scenario: + name: extra_opts +verifier: + name: testinfra + options: + verbose: true + lint: + name: flake8 diff --git a/molecule/extra_opts/playbook.yml b/molecule/extra_opts/playbook.yml new file mode 120000 index 0000000..ace0a3d --- /dev/null +++ b/molecule/extra_opts/playbook.yml @@ -0,0 +1 @@ +../generic_files/playbook.yml \ No newline at end of file diff --git a/molecule/extra_opts/prepare.yml b/molecule/extra_opts/prepare.yml new file mode 120000 index 0000000..28e9479 --- /dev/null +++ b/molecule/extra_opts/prepare.yml @@ -0,0 +1 @@ +../generic_files/prepare.yml \ No newline at end of file diff --git a/molecule/extra_opts/restore.sh.j2 b/molecule/extra_opts/restore.sh.j2 new file mode 120000 index 0000000..07356a7 --- /dev/null +++ b/molecule/extra_opts/restore.sh.j2 @@ -0,0 +1 @@ +../generic_files/restore.sh.j2 \ No newline at end of file diff --git a/molecule/extra_opts/side_effect.yml b/molecule/extra_opts/side_effect.yml new file mode 120000 index 0000000..9bfbda0 --- /dev/null +++ b/molecule/extra_opts/side_effect.yml @@ -0,0 +1 @@ +../generic_files/side_effect.yml \ No newline at end of file diff --git a/molecule/extra_opts/tests b/molecule/extra_opts/tests new file mode 120000 index 0000000..630b75e --- /dev/null +++ b/molecule/extra_opts/tests @@ -0,0 +1 @@ +../generic_files/tests \ No newline at end of file diff --git a/molecule/generic_files/Dockerfile.j2 b/molecule/generic_files/Dockerfile.j2 new file mode 100644 index 0000000..c2bd953 --- /dev/null +++ b/molecule/generic_files/Dockerfile.j2 @@ -0,0 +1,10 @@ +# Molecule managed + +FROM {{ item.image }} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python2-dnf bash && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi diff --git a/molecule/generic_files/playbook.yml b/molecule/generic_files/playbook.yml new file mode 100644 index 0000000..9b2e87b --- /dev/null +++ b/molecule/generic_files/playbook.yml @@ -0,0 +1,5 @@ +--- +- name: Converge + hosts: all + roles: + - role: borgbackup diff --git a/molecule/generic_files/prepare.yml b/molecule/generic_files/prepare.yml new file mode 100644 index 0000000..ef259ac --- /dev/null +++ b/molecule/generic_files/prepare.yml @@ -0,0 +1,54 @@ +--- +- name: prepare debian-based systems + hosts: all:!centos + gather_facts: false + tasks: + - name: install required packages + apt: + name: + - bash + - ca-certificates + - cron + - iputils-ping + - openssh-client + +- name: prepare centos-based systems + hosts: centos + gather_facts: false + tasks: + - name: install required packages + yum: + name: + - anacron + - iputils + - openssh-clients + +- name: prepare debian-based backupservers + hosts: borgbackup_servers + gather_facts: false + tasks: + - name: install required packages + apt: + name: + - openssh-server + - name: start ssh - Debian Server + command: service ssh start + +- name: prepare lamp + hosts: lamp + gather_facts: false + tasks: + - name: Install required packages + apt: + name: + - apache2 + - mysql-server + - automysqlbackup + state: present + notify: start mysql + handlers: + - name: start mysql + service: + name: mysql + state: started + enabled: true diff --git a/molecule/generic_files/restore.sh.j2 b/molecule/generic_files/restore.sh.j2 new file mode 100644 index 0000000..9367f00 --- /dev/null +++ b/molecule/generic_files/restore.sh.j2 @@ -0,0 +1,36 @@ +#!/bin/sh + +export BORG_PASSPHRASE="{{ borgbackup_passphrase }}" + +last_backup=`/usr/local/bin/borg-backup list | grep '^[0-9]' | awk ' { print $1 } ' | tail -1` + +if [ "$1" = "verify" ] + then +{% for b in borgbackup_servers %} +{% if b.type == 'hetzner' %} + REPOSITORY=ssh://{{ b.user }}@{{ b.fqdn }}:23/./{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} +{% else %} + REPOSITORY={{ b.user }}@{{ b.fqdn }}:{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} +{% endif %} + printf "verifying on {{ b.fqdn }} :\n" + /usr/local/bin/borg extract --list --dry-run $REPOSITORY::$last_backup root/sample.txt +{% endfor %} + exit 0 +fi + +if [ "$1" = "restore" ] + then + file="$2" +{% for b in borgbackup_servers %} +{% if b.type == 'hetzner' %} + REPOSITORY=ssh://{{ b.user }}@{{ b.fqdn }}:23/./{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} +{% else %} + REPOSITORY={{ b.user }}@{{ b.fqdn }}:{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} +{% endif %} + rm -rf /root/restore/{{ b.fqdn }} + mkdir -p /root/restore/{{ b.fqdn }} + cd /root/restore/{{ b.fqdn }} + /usr/local/bin/borg extract $REPOSITORY::$last_backup root/sample.txt +{% endfor %} + exit 0 +fi diff --git a/molecule/generic_files/side_effect.yml b/molecule/generic_files/side_effect.yml new file mode 100644 index 0000000..dcf0feb --- /dev/null +++ b/molecule/generic_files/side_effect.yml @@ -0,0 +1,17 @@ +--- +- hosts: all:!borgbackup_servers + gather_facts: false + tasks: + - name: generate random file + command: dd if=/dev/urandom of=/root/sample.txt bs=1M count=4 + args: + creates: /root/sample.txt + - name: template restore test + template: + src: restore.sh.j2 + dest: /root/restore.sh + mode: "0755" + - name: back up + command: /usr/local/bin/borg-backup backup + - name: restore test file + command: /root/restore.sh restore root/sample.txt diff --git a/molecule/generic_files/tests/.flake8 b/molecule/generic_files/tests/.flake8 new file mode 100644 index 0000000..e44b810 --- /dev/null +++ b/molecule/generic_files/tests/.flake8 @@ -0,0 +1,2 @@ +[flake8] +ignore = E501 diff --git a/molecule/generic_files/tests/test_all.py b/molecule/generic_files/tests/test_all.py new file mode 100644 index 0000000..a3a2c9b --- /dev/null +++ b/molecule/generic_files/tests/test_all.py @@ -0,0 +1,6 @@ +def test_borg_binary(host): + borg = host.file("/usr/local/bin/borg") + assert borg.exists + assert borg.user == "root" + assert borg.group == "root" + assert borg.mode == 0o755 diff --git a/molecule/generic_files/tests/test_client.py b/molecule/generic_files/tests/test_client.py new file mode 100644 index 0000000..f43bbd0 --- /dev/null +++ b/molecule/generic_files/tests/test_client.py @@ -0,0 +1,44 @@ +import os +import pytest +from testinfra.utils.ansible_runner import AnsibleRunner + +testinfra_hosts = ["ansible://all:!borgbackup_servers"] + + +def test_log(host): + logfile = host.file("/var/log/borg-backup.log") + assert logfile.contains("Backup succeeded") + assert logfile.user == "root" + assert logfile.group == "root" + + +# to do read inventory variable : export BORG_PASSPHRASE="{{ borgbackup_passphrase }}" +def test_passphrase(host): + pfile = host.file("/root/.borg.passphrase") + assert pfile.contains("BORG_PASSPHRASE=") + assert pfile.user == "root" + assert pfile.group == "root" + + +@pytest.mark.parametrize('server', AnsibleRunner(os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('borgbackup_servers')) +def test_sshconfig(host, server): + sshconf = host.file("/root/.ssh/config") + assert sshconf.contains(" ANSIBLE MANAGED BLOCK %s " % server) + assert sshconf.contains("Host %s" % server) + assert sshconf.user == "root" + assert sshconf.group == "root" + + +def test_scriptfile(host): + script = host.file("/usr/local/bin/borg-backup") + assert script.user == "root" + assert script.group == "root" + assert script.mode == 0o744 + + +@pytest.mark.parametrize('server', AnsibleRunner(os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('borgbackup_servers')) +def test_list_backups(host, server): + command = host.run("bash /usr/local/bin/borg-backup list") + assert command.rc == 0 + assert command.stderr == '' + assert "Archives on %s :\n2" % server in command.stdout diff --git a/molecule/generic_files/tests/test_client_extra_opts.py b/molecule/generic_files/tests/test_client_extra_opts.py new file mode 100644 index 0000000..28cd2d3 --- /dev/null +++ b/molecule/generic_files/tests/test_client_extra_opts.py @@ -0,0 +1,17 @@ +testinfra_hosts = ["ansible://extra_opts"] + + +def test_include_exclude(host): + script = host.file("/usr/local/bin/borg-backup") + assert script.contains("/var/cache") + assert script.contains("--exclude '/var/cache/apt'") + assert script.user == "root" + assert script.group == "root" + + +def test_pre_post_commands(host): + script = host.file("/usr/local/bin/borg-backup") + assert script.contains("dpkg --get-selection") + assert script.contains("apt list") + assert script.user == "root" + assert script.group == "root" diff --git a/molecule/generic_files/tests/test_client_restore.py b/molecule/generic_files/tests/test_client_restore.py new file mode 100644 index 0000000..6e95090 --- /dev/null +++ b/molecule/generic_files/tests/test_client_restore.py @@ -0,0 +1,24 @@ +import os +import pytest +from testinfra.utils.ansible_runner import AnsibleRunner + +testinfra_hosts = ["ansible://all:!borgbackup_servers"] + + +def test_client_sample_file(host): + sample = host.file("/root/sample.txt") + assert sample.is_file + + +@pytest.mark.parametrize('server', AnsibleRunner(os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('borgbackup_servers')) +def test_client_dir(host, server): + command = host.run("diff -s /root/sample.txt /root/restore/%s/root/sample.txt" % server) + assert command.rc == 0 + assert "Files /root/sample.txt and /root/restore/%s/root/sample.txt are identical" % server in command.stdout + + +@pytest.mark.parametrize('server', AnsibleRunner(os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('borgbackup_servers')) +def test_client_verify(host, server): + vcommand = host.run("/root/restore.sh verify") + assert vcommand.rc == 0 + assert vcommand.stdout.rstrip("verifying on %s" % server) diff --git a/molecule/generic_files/tests/test_lamp.py b/molecule/generic_files/tests/test_lamp.py new file mode 100644 index 0000000..20a9423 --- /dev/null +++ b/molecule/generic_files/tests/test_lamp.py @@ -0,0 +1,16 @@ +testinfra_hosts = ["ansible://lamp"] + + +def test_lamp_default(host): + script = host.file("/usr/local/bin/borg-backup") + assert script.contains("/usr/sbin/automysqlbackup") + assert script.contains("/var/lib/automysqlbackup") + assert script.contains("/var/www") + assert script.user == "root" + assert script.group == "root" + + +def test_lamp_automysqlbackup(host): + backup_dir = host.file("/var/lib/automysqlbackup/daily") + assert backup_dir.exists + assert backup_dir.is_directory diff --git a/molecule/generic_files/tests/test_mgt.py b/molecule/generic_files/tests/test_mgt.py new file mode 100644 index 0000000..b9d9189 --- /dev/null +++ b/molecule/generic_files/tests/test_mgt.py @@ -0,0 +1,18 @@ +import os +import pytest +from testinfra.utils.ansible_runner import AnsibleRunner + +testinfra_hosts = ["ansible://borgbackup_management"] + + +# to do read inventory variable : export BORG_PASSPHRASE="{{ borgbackup_passphrase }}" +@pytest.mark.parametrize('client', AnsibleRunner(os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all:!borgbackup_management:!borgbackup_servers')) +def test_prune_script(host, client): + prune = host.file("/root/prune.sh") + assert prune.user == "root" + assert prune.group == "root" + assert prune.mode == 0o700 + assert prune.contains("/usr/local/bin/borg prune") + assert prune.contains("export BORG_PASSPHRASE=") + assert prune.contains("Host: %s" % client) + assert prune.contains(":/var/backup/repos/%s" % client) diff --git a/molecule/generic_files/tests/test_server.py b/molecule/generic_files/tests/test_server.py new file mode 100644 index 0000000..f1e4f9b --- /dev/null +++ b/molecule/generic_files/tests/test_server.py @@ -0,0 +1,23 @@ +import os +import pytest +from testinfra.utils.ansible_runner import AnsibleRunner + +testinfra_hosts = ["ansible://borgbackup_servers"] + + +def test_client_parent_dir(host): + parentdir = host.file("/var/backup/repos") + assert parentdir.is_directory + + +@pytest.mark.parametrize('client', AnsibleRunner(os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all:!borgbackup_servers')) +def test_client_dir(host, client): + clientdir = host.file("/var/backup/repos/%s" % client) + assert clientdir.is_directory + + +@pytest.mark.parametrize('client', AnsibleRunner(os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all:!borgbackup_servers')) +def test_ssh_client_conf(host, client): + sshconf = host.file("/var/backup/.ssh/authorized_keys") + assert sshconf.is_file + assert sshconf.contains("%s;borg serve" % client) diff --git a/molecule/lamp/Dockerfile.j2 b/molecule/lamp/Dockerfile.j2 new file mode 120000 index 0000000..0373a45 --- /dev/null +++ b/molecule/lamp/Dockerfile.j2 @@ -0,0 +1 @@ +../generic_files/Dockerfile.j2 \ No newline at end of file diff --git a/molecule/lamp/INSTALL.rst b/molecule/lamp/INSTALL.rst new file mode 100644 index 0000000..3904805 --- /dev/null +++ b/molecule/lamp/INSTALL.rst @@ -0,0 +1,26 @@ +******* +Docker driver installation guide +******* + +Requirements +============ + +* General molecule dependencies (see https://molecule.readthedocs.io/en/latest/installation.html) +* Docker Engine +* docker-py +* docker + +Install +======= + +Ansible < 2.6 + +.. code-block:: bash + + $ sudo pip install docker-py + +Ansible >= 2.6 + +.. code-block:: bash + + $ sudo pip install docker diff --git a/molecule/lamp/molecule.yml b/molecule/lamp/molecule.yml new file mode 100644 index 0000000..810f148 --- /dev/null +++ b/molecule/lamp/molecule.yml @@ -0,0 +1,46 @@ +--- +driver: + name: docker +lint: + name: yamllint +platforms: + - name: ${MOLECULE_SCENARIO_NAME}-cds + hostname: ${MOLECULE_SCENARIO_NAME}-cds + image: debian:stretch + networks: + - name: ${MOLECULE_SCENARIO_NAME}-borg + groups: + - lamp + - name: ${MOLECULE_SCENARIO_NAME}-sds + hostname: ${MOLECULE_SCENARIO_NAME}-sds + image: debian:stretch + networks: + - name: ${MOLECULE_SCENARIO_NAME}-borg + groups: + - borgbackup_servers +provisioner: + name: ansible + inventory: + group_vars: + all: + borgbackup_appendonly: true + borgbackup_servers: + - fqdn: ${MOLECULE_SCENARIO_NAME}-sds + user: borgds + type: normal + home: /var/backup/ + pool: repos + options: "" + host_vars: + ${MOLECULE_SCENARIO_NAME}-cds: + borgbackup_passphrase: soo6Uabiex + lint: + name: ansible-lint +scenario: + name: lamp +verifier: + name: testinfra + options: + verbose: true + lint: + name: flake8 diff --git a/molecule/lamp/playbook.yml b/molecule/lamp/playbook.yml new file mode 120000 index 0000000..ace0a3d --- /dev/null +++ b/molecule/lamp/playbook.yml @@ -0,0 +1 @@ +../generic_files/playbook.yml \ No newline at end of file diff --git a/molecule/lamp/prepare.yml b/molecule/lamp/prepare.yml new file mode 120000 index 0000000..28e9479 --- /dev/null +++ b/molecule/lamp/prepare.yml @@ -0,0 +1 @@ +../generic_files/prepare.yml \ No newline at end of file diff --git a/molecule/lamp/restore.sh.j2 b/molecule/lamp/restore.sh.j2 new file mode 120000 index 0000000..07356a7 --- /dev/null +++ b/molecule/lamp/restore.sh.j2 @@ -0,0 +1 @@ +../generic_files/restore.sh.j2 \ No newline at end of file diff --git a/molecule/lamp/side_effect.yml b/molecule/lamp/side_effect.yml new file mode 120000 index 0000000..9bfbda0 --- /dev/null +++ b/molecule/lamp/side_effect.yml @@ -0,0 +1 @@ +../generic_files/side_effect.yml \ No newline at end of file diff --git a/molecule/lamp/tests b/molecule/lamp/tests new file mode 120000 index 0000000..630b75e --- /dev/null +++ b/molecule/lamp/tests @@ -0,0 +1 @@ +../generic_files/tests \ No newline at end of file diff --git a/molecule/mgt/Dockerfile.j2 b/molecule/mgt/Dockerfile.j2 new file mode 120000 index 0000000..0373a45 --- /dev/null +++ b/molecule/mgt/Dockerfile.j2 @@ -0,0 +1 @@ +../generic_files/Dockerfile.j2 \ No newline at end of file diff --git a/molecule/mgt/INSTALL.rst b/molecule/mgt/INSTALL.rst new file mode 100644 index 0000000..3904805 --- /dev/null +++ b/molecule/mgt/INSTALL.rst @@ -0,0 +1,26 @@ +******* +Docker driver installation guide +******* + +Requirements +============ + +* General molecule dependencies (see https://molecule.readthedocs.io/en/latest/installation.html) +* Docker Engine +* docker-py +* docker + +Install +======= + +Ansible < 2.6 + +.. code-block:: bash + + $ sudo pip install docker-py + +Ansible >= 2.6 + +.. code-block:: bash + + $ sudo pip install docker diff --git a/molecule/mgt/molecule.yml b/molecule/mgt/molecule.yml new file mode 100644 index 0000000..c7c3bfc --- /dev/null +++ b/molecule/mgt/molecule.yml @@ -0,0 +1,78 @@ +--- +driver: + name: docker +lint: + name: yamllint +platforms: + - name: ${MOLECULE_SCENARIO_NAME}-cds + hostname: ${MOLECULE_SCENARIO_NAME}-cds + image: debian:stretch + networks: + - name: ${MOLECULE_SCENARIO_NAME}-borg + - name: ${MOLECULE_SCENARIO_NAME}-cub + hostname: ${MOLECULE_SCENARIO_NAME}-cub + image: ubuntu:bionic + networks: + - name: ${MOLECULE_SCENARIO_NAME}-borg + - name: ${MOLECULE_SCENARIO_NAME}-sds + hostname: ${MOLECULE_SCENARIO_NAME}-sds + image: debian:stretch + networks: + - name: ${MOLECULE_SCENARIO_NAME}-borg + groups: + - borgbackup_servers + - name: ${MOLECULE_SCENARIO_NAME}-mds + hostname: ${MOLECULE_SCENARIO_NAME}-mds + image: debian:stretch + networks: + - name: ${MOLECULE_SCENARIO_NAME}-borg + groups: + - borgbackup_management +provisioner: + name: ansible + inventory: + group_vars: + all: + borgbackup_appendonly: true + borgbackup_servers: + - fqdn: ${MOLECULE_SCENARIO_NAME}-sds + user: borgds + type: normal + home: /var/backup/ + pool: repos + options: "" + borgbackup_retention: + hourly: 12 + daily: 7 + weekly: 4 + monthly: 6 + yearly: 1 + borgbackup_include: + - "/etc" + - "/root" + - "/var/log" + - "/home" + borgbackup_management_station: ${MOLECULE_SCENARIO_NAME}-mds + borgbackup_servers: + borgbackup_required: false + borgbackup_management: + borgbackup_management_user: root + borgbackup_management_sshkey: | + ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDXM5MuTbKbciopdHP314g0b72O5xaCf+4mt97bLPBwmW8UpOebWMkE+jYKET2B9fiOkyU9aWq+Tx7tfo31avN0nRf/VsxNupLcH605KdYOmw+EPjVDFLRXZb3j0AGDJM9YCd/KrwJlf8Tu4sXq0RhRNmqRBsBMjevpKCioQLzWJ1oCs1TLWxTS4nN7Jwy+Ou2gJEB7jKj907pFiCIPWRU4LLHlwE7DjunBr4THgE2Sj0tWkiYDKKoH++Rm62wVYL0oTLOJrQuqu52FEniPDgTnZwhy7b5MAHF5f2BscBorQxEkToWOVfX14Mzf+HC/arT2IPoE1Iv2XdWoiGiaWnyr + host_vars: + ${MOLECULE_SCENARIO_NAME}-cds: + borgbackup_passphrase: soo6Uabiex + ${MOLECULE_SCENARIO_NAME}-cub: + borgbackup_passphrase: iousheafqj + ${MOLECULE_SCENARIO_NAME}-mds: + borgbackup_passphrase: Zuequei1ro + lint: + name: ansible-lint +scenario: + name: mgt +verifier: + name: testinfra + options: + verbose: true + lint: + name: flake8 diff --git a/molecule/mgt/playbook.yml b/molecule/mgt/playbook.yml new file mode 120000 index 0000000..ace0a3d --- /dev/null +++ b/molecule/mgt/playbook.yml @@ -0,0 +1 @@ +../generic_files/playbook.yml \ No newline at end of file diff --git a/molecule/mgt/prepare.yml b/molecule/mgt/prepare.yml new file mode 120000 index 0000000..28e9479 --- /dev/null +++ b/molecule/mgt/prepare.yml @@ -0,0 +1 @@ +../generic_files/prepare.yml \ No newline at end of file diff --git a/molecule/mgt/restore.sh.j2 b/molecule/mgt/restore.sh.j2 new file mode 120000 index 0000000..07356a7 --- /dev/null +++ b/molecule/mgt/restore.sh.j2 @@ -0,0 +1 @@ +../generic_files/restore.sh.j2 \ No newline at end of file diff --git a/molecule/mgt/side_effect.yml b/molecule/mgt/side_effect.yml new file mode 120000 index 0000000..9bfbda0 --- /dev/null +++ b/molecule/mgt/side_effect.yml @@ -0,0 +1 @@ +../generic_files/side_effect.yml \ No newline at end of file diff --git a/molecule/mgt/tests b/molecule/mgt/tests new file mode 120000 index 0000000..630b75e --- /dev/null +++ b/molecule/mgt/tests @@ -0,0 +1 @@ +../generic_files/tests \ No newline at end of file diff --git a/molecule/multiple/Dockerfile.j2 b/molecule/multiple/Dockerfile.j2 new file mode 120000 index 0000000..0373a45 --- /dev/null +++ b/molecule/multiple/Dockerfile.j2 @@ -0,0 +1 @@ +../generic_files/Dockerfile.j2 \ No newline at end of file diff --git a/molecule/multiple/INSTALL.rst b/molecule/multiple/INSTALL.rst new file mode 100644 index 0000000..3904805 --- /dev/null +++ b/molecule/multiple/INSTALL.rst @@ -0,0 +1,26 @@ +******* +Docker driver installation guide +******* + +Requirements +============ + +* General molecule dependencies (see https://molecule.readthedocs.io/en/latest/installation.html) +* Docker Engine +* docker-py +* docker + +Install +======= + +Ansible < 2.6 + +.. code-block:: bash + + $ sudo pip install docker-py + +Ansible >= 2.6 + +.. code-block:: bash + + $ sudo pip install docker diff --git a/molecule/multiple/molecule.yml b/molecule/multiple/molecule.yml new file mode 100644 index 0000000..ef0a37b --- /dev/null +++ b/molecule/multiple/molecule.yml @@ -0,0 +1,73 @@ +--- +driver: + name: docker +lint: + name: yamllint + options: + config-file: ${MOLECULE_PROJECT_DIRECTORY}/.yamllint +platforms: + - name: ${MOLECULE_SCENARIO_NAME}-cds + hostname: ${MOLECULE_SCENARIO_NAME}-cds + image: debian:stretch + networks: + - name: ${MOLECULE_SCENARIO_NAME}-borg + - name: ${MOLECULE_SCENARIO_NAME}-cub + hostname: ${MOLECULE_SCENARIO_NAME}-cub + image: ubuntu:bionic + networks: + - name: ${MOLECULE_SCENARIO_NAME}-borg + - name: ${MOLECULE_SCENARIO_NAME}-sds + hostname: ${MOLECULE_SCENARIO_NAME}-sds + image: ubuntu:xenial + networks: + - name: ${MOLECULE_SCENARIO_NAME}-borg + groups: + - borgbackup_servers + - name: ${MOLECULE_SCENARIO_NAME}-sub + hostname: ${MOLECULE_SCENARIO_NAME}-sub + image: ubuntu:bionic + networks: + - name: ${MOLECULE_SCENARIO_NAME}-borg + groups: + - borgbackup_servers +provisioner: + name: ansible + inventory: + group_vars: + all: + borgbackup_appendonly: true + borgbackup_servers: + - fqdn: ${MOLECULE_SCENARIO_NAME}-sds + user: borgdj + type: normal + home: /var/backup/ + pool: repos + options: "" + - fqdn: ${MOLECULE_SCENARIO_NAME}-sub + user: borgux + type: normal + home: /var/backup/ + pool: repos + options: "" + borgbackup_include: + - "/etc" + - "/root" + - "/var/log" + - "/home" + host_vars: + ${MOLECULE_SCENARIO_NAME}-cds: + borgbackup_passphrase: soo6Uabiex + ${MOLECULE_SCENARIO_NAME}-cub: + borgbackup_passphrase: lidafruhkj + ${MOLECULE_SCENARIO_NAME}-sub: + borgbackup_passphrase: oiasdfoain + lint: + name: ansible-lint +scenario: + name: multiple +verifier: + name: testinfra + options: + verbose: true + lint: + name: flake8 diff --git a/molecule/multiple/playbook.yml b/molecule/multiple/playbook.yml new file mode 120000 index 0000000..ace0a3d --- /dev/null +++ b/molecule/multiple/playbook.yml @@ -0,0 +1 @@ +../generic_files/playbook.yml \ No newline at end of file diff --git a/molecule/multiple/prepare.yml b/molecule/multiple/prepare.yml new file mode 120000 index 0000000..28e9479 --- /dev/null +++ b/molecule/multiple/prepare.yml @@ -0,0 +1 @@ +../generic_files/prepare.yml \ No newline at end of file diff --git a/molecule/multiple/restore.sh.j2 b/molecule/multiple/restore.sh.j2 new file mode 120000 index 0000000..07356a7 --- /dev/null +++ b/molecule/multiple/restore.sh.j2 @@ -0,0 +1 @@ +../generic_files/restore.sh.j2 \ No newline at end of file diff --git a/molecule/multiple/side_effect.yml b/molecule/multiple/side_effect.yml new file mode 120000 index 0000000..9bfbda0 --- /dev/null +++ b/molecule/multiple/side_effect.yml @@ -0,0 +1 @@ +../generic_files/side_effect.yml \ No newline at end of file diff --git a/molecule/multiple/tests b/molecule/multiple/tests new file mode 120000 index 0000000..630b75e --- /dev/null +++ b/molecule/multiple/tests @@ -0,0 +1 @@ +../generic_files/tests \ No newline at end of file diff --git a/tasks/borg-client.yml b/tasks/borg-client.yml index d67df3b..ee71a62 100644 --- a/tasks/borg-client.yml +++ b/tasks/borg-client.yml @@ -2,7 +2,7 @@ - name: client | generate ssh key for this machine user: name: "{{ borgbackup_client_user }}" - generate_ssh_key: yes + generate_ssh_key: true ssh_key_bits: 2048 ssh_key_file: "{{ borgbackup_ssh_key }}" ssh_key_type: rsa @@ -10,19 +10,19 @@ - name: client | fetch ssh-key shell: "cat {{ borgbackup_ssh_key }}.pub" register: sshkey - changed_when: False + changed_when: false - name: client | write passphrase lineinfile: dest: "~{{ borgbackup_client_user }}/.borg.passphrase" state: "present" line: 'export BORG_PASSPHRASE="{{ borgbackup_passphrase }}"' - create: "yes" + create: true - name: client | disable strict key checking for backup servers blockinfile: dest: "~{{ borgbackup_client_user }}/.ssh/config" - create: yes + create: true marker: "### {mark} ANSIBLE MANAGED BLOCK {{ item.fqdn }} ###" content: | Host {{ item.fqdn }} @@ -46,7 +46,7 @@ - name: client | get authorized_keys file raw: scp {{ item.user }}@{{ item.fqdn }}:.ssh/authorized_keys /tmp/authkeys-{{ item.type }}-{{ item.fqdn }}-authkeys delegate_to: localhost - become: no + become: false when: item.type in ['rsync.net','hetzner'] with_items: "{{ borgbackup_servers }}" changed_when: false @@ -57,9 +57,9 @@ key: "{{ sshkey.stdout }}" key_options: 'command="cd {{ item.pool }}/{{ inventory_hostname }};/usr/local/bin/borg1 serve {% if borgbackup_appendonly %}--append-only {% endif %} --restrict-to-path {{ item.pool }}/{{ inventory_hostname }}",no-port-forwarding,no-X11-forwarding,no-pty,no-agent-forwarding,no-user-rc' path: "/tmp/authkeys-{{ item.type }}-{{ item.fqdn }}-authkeys" - manage_dir: no + manage_dir: false delegate_to: localhost - become: no + become: false when: item.type in ['rsync.net','hetzner'] with_items: "{{ borgbackup_servers }}" register: authkeys @@ -67,7 +67,7 @@ - name: client | upload local authorized_keys to rsync.net / hetzner raw: scp /tmp/authkeys-{{ item.type }}-{{ item.fqdn }}-authkeys {{ item.user }}@{{ item.fqdn }}:.ssh/authorized_keys delegate_to: localhost - become: no + become: false when: item.type in ['rsync.net','hetzner'] and authkeys.changed with_items: "{{ borgbackup_servers }}" @@ -76,7 +76,7 @@ path: /tmp/authkeys-{{ item.type }}-{{ item.fqdn }}-authkeys state: absent delegate_to: localhost - become: no + become: false with_items: "{{ borgbackup_servers }}" when: authkeys.changed changed_when: false @@ -109,12 +109,12 @@ day: "{{ borgbackup_cron_day }}" job: "/usr/local/bin/borg-backup backup" -- name: client | disable automysqlbackup cronjob, it's in our pre-backup-tasks +- name: client | disable automysqlbackup cronjob, it's in our pre-backup-tasks lineinfile: dest: "/etc/cron.daily/automysqlbackup" regexp: "^/usr/sbin/automysqlbackup$" line: "#/usr/sbin/automysqlbackup" state: "present" - backrefs: "yes" - create: "no" + backrefs: true + create: false when: automysql.stat.isdir is defined and automysql.stat.isdir == True diff --git a/tasks/borg-server.yml b/tasks/borg-server.yml index 8a4abf1..212ba69 100644 --- a/tasks/borg-server.yml +++ b/tasks/borg-server.yml @@ -10,6 +10,15 @@ run_once: true when: item.type == 'normal' +- name: server | set permissions + file: + dest: "{{ item.home }}" + owner: "{{ item.user }}" + group: "{{ item.user }}" + delegate_to: "{{ item.fqdn }}" + with_items: "{{ borgbackup_servers }}" + when: item.type == 'normal' + - name: server | create directories file: path: "{{ item.home}}{{ item.pool }}" diff --git a/tasks/main.yml b/tasks/main.yml index a394037..a09c8a4 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,6 +1,7 @@ --- # Due to inverse logic behaviour when searching for an item in an undefined list. -- set_fact: +- name: setting facts + set_fact: borgbackup_servers_group: "{{ groups.borgbackup_servers | default([]) }} " borgbackup_management_group: "{{ groups.borgbackup_management | default([]) }}" diff --git a/tasks/management.yml b/tasks/management.yml index 7f280a0..74a2e2e 100644 --- a/tasks/management.yml +++ b/tasks/management.yml @@ -20,7 +20,7 @@ - name: management | get rsync.net authorized_keys file raw: scp {{ item.user }}@{{ item.fqdn }}:.ssh/authorized_keys /tmp/rsync.net-{{ item.fqdn }}-authkeys delegate_to: localhost - become: no + become: false when: item.type == 'rsync.net' with_items: "{{ borgbackup_servers }}" changed_when: false @@ -30,9 +30,9 @@ user: "{{ ansible_user_id }}" key: "{{ borgbackup_management_ssh_pubkey }}" path: "/tmp/rsync.net-{{ item.fqdn }}-authkeys" - manage_dir: no + manage_dir: false delegate_to: localhost - become: no + become: false when: item.type == 'rsync.net' with_items: "{{ borgbackup_servers }}" register: authkeys @@ -40,7 +40,7 @@ - name: management | upload local authorized_keys to rsync.net raw: scp /tmp/rsync.net-{{ item.fqdn }}-authkeys {{ item.user }}@{{ item.fqdn }}:.ssh/authorized_keys delegate_to: localhost - become: no + become: false when: item.type == 'rsync.net' and authkeys.changed with_items: "{{ borgbackup_servers }}" @@ -49,7 +49,7 @@ path: /tmp/rsync.net-{{ item.fqdn }}-authkeys state: absent delegate_to: localhost - become: no + become: false with_items: "{{ borgbackup_servers }}" when: authkeys.changed changed_when: false