From 4e2f4bf3fe1e96889283c2750fd6401eb8f0d0f1 Mon Sep 17 00:00:00 2001 From: Dieter Verhelst Date: Thu, 11 Jan 2018 23:21:43 +0100 Subject: [PATCH 01/15] Hetzner support --- README.md | 9 ++++++++- tasks/borg-client.yml | 30 +++++++++++++++++++++--------- templates/borg-backup.sh.j2 | 14 +++++++------- 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 5033482..ec9f9d1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Borg backup role This role installs Borg backup on borgbackup\_servers and clients. The role contains a wrapper-script 'borg-backup' to ease the usage on the client. Supported options include borg-backup info | init | list | backup | mount. Automysqlbackup will run as pre-backup command if it's installed. -The role supports both self hosted and rsync.net as Borg server. +The role supports both self hosted and offsite backup-storage such as rsync.net and hetzner storage box as Borg server. It's possible to configure append-only repositories to secure the backups against deletion from the client. @@ -29,6 +29,13 @@ borgbackup_servers: home: "" pool: repos options: "--remote-path=borg1" + - fqdn: username.your-storagebox.de + user: username + type: hetzner + home: "" + pool: repos + options: "" + borgbackup_retention: hourly: 12 diff --git a/tasks/borg-client.yml b/tasks/borg-client.yml index 919a026..fbd6ffa 100644 --- a/tasks/borg-client.yml +++ b/tasks/borg-client.yml @@ -37,12 +37,12 @@ when: item.type == 'normal' with_items: "{{ borgbackup_servers }}" -# rsync.net has no python, so we can only use raw to manage ssh keys - workaround with local tmp file -- name: client | get rsync.net authorized_keys file - raw: scp {{ item.user }}@{{ item.fqdn }}:.ssh/authorized_keys /tmp/rsync.net-{{ item.fqdn }}-authkeys +# rsync.net and hetzner have no python, so we can only use raw to manage ssh keys - workaround with local tmp file +- name: client | get authorized_keys file + raw: scp {{ item.user }}@{{ item.fqdn }}:.ssh/authorized_keys /tmp/authkeys-{{ item.type }}-{{ item.fqdn }}-authkeys delegate_to: localhost become: no - when: item.type == 'rsync.net' + when: item.type in ['rsync.net','hetzner'] with_items: "{{ borgbackup_servers }}" changed_when: false @@ -51,7 +51,7 @@ user: "{{ ansible_user_id }}" key: "{{ sshkey.stdout }}" key_options: 'command="cd {{ item.pool }}/{{ inventory_hostname }};/usr/local/bin/borg1 serve {% if borgbackup_appendonly %}--append-only {% endif %} --restrict-to-path {{ item.pool }}/{{ inventory_hostname }}",no-port-forwarding,no-X11-forwarding,no-pty,no-agent-forwarding,no-user-rc' - path: "/tmp/rsync.net-{{ item.fqdn }}-authkeys" + path: "/tmp/authkeys-{{ item.type }}-{{ item.fqdn }}-authkeys" manage_dir: no delegate_to: localhost become: no @@ -59,16 +59,28 @@ with_items: "{{ borgbackup_servers }}" register: authkeys -- name: client | upload local authorized_keys to rsync.net - raw: scp /tmp/rsync.net-{{ item.fqdn }}-authkeys {{ item.user }}@{{ item.fqdn }}:.ssh/authorized_keys +- name: client | modify local hetzner authorized_keys + authorized_key: + user: "{{ ansible_user_id }}" + key: "{{ sshkey.stdout }}" + path: "/tmp/authkeys-{{ item.type }}-{{ item.fqdn }}-authkeys" + manage_dir: no delegate_to: localhost become: no - when: item.type == 'rsync.net' and authkeys.changed + when: item.type == 'hetzner' + with_items: "{{ borgbackup_servers }}" + register: authkeys + +- name: client | upload local authorized_keys to rsync.net / hetzner + raw: scp /tmp/authkeys-{{ item.type }}-{{ item.fqdn }}-authkeys {{ item.user }}@{{ item.fqdn }}:.ssh/authorized_keys + delegate_to: localhost + become: no + when: item.type in ['rsync.net','hetzner'] and authkeys.changed with_items: "{{ borgbackup_servers }}" - name: client | remove tmp authorized_keys files file: - path: /tmp/rsync.net-{{ item.fqdn }}-authkeys + path: /tmp/authkeys-{{ item.type }}-{{ item.fqdn }}-authkeys state: absent delegate_to: localhost become: no diff --git a/templates/borg-backup.sh.j2 b/templates/borg-backup.sh.j2 index 0e7bd60..96f9348 100644 --- a/templates/borg-backup.sh.j2 +++ b/templates/borg-backup.sh.j2 @@ -15,7 +15,7 @@ if [ "$1" = "info" ] then if [ -z "$2" ]; then printf "run $0 with list and use the backup-tag to request information\n"; exit 1; fi {% for b in borgbackup_servers %} - REPOSITORY={{ b.user }}@{{ b.fqdn }}:{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} + REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} /usr/local/bin/borg info $REPOSITORY::$2 {{ b.options }} {% endfor %} exit 0 @@ -27,7 +27,7 @@ if [ "$1" = "mount" ] if [ -z "$3" ]; then printf "Select the backup to mount\n"; exit 1; fi if [ -z "$4" ]; then printf "Select the path to mount the backup on\n"; exit 1; fi {% for b in borgbackup_servers %} - REPOSITORY={{ b.user }}@{{ b.fqdn }}:{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} + REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} /usr/local/bin/borg mount $REPOSITORY::$3 $4 {{ b.options }} if [ "$?" = "0" ]; then printf "Backup mounted on $4, do not forget to unmount!\n"; fi exit 0 @@ -37,7 +37,7 @@ fi if [ "$1" = "list" ] then {% for b in borgbackup_servers %} - REPOSITORY={{ b.user }}@{{ b.fqdn }}:{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} + REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} printf "Archives on {{ b.fqdn }} :\n" /usr/local/bin/borg list -v $REPOSITORY {{ b.options }} {% endfor %} @@ -47,8 +47,8 @@ fi if [ "$1" = "init" ] then {% for b in borgbackup_servers %} - REPOSITORY={{ b.user }}@{{ b.fqdn }}:{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} - borg init $REPOSITORY {{ b.options }} + REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} + /usr/local/bin/borg init $REPOSITORY {{ b.options }} {% endfor %} exit 0 fi @@ -64,11 +64,11 @@ if [ "$1" = "backup" ] {% for b in borgbackup_servers %} printf "Backing up to {{ b.fqdn }} :\n" - REPOSITORY={{ b.user }}@{{ b.fqdn }}:{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} + REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} /usr/local/bin/borg create --compression zlib,6 --stats $REPOSITORY::$date {{ b.options }} {% for dir in borgbackup_include %}{{ dir }} {% endfor %}{% if automysql.stat.isdir is defined and automysql.stat.isdir == True %}/var/lib/automysqlbackup{% endif %} - if [ "$?" -eq "0" ]; then printf "Backup succeeded on $date\n" >> /var/log/borg-backup.log; fi + if [ "$?" -eq "0" ]; then printf "Backup succeeded on $date to {{ b.fqdn }}\n" >> /var/log/borg-backup.log; fi {% if not borgbackup_appendonly %} # prune old backups From 8cfe198f7ec11178d5a93dfe91a3d2373e396744 Mon Sep 17 00:00:00 2001 From: Dieter Verhelst Date: Thu, 11 Jan 2018 23:24:11 +0100 Subject: [PATCH 02/15] Adding exclude dirs and post-backup tasks --- defaults/main.yml | 4 ++++ templates/borg-backup.sh.j2 | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/defaults/main.yml b/defaults/main.yml index cedf50a..cd13ae7 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -10,6 +10,8 @@ borgbackup_download_url: "https://github.com/borgbackup/borg/releases/download/{ borgbackup_pre_commands: - '[[ ! -f "/usr/sbin/automysqlbackup" ]] || /usr/sbin/automysqlbackup' +borgbackup_post_commands: [] + borgbackup_include: - "/etc" - "/home" @@ -17,6 +19,8 @@ borgbackup_include: - "/var/www" - "/var/log" +borgbackup_exclude: [] + borgbackup_retention: hourly: 12 daily: 7 diff --git a/templates/borg-backup.sh.j2 b/templates/borg-backup.sh.j2 index 96f9348..bf85067 100644 --- a/templates/borg-backup.sh.j2 +++ b/templates/borg-backup.sh.j2 @@ -66,7 +66,7 @@ if [ "$1" = "backup" ] printf "Backing up to {{ b.fqdn }} :\n" REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} - /usr/local/bin/borg create --compression zlib,6 --stats $REPOSITORY::$date {{ b.options }} {% for dir in borgbackup_include %}{{ dir }} {% endfor %}{% if automysql.stat.isdir is defined and automysql.stat.isdir == True %}/var/lib/automysqlbackup{% endif %} + /usr/local/bin/borg create -p --compression zlib,6 --stats $REPOSITORY::$date {{ b.options }} {% for dir in borgbackup_include %}{{ dir }} {% endfor %}{% if automysql.stat.isdir is defined and automysql.stat.isdir == True %}/var/lib/automysqlbackup{% endif %} {% for dir in borgbackup_exclude %} --exclude '{{ dir }}'{% endfor %} if [ "$?" -eq "0" ]; then printf "Backup succeeded on $date to {{ b.fqdn }}\n" >> /var/log/borg-backup.log; fi @@ -75,5 +75,11 @@ if [ "$1" = "backup" ] /usr/local/bin/borg prune -v $REPOSITORY {{ b.options }} -H {{ borgbackup_retention.hourly }} -d {{ borgbackup_retention.daily }} -w {{ borgbackup_retention.weekly }} -m {{ borgbackup_retention.monthly }} -y {{ borgbackup_retention.yearly }} {% endif %} {% endfor %} + + # Running some commands post-backup +{% for postcommand in borgbackup_post_commands %} + {{ postcommand }} +{% endfor %} + fi From 0aa85752278a871ead1f0564d692add28fa92c54 Mon Sep 17 00:00:00 2001 From: Dieter Verhelst Date: Thu, 11 Jan 2018 23:26:43 +0100 Subject: [PATCH 03/15] Making it possible to define shell and support other os-ses --- defaults/main.yml | 4 ++++ tasks/borg-client.yml | 4 ++-- tasks/install.yml | 4 ++-- templates/borg-backup.sh.j2 | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index cd13ae7..cc09f97 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -37,3 +37,7 @@ borgbackup_appendonly: False borgbackup_management_station: '' borgbackup_management_user: '' borgbackup_management_ssh_pubkey: '' + +borgbackup_owner: root +borgbackup_group: root +borgbackup_shell: "/bin/bash" diff --git a/tasks/borg-client.yml b/tasks/borg-client.yml index fbd6ffa..3505876 100644 --- a/tasks/borg-client.yml +++ b/tasks/borg-client.yml @@ -96,8 +96,8 @@ template: src: "borg-backup.sh.j2" dest: "/usr/local/bin/borg-backup" - owner: "root" - group: "root" + owner: "{{ borgbackup_owner }}" + group: "{{ borgbackup_group }}" mode: "0744" - name: client | create backup-directory on backup server diff --git a/tasks/install.yml b/tasks/install.yml index 87ff287..1a59145 100644 --- a/tasks/install.yml +++ b/tasks/install.yml @@ -3,8 +3,8 @@ get_url: dest: "/usr/local/bin/borg" checksum: "{{ borgbackup_checksum }}" - owner: "root" - group: "root" + owner: "{{ borgbackup_owner }}" + group: "{{ borgbackup_group }}" mode: "0755" url: "{{ borgbackup_download_url }}" tags: borginstall diff --git a/templates/borg-backup.sh.j2 b/templates/borg-backup.sh.j2 index bf85067..3f89d6d 100644 --- a/templates/borg-backup.sh.j2 +++ b/templates/borg-backup.sh.j2 @@ -1,4 +1,4 @@ -#!/bin/bash +#!{{ borgbackup_shell }} if [ -z "$1" ] || [ ! -z "$2" ] then From 613825ae04072cf42329a604662b2f1cd36e2ccf Mon Sep 17 00:00:00 2001 From: Luc Stroobant Date: Sun, 21 Jan 2018 19:40:10 +0100 Subject: [PATCH 04/15] bump version --- defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/defaults/main.yml b/defaults/main.yml index cedf50a..59f3745 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -3,7 +3,7 @@ borgbackup_required: True borgbackup_ssh_key: "~/.ssh/id_borg_rsa" -borgbackup_version: "1.0.11" +borgbackup_version: "1.1.4" borgbackup_checksum: "sha256:fbdc0e0d6d05a0935551f2f408f370236a76b7a30d3bb90a31c3628fe3611359" borgbackup_download_url: "https://github.com/borgbackup/borg/releases/download/{{ borgbackup_version }}/borg-linux64" From 051094c3a16474a26e4236b3a367a0519d78364a Mon Sep 17 00:00:00 2001 From: Luc Stroobant Date: Sun, 21 Jan 2018 19:42:31 +0100 Subject: [PATCH 05/15] update meta --- meta/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/main.yml b/meta/main.yml index af4ad7e..a79e23f 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -1,7 +1,7 @@ --- galaxy_info: author: Luc Stroobant and Dieter Verhelst - description: Borg backup server and client role + description: Install Borg backup server and client (with rsync.net server support) license: MIT min_ansible_version: 1.9 platforms: From 1afb01aa4a9d2ead5fd0c093c71f14146b442580 Mon Sep 17 00:00:00 2001 From: Dieter Verhelst Date: Sun, 21 Jan 2018 23:12:48 +0100 Subject: [PATCH 06/15] adding key-options for hetzner too, unifying modification of authorized_keys with rsync and hetzner --- tasks/borg-client.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/tasks/borg-client.yml b/tasks/borg-client.yml index 3505876..fce6b6a 100644 --- a/tasks/borg-client.yml +++ b/tasks/borg-client.yml @@ -46,7 +46,7 @@ with_items: "{{ borgbackup_servers }}" changed_when: false -- name: client | modify local rsync.net authorized_keys +- name: client | modify local rsync.net/hetzner authorized_keys authorized_key: user: "{{ ansible_user_id }}" key: "{{ sshkey.stdout }}" @@ -55,19 +55,7 @@ manage_dir: no delegate_to: localhost become: no - when: item.type == 'rsync.net' - with_items: "{{ borgbackup_servers }}" - register: authkeys - -- name: client | modify local hetzner authorized_keys - authorized_key: - user: "{{ ansible_user_id }}" - key: "{{ sshkey.stdout }}" - path: "/tmp/authkeys-{{ item.type }}-{{ item.fqdn }}-authkeys" - manage_dir: no - delegate_to: localhost - become: no - when: item.type == 'hetzner' + when: item.type in ['rsync.net','hetzner'] with_items: "{{ borgbackup_servers }}" register: authkeys From 645694e4119196666b0d5965011a168bef2fdcc8 Mon Sep 17 00:00:00 2001 From: Luc Stroobant Date: Sat, 27 Jan 2018 19:55:30 +0100 Subject: [PATCH 07/15] 1.0.11 --- defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/defaults/main.yml b/defaults/main.yml index d4312d4..cc09f97 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -3,7 +3,7 @@ borgbackup_required: True borgbackup_ssh_key: "~/.ssh/id_borg_rsa" -borgbackup_version: "1.1.4" +borgbackup_version: "1.0.11" borgbackup_checksum: "sha256:fbdc0e0d6d05a0935551f2f408f370236a76b7a30d3bb90a31c3628fe3611359" borgbackup_download_url: "https://github.com/borgbackup/borg/releases/download/{{ borgbackup_version }}/borg-linux64" From 443b5b4ee516f3e4b85df895c8db18b587a8de5e Mon Sep 17 00:00:00 2001 From: Luc Stroobant Date: Sun, 28 Jan 2018 18:01:15 +0100 Subject: [PATCH 08/15] client user --- defaults/main.yml | 4 ++-- tasks/borg-client.yml | 13 +++++++++---- templates/borg-backup.sh.j2 | 4 ++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index cc09f97..d97f8bc 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,7 +1,8 @@ --- borgbackup_required: True -borgbackup_ssh_key: "~/.ssh/id_borg_rsa" +borgbackup_client_user: root +borgbackup_ssh_key: "~{{ borgbackup_client_user }}/.ssh/id_borg_rsa" borgbackup_version: "1.0.11" borgbackup_checksum: "sha256:fbdc0e0d6d05a0935551f2f408f370236a76b7a30d3bb90a31c3628fe3611359" @@ -28,7 +29,6 @@ borgbackup_retention: monthly: 6 yearly: 1 -borgbackup_cron_user: root borgbackup_cron_day: "*" borgbackup_cron_hour: "{{ 5|random }}" borgbackup_cron_minute: "{{ 59|random }}" diff --git a/tasks/borg-client.yml b/tasks/borg-client.yml index fce6b6a..1f297a7 100644 --- a/tasks/borg-client.yml +++ b/tasks/borg-client.yml @@ -1,6 +1,11 @@ --- - name: client | generate ssh key for this machine - shell: if [ -f {{ borgbackup_ssh_key }} ]; then rm -f {{ borgbackup_ssh_key }}; fi && ssh-keygen -q -t rsa -b 4096 -f {{ borgbackup_ssh_key }} -N "" creates="{{ borgbackup_ssh_key }}.pub" + user: + name: "{{ borgbackup_client_user }}" + generate_ssh_key: yes + ssh_key_bits: 2048 + ssh_key_file: "{{ borgbackup_ssh_key }}" + ssh_key_type: rsa - name: client | fetch ssh-key shell: "cat {{ borgbackup_ssh_key }}.pub" @@ -9,14 +14,14 @@ - name: client | write passphrase lineinfile: - dest: "/root/.borg.passphrase" + dest: "~{{ borgbackup_client_user }}/.borg.passphrase" state: "present" line: 'export BORG_PASSPHRASE="{{ borgbackup_passphrase }}"' create: "yes" - name: client | disable strict key checking for backup servers blockinfile: - dest: /root/.ssh/config + dest: "~{{ borgbackup_client_user }}/.ssh/config" create: yes marker: "### {mark} ANSIBLE MANAGED BLOCK {{ item.fqdn }} ###" content: | @@ -96,7 +101,7 @@ - name: client | create backup cronjob cron: cron_file: "borg-backup" - user: "{{ borgbackup_cron_user }}" + user: "{{ borgbackup_client_user }}" name: "borg-backup" minute: "{{ borgbackup_cron_minute }}" hour: "{{ borgbackup_cron_hour }}" diff --git a/templates/borg-backup.sh.j2 b/templates/borg-backup.sh.j2 index 3f89d6d..30afe2b 100644 --- a/templates/borg-backup.sh.j2 +++ b/templates/borg-backup.sh.j2 @@ -7,7 +7,7 @@ fi # Sourcing the backup-passphrase -. /root/.borg.passphrase +. ~{{ borgbackup_client_user }}/.borg.passphrase # Small helper commands, like listing backups, will help us in the future :) @@ -48,7 +48,7 @@ if [ "$1" = "init" ] then {% for b in borgbackup_servers %} REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} - /usr/local/bin/borg init $REPOSITORY {{ b.options }} + /usr/local/bin/borg init --encryption=keyfile $REPOSITORY {{ b.options }} {% endfor %} exit 0 fi From 34d4ce1721337c672f8429316de044ae6f9df419 Mon Sep 17 00:00:00 2001 From: Luc Stroobant Date: Sun, 28 Jan 2018 19:35:00 +0100 Subject: [PATCH 09/15] fix 1.1 support --- defaults/main.yml | 5 +++-- templates/borg-backup.sh.j2 | 12 ++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index d97f8bc..49144a6 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -4,10 +4,11 @@ borgbackup_required: True borgbackup_client_user: root borgbackup_ssh_key: "~{{ borgbackup_client_user }}/.ssh/id_borg_rsa" -borgbackup_version: "1.0.11" -borgbackup_checksum: "sha256:fbdc0e0d6d05a0935551f2f408f370236a76b7a30d3bb90a31c3628fe3611359" +borgbackup_version: "1.1.4" +borgbackup_checksum: "sha256:4ecf507f21f0db7c437b2ef34566273d7ba5a7d05e921c6f0e3406c3f96933a7" borgbackup_download_url: "https://github.com/borgbackup/borg/releases/download/{{ borgbackup_version }}/borg-linux64" +borgbackup_encryption_mode: keyfile borgbackup_pre_commands: - '[[ ! -f "/usr/sbin/automysqlbackup" ]] || /usr/sbin/automysqlbackup' diff --git a/templates/borg-backup.sh.j2 b/templates/borg-backup.sh.j2 index 30afe2b..cc0884f 100644 --- a/templates/borg-backup.sh.j2 +++ b/templates/borg-backup.sh.j2 @@ -16,7 +16,7 @@ if [ "$1" = "info" ] if [ -z "$2" ]; then printf "run $0 with list and use the backup-tag to request information\n"; exit 1; fi {% for b in borgbackup_servers %} REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} - /usr/local/bin/borg info $REPOSITORY::$2 {{ b.options }} + /usr/local/bin/borg info {{ b.options }} $REPOSITORY::$2 {% endfor %} exit 0 fi @@ -28,7 +28,7 @@ if [ "$1" = "mount" ] if [ -z "$4" ]; then printf "Select the path to mount the backup on\n"; exit 1; fi {% for b in borgbackup_servers %} REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} - /usr/local/bin/borg mount $REPOSITORY::$3 $4 {{ b.options }} + /usr/local/bin/borg mount {{ b.options }} $REPOSITORY::$3 $4 if [ "$?" = "0" ]; then printf "Backup mounted on $4, do not forget to unmount!\n"; fi exit 0 {% endfor %} @@ -39,7 +39,7 @@ if [ "$1" = "list" ] {% for b in borgbackup_servers %} REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} printf "Archives on {{ b.fqdn }} :\n" - /usr/local/bin/borg list -v $REPOSITORY {{ b.options }} + /usr/local/bin/borg list {{ b.options }} -v $REPOSITORY {% endfor %} exit 0 fi @@ -48,7 +48,7 @@ if [ "$1" = "init" ] then {% for b in borgbackup_servers %} REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} - /usr/local/bin/borg init --encryption=keyfile $REPOSITORY {{ b.options }} + /usr/local/bin/borg init --encryption={{ borgbackup_encryption_mode }} {{ b.options }} $REPOSITORY {% endfor %} exit 0 fi @@ -66,13 +66,13 @@ if [ "$1" = "backup" ] printf "Backing up to {{ b.fqdn }} :\n" REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} - /usr/local/bin/borg create -p --compression zlib,6 --stats $REPOSITORY::$date {{ b.options }} {% for dir in borgbackup_include %}{{ dir }} {% endfor %}{% if automysql.stat.isdir is defined and automysql.stat.isdir == True %}/var/lib/automysqlbackup{% endif %} {% for dir in borgbackup_exclude %} --exclude '{{ dir }}'{% endfor %} + /usr/local/bin/borg create --progress --compression zlib,6 --stats {{ b.options }} $REPOSITORY::$date {% for dir in borgbackup_include %}{{ dir }} {% endfor %}{% if automysql.stat.isdir is defined and automysql.stat.isdir == True %}/var/lib/automysqlbackup{% endif %} {% for dir in borgbackup_exclude %} --exclude '{{ dir }}'{% endfor %} if [ "$?" -eq "0" ]; then printf "Backup succeeded on $date to {{ b.fqdn }}\n" >> /var/log/borg-backup.log; fi {% if not borgbackup_appendonly %} # prune old backups - /usr/local/bin/borg prune -v $REPOSITORY {{ b.options }} -H {{ borgbackup_retention.hourly }} -d {{ borgbackup_retention.daily }} -w {{ borgbackup_retention.weekly }} -m {{ borgbackup_retention.monthly }} -y {{ borgbackup_retention.yearly }} + /usr/local/bin/borg prune {{ b.options }} -v $REPOSITORY -H {{ borgbackup_retention.hourly }} -d {{ borgbackup_retention.daily }} -w {{ borgbackup_retention.weekly }} -m {{ borgbackup_retention.monthly }} -y {{ borgbackup_retention.yearly }} {% endif %} {% endfor %} From 09103099d7cedd15cafc604a88853cd0ae42aa6d Mon Sep 17 00:00:00 2001 From: Luc Stroobant Date: Sun, 28 Jan 2018 20:33:29 +0100 Subject: [PATCH 10/15] add become for init --- tasks/borg-client.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tasks/borg-client.yml b/tasks/borg-client.yml index 1f297a7..d67df3b 100644 --- a/tasks/borg-client.yml +++ b/tasks/borg-client.yml @@ -95,6 +95,7 @@ - name: client | create backup-directory on backup server shell: /usr/local/bin/borg-backup init + become_user: "{{ borgbackup_client_user }}" register: backup_init changed_when: "'Remember your passphrase' in backup_init.stderr" From 9558d3a0acd38ca19dc95a6cb4fa578caff88b44 Mon Sep 17 00:00:00 2001 From: Luc Stroobant Date: Mon, 29 Jan 2018 20:34:41 +0100 Subject: [PATCH 11/15] configurable compression --- defaults/main.yml | 2 ++ templates/borg-backup.sh.j2 | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/defaults/main.yml b/defaults/main.yml index 49144a6..0792d88 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -8,7 +8,9 @@ borgbackup_version: "1.1.4" borgbackup_checksum: "sha256:4ecf507f21f0db7c437b2ef34566273d7ba5a7d05e921c6f0e3406c3f96933a7" borgbackup_download_url: "https://github.com/borgbackup/borg/releases/download/{{ borgbackup_version }}/borg-linux64" +borgbackup_compression: "auto,zlib,6" borgbackup_encryption_mode: keyfile + borgbackup_pre_commands: - '[[ ! -f "/usr/sbin/automysqlbackup" ]] || /usr/sbin/automysqlbackup' diff --git a/templates/borg-backup.sh.j2 b/templates/borg-backup.sh.j2 index cc0884f..9c631e1 100644 --- a/templates/borg-backup.sh.j2 +++ b/templates/borg-backup.sh.j2 @@ -66,7 +66,7 @@ if [ "$1" = "backup" ] printf "Backing up to {{ b.fqdn }} :\n" REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} - /usr/local/bin/borg create --progress --compression zlib,6 --stats {{ b.options }} $REPOSITORY::$date {% for dir in borgbackup_include %}{{ dir }} {% endfor %}{% if automysql.stat.isdir is defined and automysql.stat.isdir == True %}/var/lib/automysqlbackup{% endif %} {% for dir in borgbackup_exclude %} --exclude '{{ dir }}'{% endfor %} + /usr/local/bin/borg create --progress --compression {{ borgbackup_compression }} --stats {{ b.options }} $REPOSITORY::$date {% for dir in borgbackup_include %}{{ dir }} {% endfor %}{% if automysql.stat.isdir is defined and automysql.stat.isdir == True %}/var/lib/automysqlbackup{% endif %} {% for dir in borgbackup_exclude %} --exclude '{{ dir }}'{% endfor %} if [ "$?" -eq "0" ]; then printf "Backup succeeded on $date to {{ b.fqdn }}\n" >> /var/log/borg-backup.log; fi From 6dfadc22ee2a96f7bae2e8bef1ccb45ce5ceab84 Mon Sep 17 00:00:00 2001 From: Luc Stroobant Date: Tue, 30 Jan 2018 19:42:56 +0100 Subject: [PATCH 12/15] update readme --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ec9f9d1..133bdb2 100644 --- a/README.md +++ b/README.md @@ -54,13 +54,14 @@ borgbackup_passphrase: Ahl9EiNohr5koosh1Wohs3Shoo3ooZ6p Per default the role creates a cronjob in /etc/cron.d/borg-backup running as root every day on a random hour between 0 and 5am on a random minute. Override the defaults if necessary: ``` -borgbackup_cron_user: root +borgbackup_client_user: root borgbackup_cron_day: "*" borgbackup_cron_minute: "{{ 59|random }}" borgbackup_cron_hour: "{{ 5|random }}" ``` +Override borgbackup\_client\_user where required, for example if you have a laptop with an encrypted homedir you'll have to run the backup as the user of that homedir. -Set borgbackup\_appendonly: True in host or group vars if you want append-only repositories. In that case it's possible to define a hostname in borgbackup\_management\_station where a borg prune script will be configured. +Set borgbackup\_appendonly: True in host or group vars if you want append-only repositories. In that case it's possible to define a hostname in borgbackup\_management\_station where a borg prune script will be configured. Only the management station will have permission to prune old backups for (all) clients. *Make sure to check the configured defaults for this role, which contains the list of default locations being backed up in backup_include.* Override this in your inventory where required. From 0d50034758d1543ad138b1a97f3a9a05ca793d65 Mon Sep 17 00:00:00 2001 From: Luc Stroobant Date: Sun, 4 Feb 2018 20:00:49 +0100 Subject: [PATCH 13/15] Ansible 2.4 --- README.md | 4 +++- tasks/main.yml | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 133bdb2..af20300 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ The role supports both self hosted and offsite backup-storage such as rsync.net It's possible to configure append-only repositories to secure the backups against deletion from the client. +Ansible 2.4 is required to run this role. + ## Required variables Define a group borgbackup\_servers in your inventory with one or multiple hosts. The group borgbackup\_management is only necessary if you want to enable append-only mode and prune the backups from a secured hosts. ``` @@ -74,4 +76,4 @@ ansible-playbook -i inventory/test backup.yml -l client1.fiaas.co ``` ## Further reading -https://borgbackup.readthedocs.io/en/stable/ +* https://borgbackup.readthedocs.io/en/stable/ diff --git a/tasks/main.yml b/tasks/main.yml index 1bcdef8..b771ee4 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,18 +1,18 @@ --- -- include: install.yml +- include_tasks: install.yml when: > borgbackup_required == True or inventory_hostname in groups.borgbackup_servers -- include: borg-server.yml +- include_tasks: borg-server.yml when: inventory_hostname in groups.borgbackup_servers -- include: borg-client.yml +- include_tasks: borg-client.yml when: > borgbackup_required == True and inventory_hostname not in groups.borgbackup_servers -- include: management.yml +- include_tasks: management.yml when: > inventory_hostname in groups.borgbackup_management and inventory_hostname not in groups.borgbackup_servers From 548dfd9aee0ff639b02a2641e98a4d265e65ab59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Armando=20Lu=CC=88scher?= Date: Fri, 13 Apr 2018 23:11:10 +0200 Subject: [PATCH 14/15] Allow undefined borgbackup_servers and borgbackup_management groups. --- tasks/main.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tasks/main.yml b/tasks/main.yml index b771ee4..a394037 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,18 +1,23 @@ --- +# Due to inverse logic behaviour when searching for an item in an undefined list. +- set_fact: + borgbackup_servers_group: "{{ groups.borgbackup_servers | default([]) }} " + borgbackup_management_group: "{{ groups.borgbackup_management | default([]) }}" + - include_tasks: install.yml when: > borgbackup_required == True or - inventory_hostname in groups.borgbackup_servers + inventory_hostname in borgbackup_servers_group - include_tasks: borg-server.yml - when: inventory_hostname in groups.borgbackup_servers + when: inventory_hostname in borgbackup_servers_group - include_tasks: borg-client.yml when: > borgbackup_required == True and - inventory_hostname not in groups.borgbackup_servers + inventory_hostname not in borgbackup_servers_group - include_tasks: management.yml when: > - inventory_hostname in groups.borgbackup_management and - inventory_hostname not in groups.borgbackup_servers + inventory_hostname in borgbackup_management_group and + inventory_hostname not in borgbackup_servers_group From 6ae4bbf49ede3d3690f08bba40f4d0039edff97d Mon Sep 17 00:00:00 2001 From: Luc Stroobant Date: Sat, 19 May 2018 16:13:19 +0200 Subject: [PATCH 15/15] improve and clarify append-only --- README.md | 11 +++++++---- defaults/main.yml | 1 + tasks/borg-server.yml | 1 + templates/borg-backup.sh.j2 | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index af20300..da7f5f4 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ The role supports both self hosted and offsite backup-storage such as rsync.net It's possible to configure append-only repositories to secure the backups against deletion from the client. -Ansible 2.4 is required to run this role. +Ansible 2.4 or higher is required to run this role. ## Required variables Define a group borgbackup\_servers in your inventory with one or multiple hosts. The group borgbackup\_management is only necessary if you want to enable append-only mode and prune the backups from a secured hosts. @@ -63,9 +63,11 @@ borgbackup_cron_hour: "{{ 5|random }}" ``` Override borgbackup\_client\_user where required, for example if you have a laptop with an encrypted homedir you'll have to run the backup as the user of that homedir. -Set borgbackup\_appendonly: True in host or group vars if you want append-only repositories. In that case it's possible to define a hostname in borgbackup\_management\_station where a borg prune script will be configured. Only the management station will have permission to prune old backups for (all) clients. +Set borgbackup\_appendonly: True in host or group vars if you want append-only repositories. In that case it's possible to define a hostname in borgbackup\_management\_station where a borg prune script will be configured. Only the management station will have permission to prune old backups for (all) clients. This will generate serve with --append-only ssh key options. +If you set borgbackup\_appendonly\_repoconfig to True, this will also disable the possibility to remove backups from the management station. (Or at least: it's not possible to remove them till you reconfigure the repository and this is currently not supported in the prune script) +Be aware of the limitations of append-only mode: [pruned backups appear to be removed, but are only removed in the transaction log till something writes in normal mode to the repository](https://github.com/borgbackup/borg/issues/3504)) -*Make sure to check the configured defaults for this role, which contains the list of default locations being backed up in backup_include.* Override this in your inventory where required. +*Make sure to check the configured defaults for this role, which contains the list of default locations being backed up in backup\_include.* Override this in your inventory where required. ## Usage @@ -76,4 +78,5 @@ ansible-playbook -i inventory/test backup.yml -l client1.fiaas.co ``` ## Further reading -* https://borgbackup.readthedocs.io/en/stable/ +* [Borg documentation](https://borgbackup.readthedocs.io/en/stable/) +* [Append only mode information](http://borgbackup.readthedocs.io/en/stable/usage/notes.html#append-only-mode) diff --git a/defaults/main.yml b/defaults/main.yml index 0792d88..1c637e2 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -37,6 +37,7 @@ borgbackup_cron_hour: "{{ 5|random }}" borgbackup_cron_minute: "{{ 59|random }}" borgbackup_appendonly: False +borgbackup_appendonly_repoconfig: False borgbackup_management_station: '' borgbackup_management_user: '' borgbackup_management_ssh_pubkey: '' diff --git a/tasks/borg-server.yml b/tasks/borg-server.yml index de53c4f..3a18d12 100644 --- a/tasks/borg-server.yml +++ b/tasks/borg-server.yml @@ -18,3 +18,4 @@ mode: "0770" delegate_to: "{{ item.fqdn }}" with_items: "{{ borgbackup_servers }}" + when: item.type == 'normal' diff --git a/templates/borg-backup.sh.j2 b/templates/borg-backup.sh.j2 index 9c631e1..1313d85 100644 --- a/templates/borg-backup.sh.j2 +++ b/templates/borg-backup.sh.j2 @@ -48,7 +48,7 @@ if [ "$1" = "init" ] then {% for b in borgbackup_servers %} REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} - /usr/local/bin/borg init --encryption={{ borgbackup_encryption_mode }} {{ b.options }} $REPOSITORY + /usr/local/bin/borg init --encryption={{ borgbackup_encryption_mode }}{% if borgbackup_appendonly_repoconfig %} --append-only{% endif %} {{ b.options }} $REPOSITORY {% endfor %} exit 0 fi