From eba87a88c723e0049339513bddf51b2ca7d7f74f Mon Sep 17 00:00:00 2001 From: Dieter Verhelst Date: Tue, 3 Jul 2018 06:56:46 +0200 Subject: [PATCH 1/6] Wrong variable name in prune.sh script --- templates/prune.sh.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/prune.sh.j2 b/templates/prune.sh.j2 index f01bb2a..5513694 100644 --- a/templates/prune.sh.j2 +++ b/templates/prune.sh.j2 @@ -8,7 +8,7 @@ {% if hostvars[h].borgbackup_required | default(True) %} # Host: {{ h }} {% for b in hostvars[h].borgbackup_servers %} - {% if hostvars[h].borgbackup_managementstation is defined and inventory_hostname == hostvars[h].borgbackup_managementstation %} + {% if hostvars[h].borgbackup_management_station is defined and inventory_hostname == hostvars[h].borgbackup_management_station %} export BORG_PASSPHRASE={{ hostvars[h].borgbackup_passphrase }} REPOSITORY={{ b.user }}@{{ b.fqdn }}:{{ b.home }}{{ b.pool }}/{{ h }} /usr/local/bin/borg prune -v $REPOSITORY {{ b.options }} -H {{ hostvars[h].borgbackup_retention.hourly }} -d {{ hostvars[h].borgbackup_retention.daily }} -w {{ hostvars[h].borgbackup_retention.weekly }} -m {{ hostvars[h].borgbackup_retention.monthly }} -y {{ hostvars[h].borgbackup_retention.yearly }} From e5c82933d9614ce8f3ad4fd7ac0fdae2ce6eafa1 Mon Sep 17 00:00:00 2001 From: Dieter Verhelst Date: Tue, 3 Jul 2018 07:17:04 +0200 Subject: [PATCH 2/6] Adding correct urls for hetzner-type destination to prune.sh --- templates/prune.sh.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/prune.sh.j2 b/templates/prune.sh.j2 index 5513694..d3e41eb 100644 --- a/templates/prune.sh.j2 +++ b/templates/prune.sh.j2 @@ -10,7 +10,7 @@ {% for b in hostvars[h].borgbackup_servers %} {% if hostvars[h].borgbackup_management_station is defined and inventory_hostname == hostvars[h].borgbackup_management_station %} export BORG_PASSPHRASE={{ hostvars[h].borgbackup_passphrase }} - REPOSITORY={{ b.user }}@{{ b.fqdn }}:{{ b.home }}{{ b.pool }}/{{ h }} + REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ h }} /usr/local/bin/borg prune -v $REPOSITORY {{ b.options }} -H {{ hostvars[h].borgbackup_retention.hourly }} -d {{ hostvars[h].borgbackup_retention.daily }} -w {{ hostvars[h].borgbackup_retention.weekly }} -m {{ hostvars[h].borgbackup_retention.monthly }} -y {{ hostvars[h].borgbackup_retention.yearly }} {% endif %} From e2039b742d2b347a817d39ea9217b6da79aa8721 Mon Sep 17 00:00:00 2001 From: Dieter Verhelst Date: Tue, 3 Jul 2018 07:27:50 +0200 Subject: [PATCH 3/6] Cleaner output of prune.sh generated template --- templates/prune.sh.j2 | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/templates/prune.sh.j2 b/templates/prune.sh.j2 index d3e41eb..25755e7 100644 --- a/templates/prune.sh.j2 +++ b/templates/prune.sh.j2 @@ -1,3 +1,4 @@ +#jinja2:lstrip_blocks: True #!/bin/bash # This script is intended to run on a trusted management station to purge borg repo's in @@ -5,15 +6,18 @@ # Don't put it on the backup server, it contains all borg secrets! {% for h in groups['all'] %} - {% if hostvars[h].borgbackup_required | default(True) %} - # Host: {{ h }} + {% if hostvars[h].borgbackup_required | default(True) -%} + # Host: {{ h }} + + export BORG_PASSPHRASE={{ hostvars[h].borgbackup_passphrase }} + + {% if hostvars[h].borgbackup_management_station is defined and inventory_hostname == hostvars[h].borgbackup_management_station %} {% for b in hostvars[h].borgbackup_servers %} - {% if hostvars[h].borgbackup_management_station is defined and inventory_hostname == hostvars[h].borgbackup_management_station %} - export BORG_PASSPHRASE={{ hostvars[h].borgbackup_passphrase }} + # {{ b.fqdn }} REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ h }} /usr/local/bin/borg prune -v $REPOSITORY {{ b.options }} -H {{ hostvars[h].borgbackup_retention.hourly }} -d {{ hostvars[h].borgbackup_retention.daily }} -w {{ hostvars[h].borgbackup_retention.weekly }} -m {{ hostvars[h].borgbackup_retention.monthly }} -y {{ hostvars[h].borgbackup_retention.yearly }} - {% endif %} - {% endfor %} + {% endfor %} + {% endif %} {% endif %} {% endfor %} From 940048826766655397f42d4cd799332fb3c40ed5 Mon Sep 17 00:00:00 2001 From: Dieter Verhelst Date: Thu, 16 Aug 2018 09:01:44 +0200 Subject: [PATCH 4/6] Splitting REPOSITORY into multiple line jinja if statement for clarity --- templates/borg-backup.sh.j2 | 30 +++++++++++++++++++++++++----- templates/prune.sh.j2 | 6 +++++- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/templates/borg-backup.sh.j2 b/templates/borg-backup.sh.j2 index 1313d85..0f712bf 100644 --- a/templates/borg-backup.sh.j2 +++ b/templates/borg-backup.sh.j2 @@ -15,7 +15,11 @@ if [ "$1" = "info" ] then if [ -z "$2" ]; then printf "run $0 with list and use the backup-tag to request information\n"; exit 1; fi {% for b in borgbackup_servers %} - REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} +{% if b.type == 'hetzner' %} + REPOSITORY=ssh://{{ b.user }}@{{ b.fqdn }}:23/./{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} +{% else %} + REPOSITORY={{ b.user }}@{{ b.fqdn }}:{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} +{% endif %} /usr/local/bin/borg info {{ b.options }} $REPOSITORY::$2 {% endfor %} exit 0 @@ -27,7 +31,11 @@ if [ "$1" = "mount" ] if [ -z "$3" ]; then printf "Select the backup to mount\n"; exit 1; fi if [ -z "$4" ]; then printf "Select the path to mount the backup on\n"; exit 1; fi {% for b in borgbackup_servers %} - REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} +{% if b.type == 'hetzner' %} + REPOSITORY=ssh://{{ b.user }}@{{ b.fqdn }}:23/./{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} +{% else %} + REPOSITORY={{ b.user }}@{{ b.fqdn }}:{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} +{% endif %} /usr/local/bin/borg mount {{ b.options }} $REPOSITORY::$3 $4 if [ "$?" = "0" ]; then printf "Backup mounted on $4, do not forget to unmount!\n"; fi exit 0 @@ -37,7 +45,11 @@ fi if [ "$1" = "list" ] then {% for b in borgbackup_servers %} - REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} +{% if b.type == 'hetzner' %} + REPOSITORY=ssh://{{ b.user }}@{{ b.fqdn }}:23/./{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} +{% else %} + REPOSITORY={{ b.user }}@{{ b.fqdn }}:{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} +{% endif %} printf "Archives on {{ b.fqdn }} :\n" /usr/local/bin/borg list {{ b.options }} -v $REPOSITORY {% endfor %} @@ -47,7 +59,11 @@ fi if [ "$1" = "init" ] then {% for b in borgbackup_servers %} - REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} +{% if b.type == 'hetzner' %} + REPOSITORY=ssh://{{ b.user }}@{{ b.fqdn }}:23/./{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} +{% else %} + REPOSITORY={{ b.user }}@{{ b.fqdn }}:{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} +{% endif %} /usr/local/bin/borg init --encryption={{ borgbackup_encryption_mode }}{% if borgbackup_appendonly_repoconfig %} --append-only{% endif %} {{ b.options }} $REPOSITORY {% endfor %} exit 0 @@ -64,7 +80,11 @@ if [ "$1" = "backup" ] {% for b in borgbackup_servers %} printf "Backing up to {{ b.fqdn }} :\n" - REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} +{% if b.type == 'hetzner' %} + REPOSITORY=ssh://{{ b.user }}@{{ b.fqdn }}:23/./{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} +{% else %} + REPOSITORY={{ b.user }}@{{ b.fqdn }}:{{ b.home }}{{ b.pool }}/{{ inventory_hostname }} +{% endif %} /usr/local/bin/borg create --progress --compression {{ borgbackup_compression }} --stats {{ b.options }} $REPOSITORY::$date {% for dir in borgbackup_include %}{{ dir }} {% endfor %}{% if automysql.stat.isdir is defined and automysql.stat.isdir == True %}/var/lib/automysqlbackup{% endif %} {% for dir in borgbackup_exclude %} --exclude '{{ dir }}'{% endfor %} diff --git a/templates/prune.sh.j2 b/templates/prune.sh.j2 index 25755e7..64d0a2d 100644 --- a/templates/prune.sh.j2 +++ b/templates/prune.sh.j2 @@ -14,7 +14,11 @@ {% if hostvars[h].borgbackup_management_station is defined and inventory_hostname == hostvars[h].borgbackup_management_station %} {% for b in hostvars[h].borgbackup_servers %} # {{ b.fqdn }} - REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ h }} +{% if b.type == 'hetzner' %} + REPOSITORY=ssh://{{ b.user }}@{{ b.fqdn }}:23/./{{ b.home }}{{ b.pool }}/{{ h }} +{% else %} + REPOSITORY={{ b.user }}@{{ b.fqdn }}:{{ b.home }}{{ b.pool }}/{{ h }} +{% endif %} /usr/local/bin/borg prune -v $REPOSITORY {{ b.options }} -H {{ hostvars[h].borgbackup_retention.hourly }} -d {{ hostvars[h].borgbackup_retention.daily }} -w {{ hostvars[h].borgbackup_retention.weekly }} -m {{ hostvars[h].borgbackup_retention.monthly }} -y {{ hostvars[h].borgbackup_retention.yearly }} {% endfor %} From d40496225b3ab9e1eb9f5820e0bede6431a1e88e Mon Sep 17 00:00:00 2001 From: Dieter Verhelst Date: Tue, 28 Aug 2018 12:58:28 +0200 Subject: [PATCH 5/6] As per Ansible 2.3 it's possible to use seed from the inventory hostname. This turns 'random' numbers idempotent --- defaults/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 1c637e2..ac3e6ba 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,5 +1,5 @@ --- -borgbackup_required: True +borgbackup_required: true borgbackup_client_user: root borgbackup_ssh_key: "~{{ borgbackup_client_user }}/.ssh/id_borg_rsa" @@ -33,11 +33,11 @@ borgbackup_retention: yearly: 1 borgbackup_cron_day: "*" -borgbackup_cron_hour: "{{ 5|random }}" -borgbackup_cron_minute: "{{ 59|random }}" +borgbackup_cron_hour: "{{ 5|random(seed=inventory_hostname) }}" +borgbackup_cron_minute: "{{ 59|random(seed=inventory_hostname) }}" -borgbackup_appendonly: False -borgbackup_appendonly_repoconfig: False +borgbackup_appendonly: false +borgbackup_appendonly_repoconfig: false borgbackup_management_station: '' borgbackup_management_user: '' borgbackup_management_ssh_pubkey: '' From 033c923a9e6ed852342818a4c4e2d7e09559ee48 Mon Sep 17 00:00:00 2001 From: Dieter Verhelst Date: Tue, 28 Aug 2018 13:01:57 +0200 Subject: [PATCH 6/6] Undo branch-polution --- defaults/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index ac3e6ba..5a174f7 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,5 +1,5 @@ --- -borgbackup_required: true +borgbackup_required: True borgbackup_client_user: root borgbackup_ssh_key: "~{{ borgbackup_client_user }}/.ssh/id_borg_rsa" @@ -36,8 +36,8 @@ borgbackup_cron_day: "*" borgbackup_cron_hour: "{{ 5|random(seed=inventory_hostname) }}" borgbackup_cron_minute: "{{ 59|random(seed=inventory_hostname) }}" -borgbackup_appendonly: false -borgbackup_appendonly_repoconfig: false +borgbackup_appendonly: False +borgbackup_appendonly_repoconfig: False borgbackup_management_station: '' borgbackup_management_user: '' borgbackup_management_ssh_pubkey: ''