Merge pull request #1 from FiaasCo/master

Fetch from master

README.md

@@ -1,9 +1,11 @@
 # Borg backup role
 This role installs Borg backup on borgbackup\_servers and clients. The role contains a wrapper-script 'borg-backup' to ease the usage on the client. Supported options include borg-backup info | init | list | backup | mount. Automysqlbackup will run as pre-backup command if it's installed.
-The role supports both self hosted and rsync.net as Borg server.
+The role supports both self hosted and offsite backup-storage such as rsync.net and hetzner storage box as Borg server.
 
 It's possible to configure append-only repositories to secure the backups against deletion from the client.
 
+Ansible 2.4 or higher is required to run this role.
+
 ## Required variables
 Define a group borgbackup\_servers in your inventory with one or multiple hosts. The group borgbackup\_management is only necessary if you want to enable append-only mode and prune the backups from a secured hosts.
 ```
@@ -29,6 +31,13 @@ borgbackup_servers:
     home: ""
     pool: repos
     options: "--remote-path=borg1"
+  - fqdn: username.your-storagebox.de
+    user: username
+    type: hetzner
+    home: ""
+    pool: repos
+    options: ""
+
 
 borgbackup_retention:
   hourly: 12
@@ -47,15 +56,18 @@ borgbackup_passphrase: Ahl9EiNohr5koosh1Wohs3Shoo3ooZ6p
 
 Per default the role creates a cronjob in /etc/cron.d/borg-backup running as root every day on a random hour between 0 and 5am on a random minute. Override the defaults if necessary:
 ```
-borgbackup_cron_user: root
+borgbackup_client_user: root
 borgbackup_cron_day: "*"
 borgbackup_cron_minute: "{{ 59|random }}"
 borgbackup_cron_hour: "{{ 5|random }}"
 ```
+Override borgbackup\_client\_user where required, for example if you have a laptop with an encrypted homedir you'll have to run the backup as the user of that homedir.
 
-Set borgbackup\_appendonly: True in host or group vars if you want append-only repositories. In that case it's possible to define a hostname in borgbackup\_management\_station where a borg prune script will be configured.
+Set borgbackup\_appendonly: True in host or group vars if you want append-only repositories. In that case it's possible to define a hostname in borgbackup\_management\_station where a borg prune script will be configured. Only the management station will have permission to prune old backups for (all) clients. This will generate serve with --append-only ssh key options.
+If you set borgbackup\_appendonly\_repoconfig to True, this will also disable the possibility to remove backups from the management station. (Or at least: it's not possible to remove them till you reconfigure the repository and this is currently not supported in the prune script)
+Be aware of the limitations of append-only mode: [pruned backups appear to be removed, but are only removed in the transaction log till something writes in normal mode to the repository](https://github.com/borgbackup/borg/issues/3504))
 
-*Make sure to check the configured defaults for this role, which contains the list of default locations being backed up in backup_include.* Override this in your inventory where required.
+*Make sure to check the configured defaults for this role, which contains the list of default locations being backed up in backup\_include.* Override this in your inventory where required.
 
 ## Usage
 
@@ -66,4 +78,5 @@ ansible-playbook -i inventory/test backup.yml -l client1.fiaas.co
 ```
 
 ## Further reading
-https://borgbackup.readthedocs.io/en/stable/
+* [Borg documentation](https://borgbackup.readthedocs.io/en/stable/)
+* [Append only mode information](http://borgbackup.readthedocs.io/en/stable/usage/notes.html#append-only-mode)

defaults/main.yml

@@ -1,15 +1,21 @@
 ---
 borgbackup_required: True
 
-borgbackup_ssh_key: "~/.ssh/id_borg_rsa"
+borgbackup_client_user: root
+borgbackup_ssh_key: "~{{ borgbackup_client_user }}/.ssh/id_borg_rsa"
 
-borgbackup_version: "1.0.11"
+borgbackup_version: "1.1.4"
-borgbackup_checksum: "sha256:fbdc0e0d6d05a0935551f2f408f370236a76b7a30d3bb90a31c3628fe3611359"
+borgbackup_checksum: "sha256:4ecf507f21f0db7c437b2ef34566273d7ba5a7d05e921c6f0e3406c3f96933a7"
 borgbackup_download_url: "https://github.com/borgbackup/borg/releases/download/{{ borgbackup_version }}/borg-linux64"
 
+borgbackup_compression: "auto,zlib,6"
+borgbackup_encryption_mode: keyfile
+
 borgbackup_pre_commands:
   - '[[ ! -f "/usr/sbin/automysqlbackup" ]] || /usr/sbin/automysqlbackup'
 
+borgbackup_post_commands: []
+
 borgbackup_include:
   - "/etc"
   - "/home"
@@ -17,6 +23,8 @@ borgbackup_include:
   - "/var/www"
   - "/var/log"
 
+borgbackup_exclude: []
+
 borgbackup_retention:
   hourly: 12
   daily: 7
@@ -24,12 +32,16 @@ borgbackup_retention:
   monthly: 6
   yearly: 1
 
-borgbackup_cron_user: root
 borgbackup_cron_day: "*"
 borgbackup_cron_hour: "{{ 5|random }}"
 borgbackup_cron_minute: "{{ 59|random }}"
 
 borgbackup_appendonly: False
+borgbackup_appendonly_repoconfig: False
 borgbackup_management_station: ''
 borgbackup_management_user: ''
 borgbackup_management_ssh_pubkey: ''
+
+borgbackup_owner: root
+borgbackup_group: root
+borgbackup_shell: "/bin/bash"

meta/main.yml

@@ -1,7 +1,7 @@
 ---
 galaxy_info:
   author: Luc Stroobant and Dieter Verhelst
-  description: Borg backup server and client role
+  description: Install Borg backup server and client (with rsync.net server support)
   license: MIT
   min_ansible_version: 1.9
   platforms:

tasks/borg-client.yml

@@ -1,6 +1,11 @@
 ---
 - name: client | generate ssh key for this machine
-  shell: if [ -f {{ borgbackup_ssh_key }} ]; then rm -f {{ borgbackup_ssh_key }}; fi && ssh-keygen -q -t rsa -b 4096 -f {{ borgbackup_ssh_key }} -N "" creates="{{ borgbackup_ssh_key }}.pub"
+  user:
+    name: "{{ borgbackup_client_user }}"
+    generate_ssh_key: yes
+    ssh_key_bits: 2048
+    ssh_key_file: "{{ borgbackup_ssh_key }}"
+    ssh_key_type: rsa
 
 - name: client | fetch ssh-key
   shell: "cat {{ borgbackup_ssh_key }}.pub"
@@ -9,14 +14,14 @@
 
 - name: client | write passphrase
   lineinfile:
-    dest: "/root/.borg.passphrase"
+    dest: "~{{ borgbackup_client_user }}/.borg.passphrase"
     state: "present"
     line: 'export BORG_PASSPHRASE="{{ borgbackup_passphrase }}"'
     create: "yes"
 
 - name: client | disable strict key checking for backup servers
   blockinfile:
-    dest: /root/.ssh/config
+    dest: "~{{ borgbackup_client_user }}/.ssh/config"
     create: yes
     marker: "### {mark} ANSIBLE MANAGED BLOCK {{ item.fqdn }} ###"
     content: |
@@ -37,38 +42,38 @@
   when: item.type == 'normal'
   with_items: "{{ borgbackup_servers }}"
 
-# rsync.net has no python, so we can only use raw to manage ssh keys - workaround with local tmp file
+# rsync.net and hetzner have no python, so we can only use raw to manage ssh keys - workaround with local tmp file
-- name: client | get rsync.net authorized_keys file
+- name: client | get authorized_keys file
-  raw: scp {{ item.user }}@{{ item.fqdn }}:.ssh/authorized_keys /tmp/rsync.net-{{ item.fqdn }}-authkeys
+  raw: scp {{ item.user }}@{{ item.fqdn }}:.ssh/authorized_keys /tmp/authkeys-{{ item.type }}-{{ item.fqdn }}-authkeys
   delegate_to: localhost
   become: no
-  when: item.type == 'rsync.net'
+  when: item.type in ['rsync.net','hetzner']
   with_items: "{{ borgbackup_servers }}"
   changed_when: false
 
-- name: client | modify local rsync.net authorized_keys
+- name: client | modify local rsync.net/hetzner authorized_keys
   authorized_key:
     user: "{{ ansible_user_id }}"
     key: "{{ sshkey.stdout }}"
     key_options: 'command="cd {{ item.pool }}/{{ inventory_hostname }};/usr/local/bin/borg1 serve {% if borgbackup_appendonly %}--append-only {% endif %} --restrict-to-path {{ item.pool }}/{{ inventory_hostname }}",no-port-forwarding,no-X11-forwarding,no-pty,no-agent-forwarding,no-user-rc'
-    path: "/tmp/rsync.net-{{ item.fqdn }}-authkeys"
+    path: "/tmp/authkeys-{{ item.type }}-{{ item.fqdn }}-authkeys"
     manage_dir: no
   delegate_to: localhost
   become: no
-  when: item.type == 'rsync.net'
+  when: item.type in ['rsync.net','hetzner']
   with_items: "{{ borgbackup_servers }}"
   register: authkeys
 
-- name: client | upload local authorized_keys to rsync.net
+- name: client | upload local authorized_keys to rsync.net / hetzner
-  raw: scp /tmp/rsync.net-{{ item.fqdn }}-authkeys {{ item.user }}@{{ item.fqdn }}:.ssh/authorized_keys
+  raw: scp /tmp/authkeys-{{ item.type }}-{{ item.fqdn }}-authkeys {{ item.user }}@{{ item.fqdn }}:.ssh/authorized_keys
   delegate_to: localhost
   become: no
-  when: item.type == 'rsync.net' and authkeys.changed
+  when: item.type in ['rsync.net','hetzner'] and authkeys.changed
   with_items: "{{ borgbackup_servers }}"
 
 - name: client | remove tmp authorized_keys files
   file:
-    path: /tmp/rsync.net-{{ item.fqdn }}-authkeys
+    path: /tmp/authkeys-{{ item.type }}-{{ item.fqdn }}-authkeys
     state: absent
   delegate_to: localhost
   become: no
@@ -84,19 +89,20 @@
   template:
     src: "borg-backup.sh.j2"
     dest: "/usr/local/bin/borg-backup"
-    owner: "root"
+    owner: "{{ borgbackup_owner }}"
-    group: "root"
+    group: "{{ borgbackup_group }}"
     mode: "0744"
 
 - name: client | create backup-directory on backup server
   shell: /usr/local/bin/borg-backup init
+  become_user: "{{ borgbackup_client_user }}"
   register: backup_init
   changed_when: "'Remember your passphrase' in backup_init.stderr"
 
 - name: client | create backup cronjob
   cron:
     cron_file: "borg-backup"
-    user: "{{ borgbackup_cron_user }}"
+    user: "{{ borgbackup_client_user }}"
     name: "borg-backup"
     minute: "{{ borgbackup_cron_minute }}"
     hour: "{{ borgbackup_cron_hour }}"

tasks/borg-server.yml

@@ -18,3 +18,4 @@
     mode: "0770"
   delegate_to: "{{ item.fqdn }}"
   with_items: "{{ borgbackup_servers }}"
+  when: item.type == 'normal'

tasks/install.yml

@@ -3,8 +3,8 @@
   get_url:
     dest: "/usr/local/bin/borg"
     checksum: "{{ borgbackup_checksum }}"
-    owner: "root"
+    owner: "{{ borgbackup_owner }}"
-    group: "root"
+    group: "{{ borgbackup_group }}"
     mode: "0755"
     url: "{{ borgbackup_download_url }}"
   tags: borginstall

tasks/main.yml

@@ -1,18 +1,23 @@
 ---
-- include: install.yml
+# Due to inverse logic behaviour when searching for an item in an undefined list.
+- set_fact:
+    borgbackup_servers_group: "{{ groups.borgbackup_servers | default([]) }} "
+    borgbackup_management_group: "{{ groups.borgbackup_management | default([]) }}"
+
+- include_tasks: install.yml
   when: >
     borgbackup_required == True or
-    inventory_hostname in groups.borgbackup_servers
+    inventory_hostname in borgbackup_servers_group
 
-- include: borg-server.yml
+- include_tasks: borg-server.yml
-  when: inventory_hostname in groups.borgbackup_servers
+  when: inventory_hostname in borgbackup_servers_group
 
-- include: borg-client.yml
+- include_tasks: borg-client.yml
   when: >
     borgbackup_required == True and
-    inventory_hostname not in groups.borgbackup_servers
+    inventory_hostname not in borgbackup_servers_group
 
-- include: management.yml
+- include_tasks: management.yml
   when: >
-    inventory_hostname in groups.borgbackup_management and
+    inventory_hostname in borgbackup_management_group and
-    inventory_hostname not in groups.borgbackup_servers
+    inventory_hostname not in borgbackup_servers_group

templates/borg-backup.sh.j2

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!{{ borgbackup_shell }}
 
 if [ -z "$1" ] || [ ! -z "$2" ]
   then
@@ -7,7 +7,7 @@ fi
 
 # Sourcing the backup-passphrase
 
-. /root/.borg.passphrase
+. ~{{ borgbackup_client_user }}/.borg.passphrase
 
 # Small helper commands, like listing backups, will help us in the future :)
 
@@ -15,8 +15,8 @@ if [ "$1" = "info" ]
   then
     if [ -z "$2" ]; then printf "run $0 with list and use the backup-tag to request information\n"; exit 1; fi
 {% for b in borgbackup_servers %}
-    REPOSITORY={{ b.user }}@{{ b.fqdn }}:{{ b.home }}{{ b.pool }}/{{ inventory_hostname }}
+    REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }}
-    /usr/local/bin/borg info $REPOSITORY::$2 {{ b.options }}
+    /usr/local/bin/borg info {{ b.options }} $REPOSITORY::$2
 {% endfor %}
     exit 0
 fi
@@ -27,8 +27,8 @@ if [ "$1" = "mount" ]
     if [ -z "$3" ]; then printf "Select the backup to mount\n"; exit 1; fi
     if [ -z "$4" ]; then printf "Select the path to mount the backup on\n"; exit 1; fi
 {% for b in borgbackup_servers %}
-    REPOSITORY={{ b.user }}@{{ b.fqdn }}:{{ b.home }}{{ b.pool }}/{{ inventory_hostname }}
+    REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }}
-    /usr/local/bin/borg mount $REPOSITORY::$3 $4 {{ b.options }}
+    /usr/local/bin/borg mount {{ b.options }} $REPOSITORY::$3 $4
     if [ "$?" = "0" ]; then printf "Backup mounted on $4, do not forget to unmount!\n"; fi
     exit 0
 {% endfor %}
@@ -37,9 +37,9 @@ fi
 if [ "$1" = "list" ]
   then
 {% for b in borgbackup_servers %}
-    REPOSITORY={{ b.user }}@{{ b.fqdn }}:{{ b.home }}{{ b.pool }}/{{ inventory_hostname }}
+    REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }}
     printf "Archives on {{ b.fqdn }} :\n"
-    /usr/local/bin/borg list -v $REPOSITORY {{ b.options }}
+    /usr/local/bin/borg list {{ b.options }} -v $REPOSITORY
 {% endfor %}
     exit 0
 fi
@@ -47,8 +47,8 @@ fi
 if [ "$1" = "init" ]
   then
 {% for b in borgbackup_servers %}
-    REPOSITORY={{ b.user }}@{{ b.fqdn }}:{{ b.home }}{{ b.pool }}/{{ inventory_hostname }}
+    REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }}
-    borg init $REPOSITORY {{ b.options }}
+    /usr/local/bin/borg init --encryption={{ borgbackup_encryption_mode }}{% if borgbackup_appendonly_repoconfig %} --append-only{% endif %} {{ b.options }} $REPOSITORY
 {% endfor %}
     exit 0
 fi
@@ -64,16 +64,22 @@ if [ "$1" = "backup" ]
 
 {% for b in borgbackup_servers %}
     printf "Backing up to {{ b.fqdn }} :\n"
-    REPOSITORY={{ b.user }}@{{ b.fqdn }}:{{ b.home }}{{ b.pool }}/{{ inventory_hostname }}
+    REPOSITORY={% if b.type == 'hetzner' %}ssh://{% endif %}{{ b.user }}@{{ b.fqdn }}:{% if b.type == 'hetzner' %}23/./{% endif %}{{ b.home }}{{ b.pool }}/{{ inventory_hostname }}
     
-    /usr/local/bin/borg create --compression zlib,6 --stats $REPOSITORY::$date {{ b.options }} {% for dir in borgbackup_include %}{{ dir }} {% endfor %}{% if automysql.stat.isdir is defined and automysql.stat.isdir == True %}/var/lib/automysqlbackup{% endif %}
+    /usr/local/bin/borg create --progress --compression {{ borgbackup_compression }} --stats {{ b.options }} $REPOSITORY::$date {% for dir in borgbackup_include %}{{ dir }} {% endfor %}{% if automysql.stat.isdir is defined and automysql.stat.isdir == True %}/var/lib/automysqlbackup{% endif %} {% for dir in borgbackup_exclude %} --exclude '{{ dir }}'{% endfor %}
 
-    if [ "$?" -eq "0" ]; then printf "Backup succeeded on $date\n" >> /var/log/borg-backup.log; fi
+    if [ "$?" -eq "0" ]; then printf "Backup succeeded on $date to {{ b.fqdn }}\n" >> /var/log/borg-backup.log; fi
 
   {% if not borgbackup_appendonly %}
     # prune old backups
-    /usr/local/bin/borg prune -v $REPOSITORY {{ b.options }} -H {{ borgbackup_retention.hourly }} -d {{ borgbackup_retention.daily }} -w {{ borgbackup_retention.weekly }} -m {{ borgbackup_retention.monthly }} -y {{ borgbackup_retention.yearly }}
+    /usr/local/bin/borg prune {{ b.options }} -v $REPOSITORY -H {{ borgbackup_retention.hourly }} -d {{ borgbackup_retention.daily }} -w {{ borgbackup_retention.weekly }} -m {{ borgbackup_retention.monthly }} -y {{ borgbackup_retention.yearly }}
   {% endif %}
 {% endfor %}
+
+    # Running some commands post-backup
+{% for postcommand in borgbackup_post_commands %}
+    {{ postcommand }}
+{% endfor %}
+
 fi