Browse Source

[fix] Management server should have keys and should work

master
Alban 1 year ago
parent
commit
23609395dc
6 changed files with 130 additions and 85 deletions
  1. 28
    9
      defaults/main.yml
  2. 14
    50
      tasks/borg-client.yml
  3. 6
    0
      tasks/main.yml
  4. 23
    0
      tasks/management-keys.yml
  5. 33
    8
      templates/borg-backup.sh.j2
  6. 26
    18
      templates/prune.sh.j2

+ 28
- 9
defaults/main.yml View File

@@ -4,26 +4,40 @@ borgbackup_required: true
4 4
 borgbackup_client_user: root
5 5
 borgbackup_ssh_key: "~{{ borgbackup_client_user }}/.ssh/id_borg_rsa"
6 6
 
7
-borgbackup_version: "1.1.4"
8
-borgbackup_checksum: "sha256:4ecf507f21f0db7c437b2ef34566273d7ba5a7d05e921c6f0e3406c3f96933a7"
7
+borgbackup_version: "1.1.10"
8
+borgbackup_checksum: "sha256:6338d67aad4b5cd327b25ea363e30f0ed4abc425ce2d6a597c75a67a876ef9af"
9 9
 borgbackup_download_url: "https://github.com/borgbackup/borg/releases/download/{{ borgbackup_version }}/borg-linux64"
10 10
 
11 11
 borgbackup_compression: "auto,zlib,6"
12 12
 borgbackup_encryption_mode: keyfile
13 13
 
14 14
 borgbackup_pre_commands:
15
-  - '[[ ! -f "/usr/sbin/automysqlbackup" ]] || /usr/sbin/automysqlbackup'
15
+  - "[ -d /etc/backup.d/ ] && run-parts --verbose /etc/backup.d/"
16 16
 
17 17
 borgbackup_post_commands: []
18 18
 
19 19
 borgbackup_include:
20
-  - "/etc"
21
-  - "/home"
22
-  - "/root"
23
-  - "/var/www"
24
-  - "/var/log"
20
+  - "/"
25 21
 
26
-borgbackup_exclude: []
22
+borgbackup_exclude:
23
+  - "*/local/*"
24
+  - "*/tmp/*"
25
+  - "*CACHE*"
26
+  - "/dev"
27
+  - "/proc"
28
+  - "/run"
29
+  - "/sys"
30
+  - "/tmp"
31
+  - "/var/cache/apt"
32
+  - "/var/lib/amavis/tmp"
33
+  - "/var/lib/amavis/virusmails"
34
+  - "/var/lib/lxc"
35
+  - "/var/lib/lxcfs"
36
+  - "/var/lib/php/sessions"
37
+  - "/var/lib/php5"
38
+  - "/var/run"
39
+  - "/var/spool/postfix"
40
+  - "/var/tmp"
27 41
 
28 42
 borgbackup_retention:
29 43
   hourly: 12
@@ -42,6 +56,11 @@ borgbackup_management_station: ''
42 56
 borgbackup_management_user: ''
43 57
 borgbackup_management_ssh_pubkey: ''
44 58
 
59
+borgbackup_management_copy_keys: true
60
+borgbackup_management_key_name: 'backupserver_example_com__repo_dir_'
61
+
62
+
63
+borgbackup_remote_ratelimit: 6250 # in KiB/s so 6250 <-> 50Mb/s 
45 64
 borgbackup_owner: root
46 65
 borgbackup_group: root
47 66
 borgbackup_shell: "/bin/bash"

+ 14
- 50
tasks/borg-client.yml View File

@@ -33,57 +33,23 @@
33 33
         {% endif %}
34 34
   with_items: "{{ borgbackup_servers }}"
35 35
 
36
-- name: client | put sshpubkey on the normal backupserver
36
+- name: client | put non management sshpubkey on the normal backupserver
37 37
   authorized_key:
38 38
     user: "{{ item.user }}"
39 39
     key: "{{ sshkey.stdout }}"
40 40
     key_options: 'command="cd {{ item.home }}{{ item.pool }}/{{ inventory_hostname }};borg serve {% if borgbackup_appendonly %}--append-only {% endif %}--restrict-to-path {{ item.home }}/{{ item.pool }}/{{ inventory_hostname }}",no-port-forwarding,no-X11-forwarding,no-pty,no-agent-forwarding,no-user-rc'
41 41
   delegate_to: "{{ item.fqdn }}"
42
-  when: item.type == 'normal'
42
+  when: inventory_hostname != borgbackup_management_station
43 43
   with_items: "{{ borgbackup_servers }}"
44 44
 
45
-# rsync.net and hetzner have no python, so we can only use raw to manage ssh keys - workaround with local tmp file
46
-- name: client | get authorized_keys file
47
-  raw: scp {{ item.user }}@{{ item.fqdn }}:.ssh/authorized_keys /tmp/authkeys-{{ item.type }}-{{ item.fqdn }}-authkeys
48
-  delegate_to: localhost
49
-  become: false
50
-  when: item.type in ['rsync.net','hetzner']
51
-  with_items: "{{ borgbackup_servers }}"
52
-  changed_when: false
53
-
54
-- name: client | modify local rsync.net/hetzner authorized_keys
45
+- name: client | put management sshpubkey on backupservers, no appendonly nor path restriction
55 46
   authorized_key:
56
-    user: "{{ ansible_user_id }}"
47
+    user: "{{ item.user }}"
57 48
     key: "{{ sshkey.stdout }}"
58
-    key_options: 'command="cd {{ item.pool }}/{{ inventory_hostname }};/usr/local/bin/borg1 serve {% if borgbackup_appendonly %}--append-only {% endif %} --restrict-to-path {{ item.pool }}/{{ inventory_hostname }}",no-port-forwarding,no-X11-forwarding,no-pty,no-agent-forwarding,no-user-rc'
59
-    path: "/tmp/authkeys-{{ item.type }}-{{ item.fqdn }}-authkeys"
60
-    manage_dir: false
61
-  delegate_to: localhost
62
-  become: false
63
-  when: item.type in ['rsync.net','hetzner']
64
-  with_items: "{{ borgbackup_servers }}"
65
-  register: authkeys
66
-
67
-- name: client | upload local authorized_keys to rsync.net / hetzner
68
-  raw: scp /tmp/authkeys-{{ item.type }}-{{ item.fqdn }}-authkeys {{ item.user }}@{{ item.fqdn }}:.ssh/authorized_keys
69
-  delegate_to: localhost
70
-  become: false
71
-  when: item.type in ['rsync.net','hetzner'] and authkeys.changed
72
-  with_items: "{{ borgbackup_servers }}"
73
-
74
-- name: client | remove tmp authorized_keys files
75
-  file:
76
-    path: /tmp/authkeys-{{ item.type }}-{{ item.fqdn }}-authkeys
77
-    state: absent
78
-  delegate_to: localhost
79
-  become: false
49
+    key_options: 'command="cd {{ item.home }}{{ item.pool }}/{{ inventory_hostname }};borg serve ",no-port-forwarding,no-X11-forwarding,no-pty,no-agent-forwarding,no-user-rc'
50
+  delegate_to: "{{ item.fqdn }}"
51
+  when: inventory_hostname == borgbackup_management_station
80 52
   with_items: "{{ borgbackup_servers }}"
81
-  when: authkeys.changed
82
-  changed_when: false
83
-
84
-- name: client | check for mysql
85
-  stat: path=/var/lib/automysqlbackup
86
-  register: automysql
87 53
 
88 54
 - name: client | put wrapper script
89 55
   template:
@@ -109,12 +75,10 @@
109 75
     day: "{{ borgbackup_cron_day }}"
110 76
     job: "/usr/local/bin/borg-backup backup"
111 77
 
112
-- name: client | disable automysqlbackup cronjob, it's in our pre-backup-tasks
113
-  lineinfile:
114
-    dest: "/etc/cron.daily/automysqlbackup"
115
-    regexp: "^/usr/sbin/automysqlbackup$"
116
-    line: "#/usr/sbin/automysqlbackup"
117
-    state: "present"
118
-    backrefs: true
119
-    create: false
120
-  when: automysql.stat.isdir is defined and automysql.stat.isdir == True
78
+- name: client | create log directory
79
+  file:
80
+    path: "/var/log/borgbackup"
81
+    state: "directory"
82
+    owner: "root"
83
+    group: "root"
84
+    mode: "0755"

+ 6
- 0
tasks/main.yml View File

@@ -22,3 +22,9 @@
22 22
   when: >
23 23
     inventory_hostname in borgbackup_management_group and
24 24
     inventory_hostname not in borgbackup_servers_group
25
+
26
+- include_tasks: management-keys.yml
27
+  when: >
28
+    inventory_hostname not in borgbackup_management_group and
29
+    inventory_hostname not in borgbackup_servers_group and
30
+    borgbackup_management_copy_keys

+ 23
- 0
tasks/management-keys.yml View File

@@ -0,0 +1,23 @@
1
+---
2
+
3
+- name: management | get key file
4
+  fetch: 
5
+       src: "~{{ borgbackup_client_user}}/.config/borg/keys/{{ borgbackup_management_key_name }}{{ inventory_hostname }}" 
6
+       dest: /tmp/.borgbackup_key_{{ inventory_hostname }}
7
+       flat: yes
8
+  changed_when: false
9
+
10
+- name: management | upload key to management
11
+  raw: scp /tmp/.borgbackup_key_{{ inventory_hostname }} {{ borgbackup_management_user }}@{{ borgbackup_management_station }}:~/.config/borg/keys/{{ borgbackup_management_key_name }}{{ inventory_hostname }}
12
+  delegate_to: localhost
13
+  become: false
14
+  changed_when: false
15
+
16
+- name: management | clean local copy 
17
+  raw: rm -f /tmp/.borgbackup_key_{{ inventory_hostname }}
18
+  delegate_to: localhost
19
+  become: false
20
+  changed_when: false
21
+
22
+
23
+ 

+ 33
- 8
templates/borg-backup.sh.j2 View File

@@ -73,33 +73,58 @@ if [ "$1" = "backup" ]
73 73
   then
74 74
     date=`date +%Y%m%d-%H%M`
75 75
 
76
-    # Running some commands pre-backup
76
+    LOG_DEST="/var/log/borgbackup/${date}"
77
+    LOG_FILE="${LOG_DEST}.log"
78
+    log(){ echo -e "\n$(date '+%Y-%m-%d %H:%M:%S') $@" | tee -a "$LOG_FILE"; }
79
+    _term(){ echo -e "\n## END ##" | tee -a "$LOG_FILE"; exit 1; }
80
+    trap _term SIGINT SIGTERM
81
+
82
+# Running some commands pre-backup
83
+(
77 84
 {% for precommand in borgbackup_pre_commands %}
78
-    {{ precommand }}
85
+{{ precommand }}
79 86
 {% endfor %}
87
+) &>> "$LOG_FILE"
88
+
89
+{% if borgbackup_remote_ratelimit %}
90
+    	{% set rate_limit %} --remote-ratelimit={{borgbackup_remote_ratelimit}} {% endset %}
91
+{% else %}
92
+    	{% set rate_limit = " " %}
93
+{% endif %}
80 94
 
81 95
 {% for b in borgbackup_servers %}
82
-    printf "Backing up to {{ b.fqdn }} :\n"
83 96
 {% if b.type == 'hetzner' %}
84 97
     REPOSITORY=ssh://{{ b.user }}@{{ b.fqdn }}:23/./{{ b.home }}{{ b.pool }}/{{ inventory_hostname }}
85 98
 {% else %}
86 99
     REPOSITORY={{ b.user }}@{{ b.fqdn }}:{{ b.home }}{{ b.pool }}/{{ inventory_hostname }}
87 100
 {% endif %}
88 101
     
89
-    /usr/local/bin/borg create -x --progress --compression {{ borgbackup_compression }} --stats {{ b.options }} $REPOSITORY::$date {% for dir in borgbackup_include %}{{ dir }} {% endfor %}{% if automysql.stat.isdir is defined and automysql.stat.isdir == True %}/var/lib/automysqlbackup{% endif %} {% for dir in borgbackup_exclude %} --exclude '{{ dir }}'{% endfor %}
102
+    log "## Backing up to {{ b.fqdn }} "
103
+    /usr/local/bin/borg create {{ rate_limit }} -x --compression {{ borgbackup_compression }} --stats {{ b.options }} $REPOSITORY::$date {% for dir in borgbackup_include %}{{ dir }} {% endfor %} {% for dir in borgbackup_exclude %} --exclude '{{ dir }}'{% endfor %} &>> $LOG_FILE
90 104
 
91 105
     if [ "$?" -eq "0" ]; then printf "Backup succeeded on $date to {{ b.fqdn }}\n" >> /var/log/borg-backup.log; fi
92 106
 
107
+    log "## Checking the archive integrity "
108
+    /usr/local/bin/borg check $REPOSITORY::$date -v &>> "$LOG_FILE"
109
+
110
+    log "## Retrieving archive json file"
111
+    /usr/local/bin/borg info $REPOSITORY::$date --json > "${LOG_DEST}.json"
112
+
93 113
   {% if not borgbackup_appendonly %}
94
-    # prune old backups
95
-    /usr/local/bin/borg prune {{ b.options }} -v $REPOSITORY -H {{ borgbackup_retention.hourly }} -d {{ borgbackup_retention.daily }} -w {{ borgbackup_retention.weekly }} -m {{ borgbackup_retention.monthly }} -y {{ borgbackup_retention.yearly }}
114
+    log "## Pruning the repository"
115
+    /usr/local/bin/borg prune {{ b.options }} -v $REPOSITORY -H {{ borgbackup_retention.hourly }} -d {{ borgbackup_retention.daily }} -w {{ borgbackup_retention.weekly }} -m {{ borgbackup_retention.monthly }} -y {{ borgbackup_retention.yearly }} &>> "$LOG_FILE"
96 116
   {% endif %}
97 117
 {% endfor %}
98 118
 
99 119
     # Running some commands post-backup
100
-{% for postcommand in borgbackup_post_commands %}
120
+{% if borgbackup_post_commands |length > 1  %} 
121
+    (
122
+    {% for postcommand in borgbackup_post_commands %}
101 123
     {{ postcommand }}
102
-{% endfor %}
124
+    {% endfor %}
125
+    ) &>> "$LOG_FILE"
126
+{% endif %}
103 127
 
128
+    _term
104 129
 fi
105 130
 

+ 26
- 18
templates/prune.sh.j2 View File

@@ -1,27 +1,35 @@
1 1
 #jinja2:lstrip_blocks: True
2 2
 #!/bin/bash
3
+usage(){
4
+cat << EOL
3 5
 
4
-# This script is intended to run on a trusted management station to purge borg repo's in
5
-# append-only mode.
6
-# Don't put it on the backup server, it contains all borg secrets!
6
+  This script is intended to run on a trusted management station to purge borg repo's in
7
+  append-only mode.
8
+  Don't put it on the backup server, it contains all borg secrets!
7 9
 
8
-{% for h in groups['all'] %}
9
-  {% if hostvars[h].borgbackup_required | default(True) -%}
10
-  # Host: {{ h }}
10
+EOL
11
+}
11 12
 
12
-    export BORG_PASSPHRASE={{ hostvars[h].borgbackup_passphrase }}
13
+DATE=$(date +%y%m%d)
14
+LOG_DIR=/var/log/borgbackup
15
+[ ! -d $LOG_DIR ] && mkdir $LOG_DIR
16
+LOG_FILE=/var/log/borgbackup-prune/${DATE}.log
17
+exec &> >(tee "$LOG_FILE")
13 18
 
14
-    {% if hostvars[h].borgbackup_management_station is defined and inventory_hostname == hostvars[h].borgbackup_management_station %}
15
-    {% for b in hostvars[h].borgbackup_servers %}
16
-    # {{ b.fqdn }}
17
-{% if b.type == 'hetzner' %}
18
-      REPOSITORY=ssh://{{ b.user }}@{{ b.fqdn }}:23/./{{ b.home }}{{ b.pool }}/{{ h }}
19
-{% else %}
20
-      REPOSITORY={{ b.user }}@{{ b.fqdn }}:{{ b.home }}{{ b.pool }}/{{ h }}
21
-{% endif %}
22
-      /usr/local/bin/borg prune -v $REPOSITORY {{ b.options }} -H {{ hostvars[h].borgbackup_retention.hourly }} -d {{ hostvars[h].borgbackup_retention.daily }} -w {{ hostvars[h].borgbackup_retention.weekly }} -m {{ hostvars[h].borgbackup_retention.monthly }} -y {{ hostvars[h].borgbackup_retention.yearly }}
19
+{% for h in groups['all'] %}
20
+  {% if h != borgbackup_management_station and h not in groups['borgbackup_servers'] -%} 
21
+echo "Host: {{ h }}"
22
+export BORG_PASSPHRASE={{ hostvars[h].borgbackup_passphrase }}
23
+    {% if borgbackup_management_station is defined and inventory_hostname == borgbackup_management_station %}
24
+      {% for b in borgbackup_servers %}
25
+        {% if b.type == 'hetzner' %}
26
+REPOSITORY=ssh://{{ b.user }}@{{ b.fqdn }}:23/./{{ b.home }}{{ b.pool }}/{{ h }}
27
+        {% else %}
28
+REPOSITORY={{ b.user }}@{{ b.fqdn }}:{{ b.home }}{{ b.pool }}/{{ h }}
29
+        {% endif %}
30
+/usr/local/bin/borg prune -v $REPOSITORY {{ b.options }} -H {{ borgbackup_retention.hourly }} -d {{ borgbackup_retention.daily }} -w {{ borgbackup_retention.weekly }} -m {{ borgbackup_retention.monthly }} -y {{ borgbackup_retention.yearly }}
23 31
 
24
-  {% endfor %}
25
-  {% endif %}
32
+      {% endfor %}
33
+    {% endif %}
26 34
   {% endif %}
27 35
 {% endfor %}

Loading…
Cancel
Save