Add restic backups to s3 bucket

playbooks/03-backups.yml

@@ -5,3 +5,4 @@
   tasks:
     - import_tasks: tasks/backups/00-zfs-snapshots.yml
     - import_tasks: tasks/backups/01-restic-setup.yml
+    - import_tasks: tasks/backups/02-restic-enable.yml

playbooks/03-test-backups.yml

@@ -0,0 +1,6 @@
+---
+- name: Configure yggdrasil backups
+  hosts: yggdrasil
+
+  tasks:
+    - import_tasks: tasks/backups/01-restic-setup.yml

playbooks/filesystem/yggdrasil/etc/systemd/system/restic-service-data.service.j2

@@ -6,5 +6,6 @@ Documentation=man:restic(8)
 Type=oneshot
 Environment=TZ=UTC
 Environment=RESTIC_CACHE_DIR=/var/cache/restic
+Environment=RESTIC_PASSWORD_FILE=/etc/restic.password
 EnvironmentFile=/etc/scaleway.keys
-ExecStart=/usr/local/sbin/restic-service-data --password-file /etc/restic.password --data-root /var/lib/{{ ansible_hostname }}/data --bucket-endpoint {{ scw_bucket_endpoint }}
+ExecStart=/usr/local/sbin/restic-service-data --data-root /var/lib/{{ ansible_hostname }}/data --bucket-endpoint {{ scw_bucket_endpoint }}

playbooks/filesystem/yggdrasil/usr/local/sbin/restic-service-data

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import subprocess
+
+
+def get_service_dataset_paths(data_root):
+    return { d: os.path.join(data_root, d) for d in os.listdir(data_root) }
+
+def get_last_daily_snapshot_name(dataset_path):
+    dataset = ''.join(["rpool", dataset_path])
+    snapshots = subprocess.getoutput(
+        f"zfs list -t snapshot -H -r {dataset} -o name -s creation"
+    )
+    daily_snapshots = filter(lambda s: s.endswith("_daily"), snapshots.split('\n'))
+    return list(daily_snapshots)[-1]
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Backup service data using restic")
+    parser.add_argument("--data-root", type=str, required=True,
+                        help="Service data root")
+    parser.add_argument("--bucket-endpoint", type=str, required=True,
+                        help="S3 bucket endpoint")
+    args = parser.parse_args()
+
+    snapshots_for_backup = {
+        service: {
+            "dataset_path": service_dataset_path,
+            "snapshot": get_last_daily_snapshot_name(service_dataset_path),
+        } for service, service_dataset_path in get_service_dataset_paths(args.data_root).items()
+    }
+
+    for service, info in snapshots_for_backup.items():
+        backup_path = os.path.normpath(os.path.join("/", "mnt", os.path.relpath(info["dataset_path"], "/")))
+        snapshot = info["snapshot"]
+        restic_cmd_base = "restic " \
+            f"--repo s3:https://{args.bucket_endpoint}/the-nine-worlds---{service} " \
+            "--option s3.storage-class=ONEZONE_IA"
+
+        print(f"Backing up {service} : {snapshot}")
+
+        subprocess.run(f"zfs clone -o mountpoint={backup_path} {snapshot} rpool/restic",
+                       shell=True, check=True)
+        try:
+            subprocess.run(f"{restic_cmd_base} snapshots || {restic_cmd_base} init",
+                           shell=True, check=True)
+            subprocess.run(f"cd {backup_path} && "
+                           f"{restic_cmd_base} backup .",
+                           shell=True, check=True)
+            subprocess.run(f"{restic_cmd_base} forget --prune --keep-daily 90 --keep-monthly 12",
+                           shell=True, check=True)
+            subprocess.run(f"{restic_cmd_base} check", shell=True, check=True)
+
+        finally:
+            subprocess.run("zfs destroy rpool/restic", shell=True, check=True)

playbooks/filesystem/yggdrasil/usr/local/sbin/restic-service-data.j2

@@ -1,48 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import os
-import subprocess
-
-
-def get_service_data_paths(data_root):
-    dirs = os.listdir(data_root)
-    return { d: os.path.join(data_root, d) for d in dirs }
-
-def get_last_daily_snapshot_name(dataset_path):
-    dataset = ''.join(["rpool", dataset_path])
-    snapshots = subprocess.getoutput(
-        f"zfs list -t snapshot -H -r {dataset} -o name -s creation"
-    )
-    daily_snapshots = filter(lambda s: s.endswith("_daily"), snapshots.split('\n'))
-    return list(daily_snapshots)[-1]
-
-def get_snapshot_mount_path(snapshot):
-    return snapshot.replace("rpool/", "/", 1).replace("@", "/.zfs/snapshot/", 1)
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Backup service data using restic")
-    parser.add_argument("--data-root", type=str, required=True,
-                        help="Service data root")
-    parser.add_argument("--bucket-endpoint", type=str, required=True,
-                        help="S3 bucket endpoint")
-    parser.add_argument("--password-file", type=str, required=True,
-                        help="Password file for restic repo")
-    args = parser.parse_args()
-
-    snapshots_for_backup = {}
-    for service, service_data_root in get_service_data_paths(args.data_root).items():
-        last_daily_snapshot = get_last_daily_snapshot_name(service_data_root)
-        snapshots_for_backup[service] = get_snapshot_mount_path(last_daily_snapshot)
-
-    for service, snapshot_path in snapshots_for_backup.items():
-        print(f"Backing up {service} @ {snapshot_path}")
-
-        restic_cmd_base = "restic " \
-            f"--password-file {args.password_file} " \
-            f"--repo s3:https://{args.bucket_endpoint}/{{ ansible_hostname }}-{service}"
-
-        subprocess.run(f"{restic_cmd_base} snapshots || {restic_cmd_base} init",
-                      shell=True, check=True)
-        subprocess.run(f"{restic_cmd_base} backup -o s3.storage-class=ONEZONE_IA {snapshot_path}",
-                       shell=True, check=True)

playbooks/tasks/backups/01-restic-setup.yml

@@ -38,36 +38,3 @@
     path: /var/cache/restic
     state: directory
     mode: 0755
-
-- name: Install the restic backup script
-  template:
-    src: ./filesystem/{{ ansible_hostname }}/usr/local/sbin/restic-service-data.j2
-    dest: /usr/local/sbin/restic-service-data
-    mode: 0755
-
-- name: Install the restic backup service file
-  template:
-    src: ./filesystem/{{ ansible_hostname }}/etc/systemd/system/restic-service-data.service.j2
-    dest: /etc/systemd/system/restic-service-data.service
-    mode: 0644
-  register: systemd_restic_service_data_service_file
-
-- name: Install the restic backup timer file
-  copy:
-    src: ./filesystem/{{ ansible_hostname }}/etc/systemd/system/restic-service-data.timer
-    dest: /etc/systemd/system/restic-service-data.timer
-    mode: 0644
-  register: systemd_restic_service_data_timer_file
-
-- name: SystemD daemon reload
-  systemd:
-    daemon_reload: true
-  when:
-    systemd_restic_service_data_service_file is changed or
-    systemd_restic_service_data_timer_file is changed
-
-- name: Enable restic backup
-  systemd:
-    name: restic-service-data.timer
-    enabled: yes
-    state: started

playbooks/tasks/backups/02-restic-enable.yml

@@ -0,0 +1,32 @@
+- name: Install the restic backup script
+  copy:
+    src: ./filesystem/{{ ansible_hostname }}/usr/local/sbin/restic-service-data
+    dest: /usr/local/sbin/restic-service-data
+    mode: 0755
+
+- name: Install the restic backup service file
+  template:
+    src: ./filesystem/{{ ansible_hostname }}/etc/systemd/system/restic-service-data.service.j2
+    dest: /etc/systemd/system/restic-service-data.service
+    mode: 0644
+  register: systemd_restic_service_data_service_file
+
+- name: Install the restic backup timer file
+  copy:
+    src: ./filesystem/{{ ansible_hostname }}/etc/systemd/system/restic-service-data.timer
+    dest: /etc/systemd/system/restic-service-data.timer
+    mode: 0644
+  register: systemd_restic_service_data_timer_file
+
+- name: SystemD daemon reload
+  systemd:
+    daemon_reload: true
+  when:
+    systemd_restic_service_data_service_file is changed or
+    systemd_restic_service_data_timer_file is changed
+
+- name: Enable restic backup
+  systemd:
+    name: restic-service-data.timer
+    enabled: yes
+    state: started