From 227b2c50a3895e3645511fa19ef7111c8d1ad682 Mon Sep 17 00:00:00 2001 From: Wojciech Kozlowski Date: Wed, 19 Oct 2022 22:21:32 +0200 Subject: [PATCH] Add restic backups to s3 bucket --- playbooks/03-backups.yml | 1 + playbooks/03-test-backups.yml | 6 ++ .../system/restic-service-data.service.j2 | 3 +- .../usr/local/sbin/restic-service-data | 56 +++++++++++++++++++ .../usr/local/sbin/restic-service-data.j2 | 48 ---------------- playbooks/tasks/backups/01-restic-setup.yml | 33 ----------- playbooks/tasks/backups/02-restic-enable.yml | 32 +++++++++++ 7 files changed, 97 insertions(+), 82 deletions(-) create mode 100644 playbooks/03-test-backups.yml create mode 100644 playbooks/filesystem/yggdrasil/usr/local/sbin/restic-service-data delete mode 100644 playbooks/filesystem/yggdrasil/usr/local/sbin/restic-service-data.j2 create mode 100644 playbooks/tasks/backups/02-restic-enable.yml diff --git a/playbooks/03-backups.yml b/playbooks/03-backups.yml index 2a8c7fd..6b8ae35 100644 --- a/playbooks/03-backups.yml +++ b/playbooks/03-backups.yml @@ -5,3 +5,4 @@ tasks: - import_tasks: tasks/backups/00-zfs-snapshots.yml - import_tasks: tasks/backups/01-restic-setup.yml + - import_tasks: tasks/backups/02-restic-enable.yml diff --git a/playbooks/03-test-backups.yml b/playbooks/03-test-backups.yml new file mode 100644 index 0000000..eb82579 --- /dev/null +++ b/playbooks/03-test-backups.yml @@ -0,0 +1,6 @@ +--- +- name: Configure yggdrasil backups + hosts: yggdrasil + + tasks: + - import_tasks: tasks/backups/01-restic-setup.yml diff --git a/playbooks/filesystem/yggdrasil/etc/systemd/system/restic-service-data.service.j2 b/playbooks/filesystem/yggdrasil/etc/systemd/system/restic-service-data.service.j2 index 7c4da53..21694b0 100644 --- a/playbooks/filesystem/yggdrasil/etc/systemd/system/restic-service-data.service.j2 +++ b/playbooks/filesystem/yggdrasil/etc/systemd/system/restic-service-data.service.j2 @@ -6,5 +6,6 @@ Documentation=man:restic(8) Type=oneshot Environment=TZ=UTC Environment=RESTIC_CACHE_DIR=/var/cache/restic +Environment=RESTIC_PASSWORD_FILE=/etc/restic.password EnvironmentFile=/etc/scaleway.keys -ExecStart=/usr/local/sbin/restic-service-data --password-file /etc/restic.password --data-root /var/lib/{{ ansible_hostname }}/data --bucket-endpoint {{ scw_bucket_endpoint }} +ExecStart=/usr/local/sbin/restic-service-data --data-root /var/lib/{{ ansible_hostname }}/data --bucket-endpoint {{ scw_bucket_endpoint }} diff --git a/playbooks/filesystem/yggdrasil/usr/local/sbin/restic-service-data b/playbooks/filesystem/yggdrasil/usr/local/sbin/restic-service-data new file mode 100644 index 0000000..e7a1b84 --- /dev/null +++ b/playbooks/filesystem/yggdrasil/usr/local/sbin/restic-service-data @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 + +import argparse +import os +import subprocess + + +def get_service_dataset_paths(data_root): + return { d: os.path.join(data_root, d) for d in os.listdir(data_root) } + +def get_last_daily_snapshot_name(dataset_path): + dataset = ''.join(["rpool", dataset_path]) + snapshots = subprocess.getoutput( + f"zfs list -t snapshot -H -r {dataset} -o name -s creation" + ) + daily_snapshots = filter(lambda s: s.endswith("_daily"), snapshots.split('\n')) + return list(daily_snapshots)[-1] + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Backup service data using restic") + parser.add_argument("--data-root", type=str, required=True, + help="Service data root") + parser.add_argument("--bucket-endpoint", type=str, required=True, + help="S3 bucket endpoint") + args = parser.parse_args() + + snapshots_for_backup = { + service: { + "dataset_path": service_dataset_path, + "snapshot": get_last_daily_snapshot_name(service_dataset_path), + } for service, service_dataset_path in get_service_dataset_paths(args.data_root).items() + } + + for service, info in snapshots_for_backup.items(): + backup_path = os.path.normpath(os.path.join("/", "mnt", os.path.relpath(info["dataset_path"], "/"))) + snapshot = info["snapshot"] + restic_cmd_base = "restic " \ + f"--repo s3:https://{args.bucket_endpoint}/the-nine-worlds---{service} " \ + "--option s3.storage-class=ONEZONE_IA" + + print(f"Backing up {service} : {snapshot}") + + subprocess.run(f"zfs clone -o mountpoint={backup_path} {snapshot} rpool/restic", + shell=True, check=True) + try: + subprocess.run(f"{restic_cmd_base} snapshots || {restic_cmd_base} init", + shell=True, check=True) + subprocess.run(f"cd {backup_path} && " + f"{restic_cmd_base} backup .", + shell=True, check=True) + subprocess.run(f"{restic_cmd_base} forget --prune --keep-daily 90 --keep-monthly 12", + shell=True, check=True) + subprocess.run(f"{restic_cmd_base} check", shell=True, check=True) + + finally: + subprocess.run("zfs destroy rpool/restic", shell=True, check=True) diff --git a/playbooks/filesystem/yggdrasil/usr/local/sbin/restic-service-data.j2 b/playbooks/filesystem/yggdrasil/usr/local/sbin/restic-service-data.j2 deleted file mode 100644 index ea8af09..0000000 --- a/playbooks/filesystem/yggdrasil/usr/local/sbin/restic-service-data.j2 +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import os -import subprocess - - -def get_service_data_paths(data_root): - dirs = os.listdir(data_root) - return { d: os.path.join(data_root, d) for d in dirs } - -def get_last_daily_snapshot_name(dataset_path): - dataset = ''.join(["rpool", dataset_path]) - snapshots = subprocess.getoutput( - f"zfs list -t snapshot -H -r {dataset} -o name -s creation" - ) - daily_snapshots = filter(lambda s: s.endswith("_daily"), snapshots.split('\n')) - return list(daily_snapshots)[-1] - -def get_snapshot_mount_path(snapshot): - return snapshot.replace("rpool/", "/", 1).replace("@", "/.zfs/snapshot/", 1) - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Backup service data using restic") - parser.add_argument("--data-root", type=str, required=True, - help="Service data root") - parser.add_argument("--bucket-endpoint", type=str, required=True, - help="S3 bucket endpoint") - parser.add_argument("--password-file", type=str, required=True, - help="Password file for restic repo") - args = parser.parse_args() - - snapshots_for_backup = {} - for service, service_data_root in get_service_data_paths(args.data_root).items(): - last_daily_snapshot = get_last_daily_snapshot_name(service_data_root) - snapshots_for_backup[service] = get_snapshot_mount_path(last_daily_snapshot) - - for service, snapshot_path in snapshots_for_backup.items(): - print(f"Backing up {service} @ {snapshot_path}") - - restic_cmd_base = "restic " \ - f"--password-file {args.password_file} " \ - f"--repo s3:https://{args.bucket_endpoint}/{{ ansible_hostname }}-{service}" - - subprocess.run(f"{restic_cmd_base} snapshots || {restic_cmd_base} init", - shell=True, check=True) - subprocess.run(f"{restic_cmd_base} backup -o s3.storage-class=ONEZONE_IA {snapshot_path}", - shell=True, check=True) diff --git a/playbooks/tasks/backups/01-restic-setup.yml b/playbooks/tasks/backups/01-restic-setup.yml index 95b8a7a..9326815 100644 --- a/playbooks/tasks/backups/01-restic-setup.yml +++ b/playbooks/tasks/backups/01-restic-setup.yml @@ -38,36 +38,3 @@ path: /var/cache/restic state: directory mode: 0755 - -- name: Install the restic backup script - template: - src: ./filesystem/{{ ansible_hostname }}/usr/local/sbin/restic-service-data.j2 - dest: /usr/local/sbin/restic-service-data - mode: 0755 - -- name: Install the restic backup service file - template: - src: ./filesystem/{{ ansible_hostname }}/etc/systemd/system/restic-service-data.service.j2 - dest: /etc/systemd/system/restic-service-data.service - mode: 0644 - register: systemd_restic_service_data_service_file - -- name: Install the restic backup timer file - copy: - src: ./filesystem/{{ ansible_hostname }}/etc/systemd/system/restic-service-data.timer - dest: /etc/systemd/system/restic-service-data.timer - mode: 0644 - register: systemd_restic_service_data_timer_file - -- name: SystemD daemon reload - systemd: - daemon_reload: true - when: - systemd_restic_service_data_service_file is changed or - systemd_restic_service_data_timer_file is changed - -- name: Enable restic backup - systemd: - name: restic-service-data.timer - enabled: yes - state: started diff --git a/playbooks/tasks/backups/02-restic-enable.yml b/playbooks/tasks/backups/02-restic-enable.yml new file mode 100644 index 0000000..0cac96c --- /dev/null +++ b/playbooks/tasks/backups/02-restic-enable.yml @@ -0,0 +1,32 @@ +- name: Install the restic backup script + copy: + src: ./filesystem/{{ ansible_hostname }}/usr/local/sbin/restic-service-data + dest: /usr/local/sbin/restic-service-data + mode: 0755 + +- name: Install the restic backup service file + template: + src: ./filesystem/{{ ansible_hostname }}/etc/systemd/system/restic-service-data.service.j2 + dest: /etc/systemd/system/restic-service-data.service + mode: 0644 + register: systemd_restic_service_data_service_file + +- name: Install the restic backup timer file + copy: + src: ./filesystem/{{ ansible_hostname }}/etc/systemd/system/restic-service-data.timer + dest: /etc/systemd/system/restic-service-data.timer + mode: 0644 + register: systemd_restic_service_data_timer_file + +- name: SystemD daemon reload + systemd: + daemon_reload: true + when: + systemd_restic_service_data_service_file is changed or + systemd_restic_service_data_timer_file is changed + +- name: Enable restic backup + systemd: + name: restic-service-data.timer + enabled: yes + state: started