Port restic backups

This commit is contained in:
Wojciech Kozlowski 2022-12-18 17:06:27 +01:00
parent 8b083f5939
commit 735f835c8e
29 changed files with 320 additions and 195 deletions

View File

@ -79,8 +79,3 @@ services:
# services:user_setup
# --------------------------------------------------------------------------------------------------
services_bridge_gateway: "{{ vpn_bridge_address }}"
scw_bucket_endpoint: "{{ vault_scw_bucket_endpoint }}"
scw_access_key: "{{ vault_scw_access_key }}"
scw_secret_key: "{{ vault_scw_secret_key }}"
restic_password: "{{ vault_restic_password }}"

View File

@ -32,7 +32,7 @@ vpn_wireguard_server_address: "{{ vault_vpn_wireguard_server_address }}"
vpn_wireguard_routing_table: 66
# --------------------------------------------------------------------------------------------------
# backup:snapshots
# backups:snapshots
# --------------------------------------------------------------------------------------------------
backups_snapshots_sanoid_system_datasets:
- name: "bpool/BOOT"
@ -73,8 +73,45 @@ services_host_services:
# --------------------------------------------------------------------------------------------------
# services:backups
# --------------------------------------------------------------------------------------------------
services_backups_backup_dataset: "hpool/backup"
services_backups_backup_root_dataset: "{{ services_root_dataset |
replace('rpool/var/lib', 'hpool/backup') }}"
services_backups_backup_data_dataset: "{{ services_data_dataset |
replace('rpool/var/lib', 'hpool/backup') }}"
services_backups_snapshots_dataset: "hpool/backup"
services_backups_snapshots_root_dataset: "{{ services_root_dataset |
replace('rpool/var/lib', 'hpool/backup') }}"
services_backups_snapshots_data_dataset: "{{ services_data_dataset |
replace('rpool/var/lib', 'hpool/backup') }}"
services_backups_snapshots_services: "\
{% set services_backups_snapshots_service = {} %}\
{% for service in services_host_services.keys() %}\
{{ services_backups_snapshots_service.update(
{ service: {
'backup_dataset': ( services_backups_snapshots_data_dataset ~ '/pod-' ~ service ),
'recursive': true,
'skip_parent': true,
}}
) }}\
{% endfor %}\
{{ services_backups_snapshots_service }}"
services_backups_restic_restic_password: "{{ vault_services_backups_restic_restic_password }}"
services_backups_restic_aws_access_key_id: "{{ vault_services_backups_restic_aws_access_key_id }}"
services_backups_restic_aws_secret_access_key: "\
{{ vault_services_backups_restic_aws_secret_access_key }}"
services_backups_restic_aws_bucket_endpoint: "\
{{ vault_services_backups_restic_aws_bucket_endpoint }}"
services_backups_restic_services: "\
{% set services_backups_restic_service = {} %}\
{% for service in services_host_services.keys() %}\
{{ services_backups_restic_service.update(
{ service: {
'aws_access_key_id': services_backups_restic_aws_access_key_id,
'aws_secret_access_key': services_backups_restic_aws_secret_access_key,
'aws_keys_file': '/etc/restic-aws-keys.yml',
'aws_bucket_endpoint': services_backups_restic_aws_bucket_endpoint,
'aws_bucket_prefix': ( 'the-nine-worlds---pod-' ~ service ),
'restic_password': services_backups_restic_restic_password,
'restic_password_file': '/etc/restic.password',
'restic_keep_daily': 30,
'restic_keep_monthly': 3,
}}
) }}\
{% endfor %}\
{{ services_backups_restic_service }}"

View File

@ -1,7 +0,0 @@
---
- name: Configure yggdrasil backups
hosts: yggdrasil
tasks:
- import_tasks: tasks/backups/01-restic-setup.yml
- import_tasks: tasks/backups/02-restic-enable.yml

View File

@ -1 +0,0 @@
{{ restic_password }}

View File

@ -1,2 +0,0 @@
AWS_ACCESS_KEY_ID={{ scw_access_key }}
AWS_SECRET_ACCESS_KEY={{ scw_secret_key }}

View File

@ -1,12 +0,0 @@
[Unit]
Description=Backup volume snapshots using restic
Documentation=man:restic(8)
OnFailure=status-mail@%n.service
[Service]
Type=oneshot
Environment=TZ=UTC
Environment=RESTIC_CACHE_DIR=/var/cache/restic
Environment=RESTIC_PASSWORD_FILE=/etc/restic.password
EnvironmentFile=/etc/scaleway.keys
ExecStart=/usr/local/sbin/restic-volume-data --root-dataset rpool/var/lib/{{ ansible_hostname }}/data --bucket-endpoint {{ scw_bucket_endpoint }}

View File

@ -1,66 +0,0 @@
#!/usr/bin/env python3
import argparse
import os
import subprocess
def get_volume_datasets(root_dataset):
zfs_list = subprocess.getoutput(
f"zfs list -H -r {root_dataset} -o name,mountpoint"
)
zfs_list_lines = zfs_list.split('\n')
zfs_list_lines_items = map(lambda l: l.split(), zfs_list_lines)
return {
os.path.relpath(dataset, root_dataset).replace("/", "---"): {
"dataset": dataset,
"mountpoint": mountpoint,
} for dataset, mountpoint in zfs_list_lines_items if ((dataset != root_dataset) and
os.path.ismount(mountpoint))
}
def get_last_daily_snapshot(dataset):
snapshots = subprocess.getoutput(
f"zfs list -t snapshot -H -r {dataset} -o name -s creation"
)
daily_snapshots = filter(lambda s: s.endswith("_daily"), snapshots.split('\n'))
last_daily_snapshot = list(daily_snapshots)[-1]
assert '@' in last_daily_snapshot
assert last_daily_snapshot.split('@')[0] == dataset
return last_daily_snapshot
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Backup service data using restic")
parser.add_argument("--root-dataset", type=str, required=True,
help="The root data root whose children are to be backed up")
parser.add_argument("--bucket-endpoint", type=str, required=True,
help="S3 bucket endpoint for the backups")
args = parser.parse_args()
volume_datasets = get_volume_datasets(args.root_dataset)
for volume, properties in volume_datasets.items():
properties["snapshot"] = get_last_daily_snapshot(properties["dataset"])
for volume, properties in volume_datasets.items():
mountpoint = properties["mountpoint"]
snapshot = properties["snapshot"]
backup_path = os.path.normpath(os.path.join("/", "mnt", os.path.relpath(mountpoint, "/")))
bucket_name = f"the-nine-worlds---{volume}"
restic_cmd_base = "restic " \
f"--repo s3:https://{args.bucket_endpoint}/{bucket_name} " \
"--option s3.storage-class=ONEZONE_IA"
print(f"Backing up {bucket_name} : {snapshot}", flush=True)
subprocess.run(f"zfs clone -o mountpoint={backup_path} {snapshot} rpool/restic",
shell=True, check=True)
try:
subprocess.run(f"{restic_cmd_base} snapshots || {restic_cmd_base} init",
shell=True, check=True)
subprocess.run(f"{restic_cmd_base} backup .", shell=True, cwd=backup_path, check=True)
subprocess.run(f"{restic_cmd_base} forget --prune --keep-daily 30 --keep-monthly 3",
shell=True, check=True)
subprocess.run(f"{restic_cmd_base} check", shell=True, check=True)
finally:
subprocess.run("zfs destroy rpool/restic", shell=True, check=True)

View File

@ -1,40 +0,0 @@
- name: Check if restic is insalled
stat:
path: /usr/local/bin/restic
register: restic_path
- block:
- name: Download restic binary
get_url:
url: https://github.com/restic/restic/releases/download/v0.14.0/restic_0.14.0_linux_amd64.bz2
dest: /usr/local/bin/restic.bz2
mode: 0644
- name: Unpack restic binary
command: bunzip2 /usr/local/bin/restic.bz2
when:
not restic_path.stat.exists
- name: Ensure restic is executable
file:
path: /usr/local/bin/restic
mode: 0755
- name: Create scaleway key file
template:
src: ./filesystem/{{ ansible_hostname }}/etc/scaleway.keys.j2
dest: /etc/scaleway.keys
mode: 0600
- name: Create restic password file
template:
src: ./filesystem/{{ ansible_hostname }}/etc/restic.password.j2
dest: /etc/restic.password
mode: 0600
- name: Create a cache directory for restic
file:
path: /var/cache/restic
state: directory
mode: 0755

View File

@ -1,32 +0,0 @@
- name: Install the restic backup script
copy:
src: ./filesystem/{{ ansible_hostname }}/usr/local/sbin/restic-volume-data
dest: /usr/local/sbin/restic-volume-data
mode: 0755
- name: Install the restic backup service file
template:
src: ./filesystem/{{ ansible_hostname }}/etc/systemd/system/restic-volume-data.service.j2
dest: /etc/systemd/system/restic-volume-data.service
mode: 0644
register: systemd_restic_service_data_service_file
- name: Install the restic backup timer file
copy:
src: ./filesystem/{{ ansible_hostname }}/etc/systemd/system/restic-volume-data.timer
dest: /etc/systemd/system/restic-volume-data.timer
mode: 0644
register: systemd_restic_service_data_timer_file
- name: SystemD daemon reload
systemd:
daemon_reload: true
when:
systemd_restic_service_data_service_file is changed or
systemd_restic_service_data_timer_file is changed
- name: Enable restic backup
systemd:
name: restic-volume-data.timer
enabled: yes
state: started

View File

@ -5,6 +5,6 @@
- role: "snapshots"
when: the_nine_worlds_production | bool
tags: "backups:snapshots"
# - role: "backups"
# when: the_nine_worlds_production | bool
# tags: "backups:restic"
- role: "restic"
when: the_nine_worlds_production | bool
tags: "backups:restic"

View File

@ -0,0 +1,132 @@
#!/usr/bin/env python3
import argparse
import os
import subprocess
import yaml
def load_and_validate_config_dir(config_dir):
if not os.path.isdir(config_dir):
raise ValueError(f"{config_dir} is not a directory")
return [
load_and_validate_config_file(os.path.join(config_dir, file))
for file in os.listdir(config_dir)
]
def load_and_validate_config_file(config_file_path):
if not os.path.isfile(config_file_path):
raise ValueError(f"{config_file_path} is not a file")
with open(config_file_path, encoding="utf-8") as config_file:
config = yaml.safe_load(config_file)
for key in [
"dataset",
"aws_bucket_keys_file",
"aws_bucket_endpoint",
"aws_bucket_prefix",
"restic_password_file",
"restic_keep_daily",
"restic_keep_monthly",
]:
if key not in config:
raise KeyError(f"{key} must be present in {config_file_path}")
for file in [config["restic_password_file"], config["aws_bucket_keys_file"]]:
if not os.path.isfile(file):
raise ValueError(f"{file} is not a file")
return config
def get_volume_datasets(root_dataset):
zfs_list = subprocess.getoutput(
f"zfs list -H -r {root_dataset} -o name,mountpoint"
)
zfs_list_lines = zfs_list.split('\n')
zfs_list_lines_items = map(lambda l: l.split(), zfs_list_lines)
return {
os.path.relpath(dataset, root_dataset): {
"dataset": dataset,
"mountpoint": mountpoint,
} for dataset, mountpoint in zfs_list_lines_items if os.path.ismount(mountpoint)
}
def get_last_daily_snapshot(dataset):
snapshots = subprocess.getoutput(
f"zfs list -t snapshot -H -r {dataset} -o name -s creation"
)
daily_snapshots = filter(lambda s: s.endswith("_daily"), snapshots.split('\n'))
last_daily_snapshot = list(daily_snapshots)[-1]
assert '@' in last_daily_snapshot
assert last_daily_snapshot.split('@')[0] == dataset
return last_daily_snapshot
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Backup service data using restic")
parser.add_argument("--config-dir", type=str, default="/etc/restic-batch.d",
help="Path to directory with YAML config files")
args = parser.parse_args()
config_list = load_and_validate_config_dir(args.config_dir)
for config in config_list:
volume_datasets = get_volume_datasets(config["dataset"])
for volume, properties in volume_datasets.items():
properties["snapshot"] = get_last_daily_snapshot(properties["dataset"])
config["volume_datasets"] = volume_datasets
for config in config_list:
for volume, properties in config["volume_datasets"].items():
snapshot = properties["snapshot"]
mountpoint = properties["mountpoint"]
backup_path = os.path.normpath(
os.path.join("/", "mnt", os.path.relpath(mountpoint, "/"))
)
bucket_name = config["aws_bucket_prefix"]
if volume != ".":
bucket_name = f"{bucket_name}---{volume.replace('/', '---')}"
bucket_repo = f"s3:https://{config['aws_bucket_endpoint']}/{bucket_name}"
restic_cmd_base = [
"/usr/local/bin/restic",
"--password-file", config["restic_password_file"],
"--repo", bucket_repo,
"--option", "s3.storage-class=ONEZONE_IA",
]
with open(config["aws_bucket_keys_file"], encoding="utf-8") as keys_file:
aws_keys = yaml.safe_load(keys_file)
environ = {**os.environ, **aws_keys}
print(f"Backing up {bucket_name} : {snapshot}", flush=True)
subprocess.run(["/usr/sbin/zfs", "clone",
"-o", f"mountpoint={backup_path}",
snapshot, "rpool/restic"],
check=True,
)
try:
try:
subprocess.run(restic_cmd_base + ["snapshots"], env=environ, check=True)
except subprocess.CalledProcessError:
subprocess.run(restic_cmd_base + ["init"], env=environ, check=True)
subprocess.run(restic_cmd_base + ["backup", "."],
cwd=backup_path, env=environ, check=True)
subprocess.run(
restic_cmd_base + ["forget", "--prune",
"--keep-daily", config["restic_keep_daily"],
"--keep-monthly", config["restic_keep_monthly"]],
env=environ, check=True,
)
subprocess.run(restic_cmd_base + ["check"], env=environ, check=True)
finally:
subprocess.run(["/usr/sbin/zfs", "destroy", "rpool/restic"], check=True)

View File

@ -0,0 +1,10 @@
[Unit]
Description=Backup snapshots using restic
Documentation=man:restic(8)
OnFailure=status-mail@%n.service
[Service]
Type=oneshot
Environment=TZ=UTC
Environment=RESTIC_CACHE_DIR=/var/cache/restic
ExecStart=/usr/local/sbin/restic-batch --config-dir /etc/restic-batch.d

View File

@ -1,5 +1,5 @@
[Unit]
Description=Daily restic volume backup
Description=Daily restic snapshot backup
Documentation=man:restic(8)
[Timer]

View File

@ -0,0 +1,68 @@
- name: "check if restic is installed"
ansible.builtin.stat:
path: "/usr/local/bin/restic"
register: backups_restic_binary_path
- block:
- name: "download restic binary"
ansible.builtin.get_url:
url: "https://github.com/restic/restic/releases/download/v0.14.0/restic_0.14.0_linux_amd64.bz2"
dest: "/usr/local/bin/restic.bz2"
mode: 0644
- name: "unpack restic binary"
command: "bunzip2 /usr/local/bin/restic.bz2"
when:
not backups_restic_binary_path.stat.exists
- name: "ensure restic is executable"
ansible.builtin.file:
path: "/usr/local/bin/restic"
mode: 0755
- name: "create a cache directory for restic"
ansible.builtin.file:
path: "/var/cache/restic"
state: "directory"
mode: 0755
- name: "create resic-batch config directory"
ansible.builtin.file:
path: "/etc/restic-batch.d"
state: "directory"
mode: 0755
- name: "install the restic-batch script"
ansible.builtin.copy:
src: "./restic-batch"
dest: "/usr/local/sbin/restic-batch"
mode: 0755
- name: "install the restic-batch service"
ansible.builtin.copy:
src: "./restic-batch.service"
dest: "/etc/systemd/system/restic-batch.service"
mode: 0644
register: backups_restic_restic_batch_service_file
- name: "install the restic-batch timer"
ansible.builtin.copy:
src: "./restic-batch.timer"
dest: "/etc/systemd/system/restic-batch.timer"
mode: 0644
register: backups_restic_restic_batch_timer_file
- name: "systemd daemon reload"
ansible.builtin.systemd:
daemon_reload: true
when:
backups_restic_restic_batch_service_file.changed or
backups_restic_restic_batch_timer_file.changed
- name: "enable and start restic-batch service"
ansible.builtin.systemd:
name: "restic-batch.timer"
enabled: true
state: "started"

View File

@ -1,5 +1,5 @@
[Unit]
Description=Replicate volume data snapshots
Description=Replicate snapshots using syncoid
Documentation=man:syncoid(8)
After=sanoid.service
Before=sanoid-prune.service

View File

@ -38,7 +38,7 @@
state: "directory"
mode: 0755
- name: "install syncoid script"
- name: "install syncoid-batch script"
ansible.builtin.copy:
src: "./syncoid-batch"
dest: "/usr/local/sbin/syncoid-batch"
@ -51,7 +51,7 @@
mode: 0644
register: services_backups_snapshots_syncoid_volume_data_service_file
- name: "snapshots : systemd daemon reload"
- name: "systemd daemon reload"
ansible.builtin.systemd:
daemon_reload: true
when:

View File

@ -4,9 +4,17 @@ argument_specs:
ansible_hostname:
type: "str"
required: true
services_service_name:
type: "str"
required: true
services_data_dataset:
type: "str"
required: true
services_backups_backup_data_dataset:
type: "str"
services_backups_snapshots_services:
type: "dict"
elem: "dict"
required: true
services_backups_restic_services:
type: "dict"
elem: "dict"
required: true

View File

@ -0,0 +1,17 @@
- name: "create restic password file"
ansible.builtin.template:
src: "./restic/restic.password.j2"
dest: "{{ services_backups_restic_services[services_service_name].restic_password_file }}"
mode: 0600
- name: "create aws key file"
ansible.builtin.template:
src: "./restic/restic-aws-keys.yml.j2"
dest: "{{ services_backups_restic_services[services_service_name].aws_keys_file }}"
mode: 0600
- name: "{{ services_service_name }} : restic : configure service restic backups"
ansible.builtin.template:
src: "./restic/restic-volumes-service.yml.j2"
dest: "/etc/restic-batch.d/restic-volumes-{{ services_service_name }}.yml"
mode: 0644

View File

@ -9,7 +9,7 @@
recursive = yes
process_children_only = yes
[{{ services_backups_backup_user_data_dataset }}]
[{{ services_backups_snapshots_services[services_service_name].backup_dataset }}]
use_template = backup
recursive = yes
process_children_only = yes

View File

@ -3,9 +3,12 @@
name: "include"
vars_from: "user"
tags:
- "services:backups:user"
- "services:backups:snapshots"
- "services:backups:{{ services_service_name }}:snapshots"
- "services:{{ services_service_name }}:backups:snapshots"
- "services:backups:restic"
- "services:backups:{{ services_service_name }}:restic"
- "services:{{ services_service_name }}:backups:restic"
- name: "play:services : role:backups : tasks:snapshots"
ansible.builtin.import_tasks: "include/snapshots.yml"
@ -13,3 +16,10 @@
- "services:backups:snapshots"
- "services:backups:{{ services_service_name }}:snapshots"
- "services:{{ services_service_name }}:backups:snapshots"
- name: "play:services : role:backups : tasks:restic"
ansible.builtin.import_tasks: "include/restic.yml"
tags:
- "services:backups:restic"
- "services:backups:{{ services_service_name }}:restic"
- "services:{{ services_service_name }}:backups:restic"

View File

@ -0,0 +1,2 @@
AWS_ACCESS_KEY_ID: {{ services_backups_restic_services[services_service_name].aws_access_key_id }}
AWS_SECRET_ACCESS_KEY: {{ services_backups_restic_services[services_service_name].aws_secret_access_key }}

View File

@ -0,0 +1,7 @@
dataset: {{ services_backups_user_data_dataset }}
aws_bucket_keys_file: {{ services_backups_restic_services[services_service_name].aws_keys_file }}
aws_bucket_endpoint: {{ services_backups_restic_services[services_service_name].aws_bucket_endpoint }}
aws_bucket_prefix: {{ services_backups_restic_services[services_service_name].aws_bucket_prefix }}
restic_password_file: {{ services_backups_restic_services[services_service_name].restic_password_file }}
restic_keep_daily: {{ services_backups_restic_services[services_service_name].restic_keep_daily }}
restic_keep_monthly: {{ services_backups_restic_services[services_service_name].restic_keep_monthly }}

View File

@ -0,0 +1 @@
{{ services_backups_restic_services[services_service_name].restic_password }}

View File

@ -1,4 +1,4 @@
dataset: {{ services_backups_user_data_dataset }}
backup_dataset: {{ services_backups_backup_user_data_dataset }}
recursive: true
skip_parent: true
backup_dataset: {{ services_backups_snapshots_services[services_service_name].backup_dataset }}
recursive: {{ services_backups_snapshots_services[services_service_name].recursive }}
skip_parent: {{ services_backups_snapshots_services[services_service_name].skip_parent }}

View File

@ -1,3 +1 @@
services_backups_user_data_dataset: "{{ services_data_dataset }}/{{ services_service_user_name }}"
services_backups_backup_user_data_dataset: "\
{{ services_backups_backup_data_dataset }}/{{ services_service_user_name }}"

View File

@ -4,12 +4,12 @@ argument_specs:
ansible_hostname:
type: "str"
required: true
services_backups_backup_dataset:
services_backups_snapshots_dataset:
type: "str"
required: true
services_backups_backup_root_dataset:
services_backups_snapshots_root_dataset:
type: "str"
required: true
services_backups_backup_data_dataset:
services_backups_snapshots_data_dataset:
type: "str"
required: true

View File

@ -1,6 +1,6 @@
- name: "create root backup dataset"
community.general.zfs:
name: "{{ services_backups_backup_dataset }}"
name: "{{ services_backups_snapshots_dataset }}"
state: "present"
extra_zfs_properties:
canmount: "off"
@ -8,12 +8,12 @@
- name: "create services backup dataset"
community.general.zfs:
name: "{{ services_backups_backup_root_dataset }}"
name: "{{ services_backups_snapshots_root_dataset }}"
state: "present"
- name: "create services data backup dataset"
community.general.zfs:
name: "{{ services_backups_backup_data_dataset }}"
name: "{{ services_backups_snapshots_data_dataset }}"
state: "present"
extra_zfs_properties:
canmount: "off"

View File

@ -4,7 +4,7 @@ argument_specs:
ansible_hostname:
type: "str"
required: true
services_backups_backup_data_dataset:
services_backups_snapshots_data_dataset:
type: "str"
required: true
services_service_name:

View File

@ -5,7 +5,7 @@
- name: "{{ services_service_name }} : create service backup data dataset"
community.general.zfs:
name: "{{ services_backups_backup_data_dataset }}/{{ services_service_user_name }}"
name: "{{ services_backups_snapshots_data_dataset }}/{{ services_service_user_name }}"
state: "present"
extra_zfs_properties:
canmount: "off"