From 662a370f720a4a4f0481d622ca2b7198c0b0a5bf Mon Sep 17 00:00:00 2001 From: Wojciech Kozlowski Date: Sun, 16 Jul 2023 00:08:49 +0200 Subject: [PATCH] Better way to exclude volumes from backups --- inventory/group_vars/restic/vars.yml | 1 + inventory/host_vars/baldur/vars.yml | 2 + inventory/host_vars/yggdrasil/vars.yml | 2 + playbooks/baldur.yml | 3 ++ playbooks/music.yml | 1 + .../backups/restic/setup/files/restic-batch | 49 +++++++++---------- .../restic/user/meta/argument_specs.yml | 4 ++ .../backups/restic/user/templates/volumes.yml | 1 + .../roles/services/include/vars/volumes.yml | 2 - playbooks/services.yml | 2 + 10 files changed, 39 insertions(+), 28 deletions(-) diff --git a/inventory/group_vars/restic/vars.yml b/inventory/group_vars/restic/vars.yml index b2b5eb8..58a1027 100644 --- a/inventory/group_vars/restic/vars.yml +++ b/inventory/group_vars/restic/vars.yml @@ -23,6 +23,7 @@ services_backups_restic_services: "\ { service: { 'user_name': ( 'pod-' ~ service ), 'data_directory': ( services_data_directory ~ '/pod-' ~ service ), + 'exclude': ( services_host_services[service].restic_exclude | default([]) ), 'aws_bucket_prefix': ( 'the-nine-worlds---pod-' ~ service ), }} ) }}\ diff --git a/inventory/host_vars/baldur/vars.yml b/inventory/host_vars/baldur/vars.yml index 94323c7..04ccb06 100644 --- a/inventory/host_vars/baldur/vars.yml +++ b/inventory/host_vars/baldur/vars.yml @@ -41,6 +41,8 @@ services_host_services: cloud: address: "{{ vpn_bridge_prefix }}.4" restic: true + restic_exclude: + - "external" git: address: "{{ vpn_bridge_prefix }}.5" tcp: ["{{ services.git.ssh_port }}"] diff --git a/inventory/host_vars/yggdrasil/vars.yml b/inventory/host_vars/yggdrasil/vars.yml index 3e4461e..9191446 100644 --- a/inventory/host_vars/yggdrasil/vars.yml +++ b/inventory/host_vars/yggdrasil/vars.yml @@ -150,6 +150,8 @@ services_host_services: cloud: address: "{{ vpn_bridge_prefix }}.4" restic: true + restic_exclude: + - "external" git: address: "{{ vpn_bridge_prefix }}.5" tcp: ["{{ services.git.ssh_port }}"] diff --git a/playbooks/baldur.yml b/playbooks/baldur.yml index 526726a..69bcd62 100644 --- a/playbooks/baldur.yml +++ b/playbooks/baldur.yml @@ -77,6 +77,7 @@ backups_restic_user_use_dataset: "{{ 'zfs' in group_names }}" backups_restic_user_data_dataset: "{{ music_user_data_dataset | default('') }}" backups_restic_user_data_directory: "{{ music_user_data_directory }}" + backups_restic_user_data_exclude_list: [] backups_restic_user_aws_bucket_prefix: "the-nine-worlds---{{ music_user_name }}" tags: - "music:backups" @@ -147,6 +148,8 @@ {{ services_backups_restic_services[services_service_name].data_dataset | default('') }}" backups_restic_user_data_directory: "\ {{ services_backups_restic_services[services_service_name].data_directory }}" + backups_restic_user_data_exclude_list: "\ + {{ services_backups_restic_services[services_service_name].exclude }}" backups_restic_user_aws_bucket_prefix: "\ {{ services_backups_restic_services[services_service_name].aws_bucket_prefix }}" loop: "{{ services_host_services | dict2items | map(attribute='key') }}" diff --git a/playbooks/music.yml b/playbooks/music.yml index 863f328..d24a1e8 100644 --- a/playbooks/music.yml +++ b/playbooks/music.yml @@ -40,6 +40,7 @@ backups_restic_user_use_dataset: "{{ 'zfs' in group_names }}" backups_restic_user_data_dataset: "{{ music_user_data_dataset }}" backups_restic_user_data_directory: "{{ music_user_data_directory }}" + backups_restic_user_data_exclude_list: [] backups_restic_user_aws_bucket_prefix: "the-nine-worlds---{{ music_user_name }}" tags: - "music:backups" diff --git a/playbooks/roles/backups/restic/setup/files/restic-batch b/playbooks/roles/backups/restic/setup/files/restic-batch index 5c9a3c3..d63d62e 100644 --- a/playbooks/roles/backups/restic/setup/files/restic-batch +++ b/playbooks/roles/backups/restic/setup/files/restic-batch @@ -235,17 +235,18 @@ class DirectoryRepoManager(RepoManager): super().__init__(config) def _get_volumes(self): - return self.__get_volume_directories(self._config["directory"]) + return self.__get_volume_directories(self._config["directory"], self._config["exclude"]) - def __get_volume_directories(self, root_directory): - return [ - DirectoryVolume( - os.path.relpath(entry.path, root_directory), - self._repo_config, - os.path.abspath(entry.path), - ) - for entry in os.scandir(root_directory) if entry.is_dir() - ] + def __get_volume_directories(self, root_directory, exclude_list): + volumes = [] + for entry in os.scandir(root_directory): + name = os.path.relpath(entry.path, root_directory) + if entry.is_dir() and (name not in exclude_list): + volumes.append( + DirectoryVolume(name, self._repo_config, os.path.abspath(entry.path)) + ) + + return volumes class DatasetRepoManager(RepoManager): @@ -254,25 +255,20 @@ class DatasetRepoManager(RepoManager): super().__init__(config) def _get_volumes(self): - return self.__get_volume_datasets(self._config["dataset"]) + return self.__get_volume_datasets(self._config["dataset"], self._config["exclude"]) - def __get_volume_datasets(self, root_dataset): - zfs_list = subprocess.getoutput( - f"zfs list -H -r {root_dataset} " - "-o name,mountpoint,eu.wojciechkozlowski:offsite-snapshot" - ) + def __get_volume_datasets(self, root_dataset, exclude_list): + zfs_list = subprocess.getoutput(f"zfs list -H -r {root_dataset} -o name,mountpoint") zfs_list_lines = zfs_list.split('\n') zfs_list_lines_items = map(lambda l: l.split(), zfs_list_lines) - return [ - DatasetVolume( - os.path.relpath(dataset, root_dataset), - self._repo_config, - dataset, - mountpoint, - ) - for dataset, mountpoint, offsite_snapshot in zfs_list_lines_items - if os.path.ismount(mountpoint) and (offsite_snapshot.lower() != "false") - ] + + volumes = [] + for dataset, mountpoint in zfs_list_lines_items: + name = os.path.relpath(dataset, root_dataset) + if os.path.ismount(mountpoint) and (name not in exclude_list): + volumes.append(DatasetVolume(name, self._repo_config, dataset, mountpoint)) + + return volumes def restore(self): raise NotImplementedError @@ -310,6 +306,7 @@ class BatchManager: f"\"dataset\" and \"directory\" cannot be both present in {config_file_path}") for key in [ + "exclude", "aws_bucket_keys_file", "aws_bucket_endpoint", "aws_bucket_prefix", diff --git a/playbooks/roles/backups/restic/user/meta/argument_specs.yml b/playbooks/roles/backups/restic/user/meta/argument_specs.yml index 1c817eb..0871555 100644 --- a/playbooks/roles/backups/restic/user/meta/argument_specs.yml +++ b/playbooks/roles/backups/restic/user/meta/argument_specs.yml @@ -17,6 +17,10 @@ argument_specs: backups_restic_user_data_directory: type: "str" required: false + backups_restic_user_data_exclude_list: + type: "list" + elements: "str" + required: true backups_restic_user_aws_access_key_id: type: "str" required: true diff --git a/playbooks/roles/backups/restic/user/templates/volumes.yml b/playbooks/roles/backups/restic/user/templates/volumes.yml index 727c877..791198e 100644 --- a/playbooks/roles/backups/restic/user/templates/volumes.yml +++ b/playbooks/roles/backups/restic/user/templates/volumes.yml @@ -3,6 +3,7 @@ dataset: {{ backups_restic_user_data_dataset }} {% else %} directory: {{ backups_restic_user_data_directory }} {% endif %} +exclude: {{ backups_restic_user_data_exclude_list }} aws_bucket_keys_file: {{ backups_restic_user_aws_keys_file }} aws_bucket_endpoint: {{ backups_restic_user_aws_bucket_endpoint }} aws_bucket_prefix: {{ backups_restic_user_aws_bucket_prefix }} diff --git a/playbooks/roles/services/include/vars/volumes.yml b/playbooks/roles/services/include/vars/volumes.yml index 8fa5bf0..5fdac90 100644 --- a/playbooks/roles/services/include/vars/volumes.yml +++ b/playbooks/roles/services/include/vars/volumes.yml @@ -17,8 +17,6 @@ services_volumes: nextcloud: data: external: - extra_zfs_properties: - "eu.wojciechkozlowski:offsite-snapshot=false" git: data: notes: diff --git a/playbooks/services.yml b/playbooks/services.yml index e5ded9f..c576c6e 100644 --- a/playbooks/services.yml +++ b/playbooks/services.yml @@ -126,6 +126,8 @@ {{ services_backups_restic_services[services_service_name].data_dataset | default('') }}" backups_restic_user_data_directory: "\ {{ services_backups_restic_services[services_service_name].data_directory }}" + backups_restic_user_data_exclude_list: "\ + {{ services_backups_restic_services[services_service_name].exclude }}" backups_restic_user_aws_bucket_prefix: "\ {{ services_backups_restic_services[services_service_name].aws_bucket_prefix }}" loop: "{{ services_host_services | dict2items | map(attribute='key') }}"