Better way to exclude volumes from backups

This commit is contained in:
Wojciech Kozlowski 2023-07-16 00:08:49 +02:00
parent 6fb5d83d4c
commit 662a370f72
10 changed files with 39 additions and 28 deletions

View File

@ -23,6 +23,7 @@ services_backups_restic_services: "\
{ service: { { service: {
'user_name': ( 'pod-' ~ service ), 'user_name': ( 'pod-' ~ service ),
'data_directory': ( services_data_directory ~ '/pod-' ~ service ), 'data_directory': ( services_data_directory ~ '/pod-' ~ service ),
'exclude': ( services_host_services[service].restic_exclude | default([]) ),
'aws_bucket_prefix': ( 'the-nine-worlds---pod-' ~ service ), 'aws_bucket_prefix': ( 'the-nine-worlds---pod-' ~ service ),
}} }}
) }}\ ) }}\

View File

@ -41,6 +41,8 @@ services_host_services:
cloud: cloud:
address: "{{ vpn_bridge_prefix }}.4" address: "{{ vpn_bridge_prefix }}.4"
restic: true restic: true
restic_exclude:
- "external"
git: git:
address: "{{ vpn_bridge_prefix }}.5" address: "{{ vpn_bridge_prefix }}.5"
tcp: ["{{ services.git.ssh_port }}"] tcp: ["{{ services.git.ssh_port }}"]

View File

@ -150,6 +150,8 @@ services_host_services:
cloud: cloud:
address: "{{ vpn_bridge_prefix }}.4" address: "{{ vpn_bridge_prefix }}.4"
restic: true restic: true
restic_exclude:
- "external"
git: git:
address: "{{ vpn_bridge_prefix }}.5" address: "{{ vpn_bridge_prefix }}.5"
tcp: ["{{ services.git.ssh_port }}"] tcp: ["{{ services.git.ssh_port }}"]

View File

@ -77,6 +77,7 @@
backups_restic_user_use_dataset: "{{ 'zfs' in group_names }}" backups_restic_user_use_dataset: "{{ 'zfs' in group_names }}"
backups_restic_user_data_dataset: "{{ music_user_data_dataset | default('') }}" backups_restic_user_data_dataset: "{{ music_user_data_dataset | default('') }}"
backups_restic_user_data_directory: "{{ music_user_data_directory }}" backups_restic_user_data_directory: "{{ music_user_data_directory }}"
backups_restic_user_data_exclude_list: []
backups_restic_user_aws_bucket_prefix: "the-nine-worlds---{{ music_user_name }}" backups_restic_user_aws_bucket_prefix: "the-nine-worlds---{{ music_user_name }}"
tags: tags:
- "music:backups" - "music:backups"
@ -147,6 +148,8 @@
{{ services_backups_restic_services[services_service_name].data_dataset | default('') }}" {{ services_backups_restic_services[services_service_name].data_dataset | default('') }}"
backups_restic_user_data_directory: "\ backups_restic_user_data_directory: "\
{{ services_backups_restic_services[services_service_name].data_directory }}" {{ services_backups_restic_services[services_service_name].data_directory }}"
backups_restic_user_data_exclude_list: "\
{{ services_backups_restic_services[services_service_name].exclude }}"
backups_restic_user_aws_bucket_prefix: "\ backups_restic_user_aws_bucket_prefix: "\
{{ services_backups_restic_services[services_service_name].aws_bucket_prefix }}" {{ services_backups_restic_services[services_service_name].aws_bucket_prefix }}"
loop: "{{ services_host_services | dict2items | map(attribute='key') }}" loop: "{{ services_host_services | dict2items | map(attribute='key') }}"

View File

@ -40,6 +40,7 @@
backups_restic_user_use_dataset: "{{ 'zfs' in group_names }}" backups_restic_user_use_dataset: "{{ 'zfs' in group_names }}"
backups_restic_user_data_dataset: "{{ music_user_data_dataset }}" backups_restic_user_data_dataset: "{{ music_user_data_dataset }}"
backups_restic_user_data_directory: "{{ music_user_data_directory }}" backups_restic_user_data_directory: "{{ music_user_data_directory }}"
backups_restic_user_data_exclude_list: []
backups_restic_user_aws_bucket_prefix: "the-nine-worlds---{{ music_user_name }}" backups_restic_user_aws_bucket_prefix: "the-nine-worlds---{{ music_user_name }}"
tags: tags:
- "music:backups" - "music:backups"

View File

@ -235,17 +235,18 @@ class DirectoryRepoManager(RepoManager):
super().__init__(config) super().__init__(config)
def _get_volumes(self): def _get_volumes(self):
return self.__get_volume_directories(self._config["directory"]) return self.__get_volume_directories(self._config["directory"], self._config["exclude"])
def __get_volume_directories(self, root_directory): def __get_volume_directories(self, root_directory, exclude_list):
return [ volumes = []
DirectoryVolume( for entry in os.scandir(root_directory):
os.path.relpath(entry.path, root_directory), name = os.path.relpath(entry.path, root_directory)
self._repo_config, if entry.is_dir() and (name not in exclude_list):
os.path.abspath(entry.path), volumes.append(
) DirectoryVolume(name, self._repo_config, os.path.abspath(entry.path))
for entry in os.scandir(root_directory) if entry.is_dir() )
]
return volumes
class DatasetRepoManager(RepoManager): class DatasetRepoManager(RepoManager):
@ -254,25 +255,20 @@ class DatasetRepoManager(RepoManager):
super().__init__(config) super().__init__(config)
def _get_volumes(self): def _get_volumes(self):
return self.__get_volume_datasets(self._config["dataset"]) return self.__get_volume_datasets(self._config["dataset"], self._config["exclude"])
def __get_volume_datasets(self, root_dataset): def __get_volume_datasets(self, root_dataset, exclude_list):
zfs_list = subprocess.getoutput( zfs_list = subprocess.getoutput(f"zfs list -H -r {root_dataset} -o name,mountpoint")
f"zfs list -H -r {root_dataset} "
"-o name,mountpoint,eu.wojciechkozlowski:offsite-snapshot"
)
zfs_list_lines = zfs_list.split('\n') zfs_list_lines = zfs_list.split('\n')
zfs_list_lines_items = map(lambda l: l.split(), zfs_list_lines) zfs_list_lines_items = map(lambda l: l.split(), zfs_list_lines)
return [
DatasetVolume( volumes = []
os.path.relpath(dataset, root_dataset), for dataset, mountpoint in zfs_list_lines_items:
self._repo_config, name = os.path.relpath(dataset, root_dataset)
dataset, if os.path.ismount(mountpoint) and (name not in exclude_list):
mountpoint, volumes.append(DatasetVolume(name, self._repo_config, dataset, mountpoint))
)
for dataset, mountpoint, offsite_snapshot in zfs_list_lines_items return volumes
if os.path.ismount(mountpoint) and (offsite_snapshot.lower() != "false")
]
def restore(self): def restore(self):
raise NotImplementedError raise NotImplementedError
@ -310,6 +306,7 @@ class BatchManager:
f"\"dataset\" and \"directory\" cannot be both present in {config_file_path}") f"\"dataset\" and \"directory\" cannot be both present in {config_file_path}")
for key in [ for key in [
"exclude",
"aws_bucket_keys_file", "aws_bucket_keys_file",
"aws_bucket_endpoint", "aws_bucket_endpoint",
"aws_bucket_prefix", "aws_bucket_prefix",

View File

@ -17,6 +17,10 @@ argument_specs:
backups_restic_user_data_directory: backups_restic_user_data_directory:
type: "str" type: "str"
required: false required: false
backups_restic_user_data_exclude_list:
type: "list"
elements: "str"
required: true
backups_restic_user_aws_access_key_id: backups_restic_user_aws_access_key_id:
type: "str" type: "str"
required: true required: true

View File

@ -3,6 +3,7 @@ dataset: {{ backups_restic_user_data_dataset }}
{% else %} {% else %}
directory: {{ backups_restic_user_data_directory }} directory: {{ backups_restic_user_data_directory }}
{% endif %} {% endif %}
exclude: {{ backups_restic_user_data_exclude_list }}
aws_bucket_keys_file: {{ backups_restic_user_aws_keys_file }} aws_bucket_keys_file: {{ backups_restic_user_aws_keys_file }}
aws_bucket_endpoint: {{ backups_restic_user_aws_bucket_endpoint }} aws_bucket_endpoint: {{ backups_restic_user_aws_bucket_endpoint }}
aws_bucket_prefix: {{ backups_restic_user_aws_bucket_prefix }} aws_bucket_prefix: {{ backups_restic_user_aws_bucket_prefix }}

View File

@ -17,8 +17,6 @@ services_volumes:
nextcloud: nextcloud:
data: data:
external: external:
extra_zfs_properties:
"eu.wojciechkozlowski:offsite-snapshot=false"
git: git:
data: data:
notes: notes:

View File

@ -126,6 +126,8 @@
{{ services_backups_restic_services[services_service_name].data_dataset | default('') }}" {{ services_backups_restic_services[services_service_name].data_dataset | default('') }}"
backups_restic_user_data_directory: "\ backups_restic_user_data_directory: "\
{{ services_backups_restic_services[services_service_name].data_directory }}" {{ services_backups_restic_services[services_service_name].data_directory }}"
backups_restic_user_data_exclude_list: "\
{{ services_backups_restic_services[services_service_name].exclude }}"
backups_restic_user_aws_bucket_prefix: "\ backups_restic_user_aws_bucket_prefix: "\
{{ services_backups_restic_services[services_service_name].aws_bucket_prefix }}" {{ services_backups_restic_services[services_service_name].aws_bucket_prefix }}"
loop: "{{ services_host_services | dict2items | map(attribute='key') }}" loop: "{{ services_host_services | dict2items | map(attribute='key') }}"