the-nine-worlds/ansible-edda
the-nine-worlds/ansible-edda
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Better way to exclude volumes from backups

Browse Source
This commit is contained in:
Wojciech Kozlowski 2023-07-16 00:08:49 +02:00
parent 6fb5d83d4c
commit 662a370f72
10 changed files with 39 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
inventory/group_vars/restic/vars.yml
View File

@ -23,6 +23,7 @@ services_backups_restic_services: "\
{ service: {
'user_name': ( 'pod-' ~ service ),
'data_directory': ( services_data_directory ~ '/pod-' ~ service ),
'exclude': ( services_host_services[service].restic_exclude | default([]) ),
'aws_bucket_prefix': ( 'the-nine-worlds---pod-' ~ service ),
}}
) }}\

2
inventory/host_vars/baldur/vars.yml
View File

@ -41,6 +41,8 @@ services_host_services:
cloud:
address: "{{ vpn_bridge_prefix }}.4"
restic: true
restic_exclude:
- "external"
git:
address: "{{ vpn_bridge_prefix }}.5"
tcp: ["{{ services.git.ssh_port }}"]

2
inventory/host_vars/yggdrasil/vars.yml
View File

@ -150,6 +150,8 @@ services_host_services:
cloud:
address: "{{ vpn_bridge_prefix }}.4"
restic: true
restic_exclude:
- "external"
git:
address: "{{ vpn_bridge_prefix }}.5"
tcp: ["{{ services.git.ssh_port }}"]

3
playbooks/baldur.yml
View File

@ -77,6 +77,7 @@
backups_restic_user_use_dataset: "{{ 'zfs' in group_names }}"
backups_restic_user_data_dataset: "{{ music_user_data_dataset | default('') }}"
backups_restic_user_data_directory: "{{ music_user_data_directory }}"
backups_restic_user_data_exclude_list: []
backups_restic_user_aws_bucket_prefix: "the-nine-worlds---{{ music_user_name }}"
tags:
- "music:backups"
@ -147,6 +148,8 @@
{{ services_backups_restic_services[services_service_name].data_dataset | default('') }}"
backups_restic_user_data_directory: "\
{{ services_backups_restic_services[services_service_name].data_directory }}"
backups_restic_user_data_exclude_list: "\
{{ services_backups_restic_services[services_service_name].exclude }}"
backups_restic_user_aws_bucket_prefix: "\
{{ services_backups_restic_services[services_service_name].aws_bucket_prefix }}"
loop: "{{ services_host_services | dict2items | map(attribute='key') }}"

1
playbooks/music.yml
View File

@ -40,6 +40,7 @@
backups_restic_user_use_dataset: "{{ 'zfs' in group_names }}"
backups_restic_user_data_dataset: "{{ music_user_data_dataset }}"
backups_restic_user_data_directory: "{{ music_user_data_directory }}"
backups_restic_user_data_exclude_list: []
backups_restic_user_aws_bucket_prefix: "the-nine-worlds---{{ music_user_name }}"
tags:
- "music:backups"

47
playbooks/roles/backups/restic/setup/files/restic-batch
View File

@ -235,17 +235,18 @@ class DirectoryRepoManager(RepoManager):
super().__init__(config)
def _get_volumes(self):
return self.__get_volume_directories(self._config["directory"])
return self.__get_volume_directories(self._config["directory"], self._config["exclude"])
def __get_volume_directories(self, root_directory):
return [
DirectoryVolume(
os.path.relpath(entry.path, root_directory),
self._repo_config,
os.path.abspath(entry.path),
def __get_volume_directories(self, root_directory, exclude_list):
volumes = []
for entry in os.scandir(root_directory):
name = os.path.relpath(entry.path, root_directory)
if entry.is_dir() and (name not in exclude_list):
volumes.append(
DirectoryVolume(name, self._repo_config, os.path.abspath(entry.path))
)
for entry in os.scandir(root_directory) if entry.is_dir()
]
return volumes
class DatasetRepoManager(RepoManager):
@ -254,25 +255,20 @@ class DatasetRepoManager(RepoManager):
super().__init__(config)
def _get_volumes(self):
return self.__get_volume_datasets(self._config["dataset"])
return self.__get_volume_datasets(self._config["dataset"], self._config["exclude"])
def __get_volume_datasets(self, root_dataset):
zfs_list = subprocess.getoutput(
f"zfs list -H -r {root_dataset} "
"-o name,mountpoint,eu.wojciechkozlowski:offsite-snapshot"
)
def __get_volume_datasets(self, root_dataset, exclude_list):
zfs_list = subprocess.getoutput(f"zfs list -H -r {root_dataset} -o name,mountpoint")
zfs_list_lines = zfs_list.split('\n')
zfs_list_lines_items = map(lambda l: l.split(), zfs_list_lines)
return [
DatasetVolume(
os.path.relpath(dataset, root_dataset),
self._repo_config,
dataset,
mountpoint,
)
for dataset, mountpoint, offsite_snapshot in zfs_list_lines_items
if os.path.ismount(mountpoint) and (offsite_snapshot.lower() != "false")
]
volumes = []
for dataset, mountpoint in zfs_list_lines_items:
name = os.path.relpath(dataset, root_dataset)
if os.path.ismount(mountpoint) and (name not in exclude_list):
volumes.append(DatasetVolume(name, self._repo_config, dataset, mountpoint))
return volumes
def restore(self):
raise NotImplementedError
@ -310,6 +306,7 @@ class BatchManager:
f"\"dataset\" and \"directory\" cannot be both present in {config_file_path}")
for key in [
"exclude",
"aws_bucket_keys_file",
"aws_bucket_endpoint",
"aws_bucket_prefix",

4
playbooks/roles/backups/restic/user/meta/argument_specs.yml
View File

@ -17,6 +17,10 @@ argument_specs:
backups_restic_user_data_directory:
type: "str"
required: false
backups_restic_user_data_exclude_list:
type: "list"
elements: "str"
required: true
backups_restic_user_aws_access_key_id:
type: "str"
required: true

1
playbooks/roles/backups/restic/user/templates/volumes.yml
View File

@ -3,6 +3,7 @@ dataset: {{ backups_restic_user_data_dataset }}
{% else %}
directory: {{ backups_restic_user_data_directory }}
{% endif %}
exclude: {{ backups_restic_user_data_exclude_list }}
aws_bucket_keys_file: {{ backups_restic_user_aws_keys_file }}
aws_bucket_endpoint: {{ backups_restic_user_aws_bucket_endpoint }}
aws_bucket_prefix: {{ backups_restic_user_aws_bucket_prefix }}

2
playbooks/roles/services/include/vars/volumes.yml
View File

@ -17,8 +17,6 @@ services_volumes:
nextcloud:
data:
external:
extra_zfs_properties:
"eu.wojciechkozlowski:offsite-snapshot=false"
git:
data:
notes:

2
playbooks/services.yml
View File

@ -126,6 +126,8 @@
{{ services_backups_restic_services[services_service_name].data_dataset | default('') }}"
backups_restic_user_data_directory: "\
{{ services_backups_restic_services[services_service_name].data_directory }}"
backups_restic_user_data_exclude_list: "\
{{ services_backups_restic_services[services_service_name].exclude }}"
backups_restic_user_aws_bucket_prefix: "\
{{ services_backups_restic_services[services_service_name].aws_bucket_prefix }}"
loop: "{{ services_host_services | dict2items | map(attribute='key') }}"