Better way to exclude volumes from backups

inventory/group_vars/restic/vars.yml

@@ -23,6 +23,7 @@ services_backups_restic_services: "\
        { service: {
             'user_name': ( 'pod-' ~ service ),
             'data_directory': ( services_data_directory ~ '/pod-' ~ service ),
+            'exclude': ( services_host_services[service].restic_exclude | default([]) ),
             'aws_bucket_prefix': ( 'the-nine-worlds---pod-' ~ service ),
        }}
   ) }}\

inventory/host_vars/baldur/vars.yml

@@ -41,6 +41,8 @@ services_host_services:
   cloud:
     address: "{{ vpn_bridge_prefix }}.4"
     restic: true
+    restic_exclude:
+      - "external"
   git:
     address: "{{ vpn_bridge_prefix }}.5"
     tcp: ["{{ services.git.ssh_port }}"]

inventory/host_vars/yggdrasil/vars.yml

@@ -150,6 +150,8 @@ services_host_services:
   cloud:
     address: "{{ vpn_bridge_prefix }}.4"
     restic: true
+    restic_exclude:
+      - "external"
   git:
     address: "{{ vpn_bridge_prefix }}.5"
     tcp: ["{{ services.git.ssh_port }}"]

playbooks/baldur.yml

@@ -77,6 +77,7 @@
         backups_restic_user_use_dataset: "{{ 'zfs' in group_names }}"
         backups_restic_user_data_dataset: "{{ music_user_data_dataset | default('') }}"
         backups_restic_user_data_directory: "{{ music_user_data_directory }}"
+        backups_restic_user_data_exclude_list: []
         backups_restic_user_aws_bucket_prefix: "the-nine-worlds---{{ music_user_name }}"
       tags:
         - "music:backups"
@@ -147,6 +148,8 @@
           {{ services_backups_restic_services[services_service_name].data_dataset | default('') }}"
         backups_restic_user_data_directory: "\
           {{ services_backups_restic_services[services_service_name].data_directory }}"
+        backups_restic_user_data_exclude_list: "\
+          {{ services_backups_restic_services[services_service_name].exclude }}"
         backups_restic_user_aws_bucket_prefix: "\
           {{ services_backups_restic_services[services_service_name].aws_bucket_prefix }}"
       loop: "{{ services_host_services | dict2items | map(attribute='key') }}"

playbooks/music.yml

@@ -40,6 +40,7 @@
         backups_restic_user_use_dataset: "{{ 'zfs' in group_names }}"
         backups_restic_user_data_dataset: "{{ music_user_data_dataset }}"
         backups_restic_user_data_directory: "{{ music_user_data_directory }}"
+        backups_restic_user_data_exclude_list: []
         backups_restic_user_aws_bucket_prefix: "the-nine-worlds---{{ music_user_name }}"
       tags:
         - "music:backups"

playbooks/roles/backups/restic/setup/files/restic-batch

@@ -235,17 +235,18 @@ class DirectoryRepoManager(RepoManager):
         super().__init__(config)
 
     def _get_volumes(self):
-        return self.__get_volume_directories(self._config["directory"])
+        return self.__get_volume_directories(self._config["directory"], self._config["exclude"])
 
-    def __get_volume_directories(self, root_directory):
+    def __get_volume_directories(self, root_directory, exclude_list):
-        return [
+        volumes = []
-            DirectoryVolume(
+        for entry in os.scandir(root_directory):
-                os.path.relpath(entry.path, root_directory),
+            name = os.path.relpath(entry.path, root_directory)
-                self._repo_config,
+            if entry.is_dir() and (name not in exclude_list):
-                os.path.abspath(entry.path),
+                volumes.append(
+                    DirectoryVolume(name, self._repo_config, os.path.abspath(entry.path))
                 )
-            for entry in os.scandir(root_directory) if entry.is_dir()
+
-        ]
+        return volumes
 
 
 class DatasetRepoManager(RepoManager):
@@ -254,25 +255,20 @@ class DatasetRepoManager(RepoManager):
         super().__init__(config)
 
     def _get_volumes(self):
-        return self.__get_volume_datasets(self._config["dataset"])
+        return self.__get_volume_datasets(self._config["dataset"], self._config["exclude"])
 
-    def __get_volume_datasets(self, root_dataset):
+    def __get_volume_datasets(self, root_dataset, exclude_list):
-        zfs_list = subprocess.getoutput(
+        zfs_list = subprocess.getoutput(f"zfs list -H -r {root_dataset} -o name,mountpoint")
-            f"zfs list -H -r {root_dataset} "
-            "-o name,mountpoint,eu.wojciechkozlowski:offsite-snapshot"
-        )
         zfs_list_lines = zfs_list.split('\n')
         zfs_list_lines_items = map(lambda l: l.split(), zfs_list_lines)
-        return [
+
-            DatasetVolume(
+        volumes = []
-                os.path.relpath(dataset, root_dataset),
+        for dataset, mountpoint in zfs_list_lines_items:
-                self._repo_config,
+            name = os.path.relpath(dataset, root_dataset)
-                dataset,
+            if os.path.ismount(mountpoint) and (name not in exclude_list):
-                mountpoint,
+                volumes.append(DatasetVolume(name, self._repo_config, dataset, mountpoint))
-            )
+
-            for dataset, mountpoint, offsite_snapshot in zfs_list_lines_items
+        return volumes
-            if os.path.ismount(mountpoint) and (offsite_snapshot.lower() != "false")
-        ]
 
     def restore(self):
         raise NotImplementedError
@@ -310,6 +306,7 @@ class BatchManager:
                 f"\"dataset\" and \"directory\" cannot be both present in {config_file_path}")
 
         for key in [
+                "exclude",
                 "aws_bucket_keys_file",
                 "aws_bucket_endpoint",
                 "aws_bucket_prefix",

playbooks/roles/backups/restic/user/meta/argument_specs.yml

@@ -17,6 +17,10 @@ argument_specs:
       backups_restic_user_data_directory:
         type: "str"
         required: false
+      backups_restic_user_data_exclude_list:
+        type: "list"
+        elements: "str"
+        required: true
       backups_restic_user_aws_access_key_id:
         type: "str"
         required: true

playbooks/roles/backups/restic/user/templates/volumes.yml

@@ -3,6 +3,7 @@ dataset: {{ backups_restic_user_data_dataset }}
 {% else %}
 directory: {{ backups_restic_user_data_directory }}
 {% endif %}
+exclude: {{ backups_restic_user_data_exclude_list }}
 aws_bucket_keys_file: {{ backups_restic_user_aws_keys_file }}
 aws_bucket_endpoint: {{ backups_restic_user_aws_bucket_endpoint }}
 aws_bucket_prefix: {{ backups_restic_user_aws_bucket_prefix }}

playbooks/roles/services/include/vars/volumes.yml

@@ -17,8 +17,6 @@ services_volumes:
     nextcloud:
     data:
     external:
-      extra_zfs_properties:
-        "eu.wojciechkozlowski:offsite-snapshot=false"
   git:
     data:
   notes:

playbooks/services.yml

@@ -126,6 +126,8 @@
           {{ services_backups_restic_services[services_service_name].data_dataset | default('') }}"
         backups_restic_user_data_directory: "\
           {{ services_backups_restic_services[services_service_name].data_directory }}"
+        backups_restic_user_data_exclude_list: "\
+          {{ services_backups_restic_services[services_service_name].exclude }}"
         backups_restic_user_aws_bucket_prefix: "\
           {{ services_backups_restic_services[services_service_name].aws_bucket_prefix }}"
       loop: "{{ services_host_services | dict2items | map(attribute='key') }}"