External volume snapshots

inventory/host_vars/yggdrasil/vars.yml

@@ -56,6 +56,17 @@ system_backups_snapshots_dataset: "hpool/backup"
 system_backups_snapshots_root_dataset: "{{ system_backups_snapshots_dataset }}/the-nine-worlds"
 system_backups_snapshots_data_dataset: "{{ system_backups_snapshots_root_dataset }}/data"
 
+# --------------------------------------------------------------------------------------------------
+# system:external
+# --------------------------------------------------------------------------------------------------
+system_external_dataset: "hpool/external"
+system_external_root_dataset: "{{ system_external_dataset }}/the-nine-worlds"
+system_external_data_dataset: "{{ system_external_root_dataset }}/data"
+
+system_external_directory: "/media/hpool/external"
+system_external_root_directory: "{{ system_external_directory }}/the-nine-worlds"
+system_external_data_directory: "{{ system_external_root_directory }}/data"
+
 # --------------------------------------------------------------------------------------------------
 # vpn
 # --------------------------------------------------------------------------------------------------
@@ -140,6 +151,9 @@ music_user_nextcloud_videos_pswd: "{{ vault_music_user_nextcloud_videos_pswd }}"
 services_root_dataset: "{{ system_var_root_dataset }}"
 services_data_dataset: "{{ system_var_data_dataset }}"
 services_containers_dataset: "{{ system_var_containers_dataset }}"
+services_external_dataset: "{{ system_external_data_dataset }}"
+
+services_external_directory: "{{ system_external_data_directory }}"
 
 services_host_services:
   lrproxy:
@@ -201,6 +215,7 @@ services_backups_snapshots_services: "\
        { service: {
             'user_name': ( 'pod-' ~ service ),
             'data_dataset': ( services_data_dataset ~ '/pod-' ~ service ),
+            'external_dataset': ( services_external_dataset ~ '/pod-' ~ service ),
             'backup_dataset': ( services_backups_snapshots_data_dataset ~ '/pod-' ~ service ),
             'recursive': true,
             'skip_parent': true,

playbooks/roles/backups/snapshots/setup/files/sanoid-templates.conf

@@ -28,6 +28,18 @@
         autosnap = yes
         autoprune = yes
 
+# Template for datasets with external data. Unlike production datasets these are not backed up
+# anywhere. External datasets contain data that can be lost and that generally take up a lot of disk
+# space. Since they don't have a backup, they keep snapshots around for longer than production/
+[template_external]
+        frequently = 0
+        hourly = 36
+        daily = 30
+        monthly = 3
+        yearly = 0
+        autosnap = yes
+        autoprune = yes
+
 # Template for backup datasets which also doubles as a hot spare. The assumption is that these
 # backup production datasets. In addition to three months of snapshots, keep 36 hourly snapshots for
 # quick rollback in case of trouble. No monthly snapshots are kept as that would require the

playbooks/roles/backups/snapshots/user/tasks/main.yml

@@ -10,6 +10,11 @@
               recursive = yes
               process_children_only = yes
 
+      [{{ backups_snapshots_user_external_dataset }}]
+              use_template = external
+              recursive = yes
+              process_children_only = yes
+
       [{{ backups_snapshots_user_backup_dataset }}]
               use_template = backup
               recursive = yes

playbooks/roles/services/datasets/user/tasks/main.yml

@@ -31,12 +31,26 @@
         extra_zfs_properties:
           canmount: "off"
 
+    - name: "{{ services_service_name }} : create external dataset"
+      community.general.zfs:
+        name: "{{ services_external_dataset }}/{{ services_service_user_name }}"
+        state: "present"
+        extra_zfs_properties:
+          canmount: "off"
+
     - name: "{{ services_service_name }} : create volume datasets"
       community.general.zfs:
         name: "{{ services_data_dataset }}/{{ services_service_user_name }}/{{ item.key }}"
         state: "present"
         extra_zfs_properties: "{{ item.value.extra_zfs_properties | default({}) }}"
-      loop: "{{ services_service_volumes | dict2items }}"
+      loop: "{{ services_service_volumes | dict2items | rejectattr('value.external', 'true') }}"
+
+    - name: "{{ services_service_name }} : create external volume datasets"
+      community.general.zfs:
+        name: "{{ services_external_dataset }}/{{ services_service_user_name }}/{{ item.key }}"
+        state: "present"
+        extra_zfs_properties: "{{ item.value.extra_zfs_properties | default({}) }}"
+      loop: "{{ services_service_volumes | dict2items | selectattr('value.external', 'true') }}"
 
 
   when:

playbooks/roles/services/deploy/cloud/vars/main.yml

@@ -10,7 +10,7 @@ services_deploy_pod:
       podman_run_args:
         - "-v {{ services_data_directory }}/pod-cloud/nextcloud/_data:/var/www/html"
         - "-v {{ services_data_directory }}/pod-cloud/data/_data:/var/www/html/data"
-        - "-v {{ services_data_directory }}/pod-cloud/external/_data:/media/external"
+        - "-v {{ services_external_directory }}/pod-cloud/external/_data:/media/external"
         - "--user=0"
         - "--entrypoint=\"/bin/bash\""
       cmd_args:
@@ -24,7 +24,7 @@ services_deploy_pod:
         - "-v ./.config/service/hosts:/etc/hosts:ro"
         - "-v {{ services_data_directory }}/pod-cloud/nextcloud/_data:/var/www/html"
         - "-v {{ services_data_directory }}/pod-cloud/data/_data:/var/www/html/data"
-        - "-v {{ services_data_directory }}/pod-cloud/external/_data:/media/external"
+        - "-v {{ services_external_directory }}/pod-cloud/external/_data:/media/external"
       cmd_args:
         - "/cron.sh"
     nextcloud:
@@ -40,7 +40,7 @@ services_deploy_pod:
         - "-v ./.config/service/hosts:/etc/hosts:ro"
         - "-v {{ services_data_directory }}/pod-cloud/nextcloud/_data:/var/www/html"
         - "-v {{ services_data_directory }}/pod-cloud/data/_data:/var/www/html/data"
-        - "-v {{ services_data_directory }}/pod-cloud/external/_data:/media/external"
+        - "-v {{ services_external_directory }}/pod-cloud/external/_data:/media/external"
         - "-v ./.config/service/database.name:/run/secrets/database.name:ro"
         - "-v ./.config/service/database.user:/run/secrets/database.user:ro"
         - "-v ./.config/service/database.password:/run/secrets/database.password:ro"

playbooks/roles/services/include/vars/user.yml

@@ -2,5 +2,6 @@
 services_service_user_name: "pod-{{ services_service_name }}"
 services_service_user_home: "{{ services_home_directory }}/{{ services_service_user_name }}"
 services_service_user_data: "{{ services_data_directory }}/{{ services_service_user_name }}"
+services_service_user_external: "{{ services_external_directory }}/{{ services_service_user_name }}"
 services_service_user_containers: "\
   {{ services_containers_directory }}/{{ services_service_user_name }}"

playbooks/roles/services/setup/user/tasks/include/directories.yml

@@ -15,8 +15,24 @@
     group: "{{ services_service_user_name }}"
     mode: 0755
 
+- name: "{{ services_service_name }} : directories : create external directory"
+  ansible.builtin.file:
+    path: "{{ services_service_user_external }}"
+    state: "directory"
+    owner: "{{ services_service_user_name }}"
+    group: "{{ services_service_user_name }}"
+    mode: 0755
+
 - name: "{{ services_service_name }} : directories : create volume directories"
   ansible.builtin.include_tasks: "directories/volumes.yml"
+  vars:
+    services_service_volume_root_directory: "\
+      {% if services_service_volume.value.external is defined and \
+            services_service_volume.value.external %}\
+      {{ services_service_user_external }}\
+      {% else %}\
+      {{ services_service_user_data }}\
+      {% endif %}"
   loop: "{{ services_service_volumes | dict2items }}"
   loop_control:
     loop_var: "services_service_volume"

playbooks/roles/services/setup/user/tasks/include/directories/volumes.yml

@@ -2,7 +2,7 @@
 - name: "\
     {{ services_service_name }} : directories : create volume \"{{ services_service_volume.key }}\""
   ansible.builtin.file:
-    path: "{{ services_service_user_data }}/{{ services_service_volume.key }}"
+    path: "{{ services_service_volume_root_directory }}/{{ services_service_volume.key }}"
     state: "directory"
     owner: "{{ services_service_user_name }}"
     group: "{{ services_service_user_name }}"
@@ -12,13 +12,13 @@
     {{ services_service_name }} : directories : check if \"{{ services_service_volume.key }}\" \
     mount exists"
   ansible.builtin.stat:
-    path: "{{ services_service_user_data }}/{{ services_service_volume.key }}/_data"
+    path: "{{ services_service_volume_root_directory }}/{{ services_service_volume.key }}/_data"
   register: services_setup_user_volume_mount
 
 - name: "\
     {{ services_service_name }} : directories : create \"{{ services_service_volume.key }}\" mount"
   ansible.builtin.file:
-    path: "{{ services_service_user_data }}/{{ services_service_volume.key }}/_data"
+    path: "{{ services_service_volume_root_directory }}/{{ services_service_volume.key }}/_data"
     state: "directory"
     owner: "{{ services_service_user_name }}"
     group: "{{ services_service_user_name }}"

playbooks/roles/system/datasets/tasks/include/external.yml

@@ -0,0 +1,20 @@
+---
+- name: "external : create root external dataset"
+  community.general.zfs:
+    name: "{{ system_external_dataset }}"
+    state: "present"
+    extra_zfs_properties:
+      canmount: "off"
+      "com.sun:auto-snapshot": "false"
+
+- name: "external : create external dataset root"
+  community.general.zfs:
+    name: "{{ system_external_root_dataset }}"
+    state: "present"
+
+- name: "external : create external dataset for the-nine-worlds"
+  community.general.zfs:
+    name: "{{ system_external_data_dataset }}"
+    state: "present"
+    extra_zfs_properties:
+      canmount: "off"

playbooks/roles/system/datasets/tasks/main.yml

@@ -8,3 +8,8 @@
   ansible.builtin.import_tasks: "include/backups.yml"
   tags:
     - "system:datasets:backups"
+
+- name: "play:system : role:datasets : tasks:external"
+  ansible.builtin.import_tasks: "include/external.yml"
+  tags:
+    - "system:datasets:external"

playbooks/services.yml

@@ -154,6 +154,8 @@
           {{ services_backups_snapshots_services[services_service_name].user_name }}"
         backups_snapshots_user_data_dataset: "\
           {{ services_backups_snapshots_services[services_service_name].data_dataset }}"
+        backups_snapshots_user_external_dataset: "\
+          {{ services_backups_snapshots_services[services_service_name].external_dataset }}"
         backups_snapshots_user_backup_dataset: "\
           {{ services_backups_snapshots_services[services_service_name].backup_dataset }}"
         backups_snapshots_user_recursive: "\

playbooks/vars/services/volumes.yml

@@ -17,6 +17,7 @@ services_volumes:
     nextcloud:
     data:
     external:
+      external: true
   git:
     data:
     runner: