Decouple backups from services

inventory/group_vars/restic/vars.yml

@@ -1,27 +1,30 @@
 ---
+# --------------------------------------------------------------------------------------------------
+# backups:restic
+# --------------------------------------------------------------------------------------------------
+backups_restic_user_aws_access_key_id: "{{ vault_backups_restic_user_aws_access_key_id }}"
+backups_restic_user_aws_secret_access_key: "\
+  {{ vault_backups_restic_user_aws_secret_access_key }}"
+backups_restic_user_aws_keys_file: "/etc/restic-aws-keys.yml"
+backups_restic_user_aws_bucket_endpoint: "\
+  {{ vault_backups_restic_user_aws_bucket_endpoint }}"
+backups_restic_user_restic_password: "{{ vault_backups_restic_user_restic_password }}"
+backups_restic_user_restic_password_file: "/etc/restic.password"
+backups_restic_user_restic_keep_daily: 30
+backups_restic_user_restic_keep_monthly: 3
+
 # --------------------------------------------------------------------------------------------------
 # services:backups
 # --------------------------------------------------------------------------------------------------
-services_backups_restic_restic_password: "{{ vault_services_backups_restic_restic_password }}"
-services_backups_restic_aws_access_key_id: "{{ vault_services_backups_restic_aws_access_key_id }}"
-services_backups_restic_aws_secret_access_key: "\
-  {{ vault_services_backups_restic_aws_secret_access_key }}"
-services_backups_restic_aws_bucket_endpoint: "\
-  {{ vault_services_backups_restic_aws_bucket_endpoint }}"
 services_backups_restic_services: "\
   {% set services_backups_restic_service = {} %}\
   {% for service in services_host_services.keys() %}\
   {{ services_backups_restic_service.update(
        { service: {
-            'aws_access_key_id': services_backups_restic_aws_access_key_id,
+            'user_name': ( 'pod-' ~ service ),
-            'aws_secret_access_key': services_backups_restic_aws_secret_access_key,
+            'data_dataset': ( services_data_dataset ~ '/pod-' ~ service ),
-            'aws_keys_file': '/etc/restic-aws-keys.yml',
+            'data_directory': ( services_data_directory ~ '/pod-' ~ service ),
-            'aws_bucket_endpoint': services_backups_restic_aws_bucket_endpoint,
             'aws_bucket_prefix': ( 'the-nine-worlds---pod-' ~ service ),
-            'restic_password': services_backups_restic_restic_password,
-            'restic_password_file': '/etc/restic.password',
-            'restic_keep_daily': 30,
-            'restic_keep_monthly': 3,
        }}
   ) }}\
   {% endfor %}\

inventory/host_vars/yggdrasil/vars.yml

@@ -133,6 +133,8 @@ services_backups_snapshots_services: "\
   {% for service in services_host_services.keys() %}\
   {{ services_backups_snapshots_service.update(
        { service: {
+            'user_name': ( 'pod-' ~ service ),
+            'data_dataset': ( services_data_dataset ~ '/pod-' ~ service ),
             'backup_dataset': ( services_backups_snapshots_data_dataset ~ '/pod-' ~ service ),
             'recursive': true,
             'skip_parent': true,

playbooks/roles/backups/restic/user/meta/argument_specs.yml

@@ -0,0 +1,46 @@
+---
+argument_specs:
+  main:
+    options:
+      ansible_hostname:
+        type: "str"
+        required: true
+      backups_restic_user_name:
+        type: "str"
+        required: true
+      backups_restic_user_use_dataset:
+        type: "bool"
+        required: true
+      backups_restic_user_data_dataset:
+        type: "str"
+        required: false
+      backups_restic_user_data_directory:
+        type: "str"
+        required: false
+      backups_restic_user_aws_access_key_id:
+        type: "str"
+        required: true
+      backups_restic_user_aws_secret_access_key:
+        type: "str"
+        required: true
+      backups_restic_user_aws_keys_file:
+        type: "str"
+        required: true
+      backups_restic_user_aws_bucket_endpoint:
+        type: "str"
+        required: true
+      backups_restic_user_aws_bucket_prefix:
+        type: "str"
+        required: true
+      backups_restic_user_restic_password:
+        type: "str"
+        required: true
+      backups_restic_user_restic_password_file:
+        type: "str"
+        required: true
+      backups_restic_user_restic_keep_daily:
+        type: "int"
+        required: true
+      backups_restic_user_restic_keep_monthly:
+        type: "int"
+        required: true

playbooks/roles/backups/restic/user/tasks/main.yml

@@ -0,0 +1,18 @@
+---
+- name: "create restic password file"
+  ansible.builtin.template:
+    src: "./restic.password.j2"
+    dest: "{{ backups_restic_user_restic_password_file }}"
+    mode: 0600
+
+- name: "create aws key file"
+  ansible.builtin.template:
+    src: "./restic-aws-keys.yml.j2"
+    dest: "{{ backups_restic_user_aws_keys_file }}"
+    mode: 0600
+
+- name: "configure service restic backups"
+  ansible.builtin.template:
+    src: "./volumes.yml.j2"
+    dest: "/etc/restic-batch.d/{{ backups_restic_user_name }}.yml"
+    mode: 0644

playbooks/roles/backups/restic/user/templates/restic-aws-keys.yml.j2

@@ -0,0 +1,2 @@
+AWS_ACCESS_KEY_ID: {{ backups_restic_user_aws_access_key_id }}
+AWS_SECRET_ACCESS_KEY: {{ backups_restic_user_aws_secret_access_key }}

playbooks/roles/backups/restic/user/templates/restic.password.j2

@@ -0,0 +1 @@
+{{ backups_restic_user_restic_password }}

playbooks/roles/backups/restic/user/templates/volumes.yml.j2

@@ -0,0 +1,11 @@
+{% if backups_restic_user_use_dataset %}
+dataset: {{ backups_restic_user_data_dataset }}
+{% else %}
+directory: {{ backups_restic_user_data_directory }}
+{% endif %}
+aws_bucket_keys_file: {{ backups_restic_user_aws_keys_file }}
+aws_bucket_endpoint: {{ backups_restic_user_aws_bucket_endpoint }}
+aws_bucket_prefix: {{ backups_restic_user_aws_bucket_prefix }}
+restic_password_file: {{ backups_restic_user_restic_password_file }}
+restic_keep_daily: {{ backups_restic_user_restic_keep_daily }}
+restic_keep_monthly: {{ backups_restic_user_restic_keep_monthly }}

playbooks/roles/backups/snapshots/user/meta/argument_specs.yml

@@ -0,0 +1,22 @@
+---
+argument_specs:
+  main:
+    options:
+      ansible_hostname:
+        type: "str"
+        required: true
+      backups_snapshots_user_name:
+        type: "str"
+        required: true
+      backups_snapshots_user_data_dataset:
+        type: "str"
+        required: true
+      backups_snapshots_user_backup_dataset:
+        type: "str"
+        required: true
+      backups_snapshots_user_recursive:
+        type: "bool"
+        required: true
+      backups_snapshots_user_skip_parent:
+        type: "bool"
+        required: true

playbooks/roles/backups/snapshots/user/tasks/main.yml

@@ -0,0 +1,22 @@
+---
+- name: "configure service sanoid snapshots"
+  ansible.builtin.blockinfile:
+    path: "/etc/sanoid/sanoid.conf"
+    insertbefore: "# BEGIN ANSIBLE MANAGED BLOCK TEMPLATES #"
+    marker: "# {mark} ANSIBLE MANAGED BLOCK USER {{ backups_snapshots_user_name }} #"
+    block: |
+      [{{ backups_snapshots_user_data_dataset }}]
+              use_template = production
+              recursive = yes
+              process_children_only = yes
+
+      [{{ backups_snapshots_user_backup_dataset }}]
+              use_template = backup
+              recursive = yes
+              process_children_only = yes
+
+- name: "configure service syncoid snapshots"
+  ansible.builtin.template:
+    src: "./volumes.yml.j2"
+    dest: "/etc/syncoid-batch.d/{{ backups_snapshots_user_name }}.yml"
+    mode: 0644

playbooks/roles/backups/snapshots/user/templates/volumes.yml.j2

@@ -0,0 +1,4 @@
+dataset: {{ backups_snapshots_user_data_dataset }}
+backup_dataset: {{ backups_snapshots_user_backup_dataset }}
+recursive: {{ backups_snapshots_user_recursive }}
+skip_parent: {{ backups_snapshots_user_skip_parent }}

playbooks/roles/services/backups/include/vars/datasets.yml

@@ -1,4 +0,0 @@
----
-services_backups_user_data_dataset: "{{ services_data_dataset }}/{{ services_service_user_name }}"
-services_backups_user_data_directory: "\
-  {{ services_data_directory }}/{{ services_service_user_name }}"

playbooks/roles/services/backups/restic/meta/argument_specs.yml

@@ -1,20 +0,0 @@
----
-argument_specs:
-  main:
-    options:
-      ansible_hostname:
-        type: "str"
-        required: true
-      services_service_name:
-        type: "str"
-        required: true
-      services_data_dataset:
-        type: "str"
-        required: false
-      services_data_directory:
-        type: "str"
-        required: false
-      services_backups_restic_services:
-        type: "dict"
-        elem: "dict"
-        required: true

playbooks/roles/services/backups/restic/tasks/main.yml

@@ -1,28 +0,0 @@
----
-- name: "{{ services_service_name }} : tasks:vars"
-  ansible.builtin.import_role:
-    name: "services/include"
-    vars_from: "user"
-
-- name: "{{ services_service_name }} : tasks:vars"
-  ansible.builtin.import_role:
-    name: "services/backups/include"
-    vars_from: "datasets"
-
-- name: "{{ services_service_name }} : create restic password file"
-  ansible.builtin.template:
-    src: "./restic.password.j2"
-    dest: "{{ services_backups_restic_services[services_service_name].restic_password_file }}"
-    mode: 0600
-
-- name: "{{ services_service_name }} : create aws key file"
-  ansible.builtin.template:
-    src: "./restic-aws-keys.yml.j2"
-    dest: "{{ services_backups_restic_services[services_service_name].aws_keys_file }}"
-    mode: 0600
-
-- name: "{{ services_service_name }} : configure service restic backups"
-  ansible.builtin.template:
-    src: "./volumes.yml.j2"
-    dest: "/etc/restic-batch.d/{{ services_service_user_name }}.yml"
-    mode: 0644

playbooks/roles/services/backups/restic/templates/restic-aws-keys.yml.j2

@@ -1,2 +0,0 @@
-AWS_ACCESS_KEY_ID: {{ services_backups_restic_services[services_service_name].aws_access_key_id }}
-AWS_SECRET_ACCESS_KEY: {{ services_backups_restic_services[services_service_name].aws_secret_access_key }}

playbooks/roles/services/backups/restic/templates/restic.password.j2

@@ -1 +0,0 @@
-{{ services_backups_restic_services[services_service_name].restic_password }}

playbooks/roles/services/backups/restic/templates/volumes.yml.j2

@@ -1,11 +0,0 @@
-{% if services_data_dataset is defined %}
-dataset: {{ services_backups_user_data_dataset }}
-{% else %}
-directory: {{ services_backups_user_data_directory }}
-{% endif %}
-aws_bucket_keys_file: {{ services_backups_restic_services[services_service_name].aws_keys_file }}
-aws_bucket_endpoint: {{ services_backups_restic_services[services_service_name].aws_bucket_endpoint }}
-aws_bucket_prefix: {{ services_backups_restic_services[services_service_name].aws_bucket_prefix }}
-restic_password_file: {{ services_backups_restic_services[services_service_name].restic_password_file }}
-restic_keep_daily: {{ services_backups_restic_services[services_service_name].restic_keep_daily }}
-restic_keep_monthly: {{ services_backups_restic_services[services_service_name].restic_keep_monthly }}

playbooks/roles/services/backups/snapshots/meta/argument_specs.yml

@@ -1,17 +0,0 @@
----
-argument_specs:
-  main:
-    options:
-      ansible_hostname:
-        type: "str"
-        required: true
-      services_service_name:
-        type: "str"
-        required: true
-      services_data_dataset:
-        type: "str"
-        required: true
-      services_backups_snapshots_services:
-        type: "dict"
-        elem: "dict"
-        required: true

playbooks/roles/services/backups/snapshots/tasks/main.yml

@@ -1,32 +0,0 @@
----
-- name: "{{ services_service_name }} : tasks:vars"
-  ansible.builtin.import_role:
-    name: "services/include"
-    vars_from: "user"
-
-- name: "{{ services_service_name }} : tasks:vars"
-  ansible.builtin.import_role:
-    name: "services/backups/include"
-    vars_from: "datasets"
-
-- name: "{{ services_service_name }} : configure service sanoid snapshots"
-  ansible.builtin.blockinfile:
-    path: "/etc/sanoid/sanoid.conf"
-    insertbefore: "# BEGIN ANSIBLE MANAGED BLOCK TEMPLATES #"
-    marker: "# {mark} ANSIBLE MANAGED BLOCK SERVICE {{ services_service_name }} #"
-    block: |
-      [{{ services_backups_user_data_dataset }}]
-              use_template = production
-              recursive = yes
-              process_children_only = yes
-
-      [{{ services_backups_snapshots_services[services_service_name].backup_dataset }}]
-              use_template = backup
-              recursive = yes
-              process_children_only = yes
-
-- name: "{{ services_service_name }} : configure service syncoid snapshots"
-  ansible.builtin.template:
-    src: "./volumes.yml.j2"
-    dest: "/etc/syncoid-batch.d/{{ services_service_user_name }}.yml"
-    mode: 0644

playbooks/roles/services/backups/snapshots/templates/volumes.yml.j2

@@ -1,4 +0,0 @@
-dataset: {{ services_backups_user_data_dataset }}
-backup_dataset: {{ services_backups_snapshots_services[services_service_name].backup_dataset }}
-recursive: {{ services_backups_snapshots_services[services_service_name].recursive }}
-skip_parent: {{ services_backups_snapshots_services[services_service_name].skip_parent }}

playbooks/services.yml

@@ -79,14 +79,26 @@
 
     - name: "backups : snapshots"
       ansible.builtin.include_role:
-        name: "services/backups/snapshots"
+        name: "backups/snapshots/user"
         apply:
           tags:
             - "services:{{ services_service_name }}"
             - "services:backups"
             - "services:backups:snapshots"
-            - "services:backups:snapshots:{{ services_service_name }}"
+            - "services:backups:snapshots:user"
-            - "services:{{ services_service_name }}:backups:snapshots"
+            - "services:backups:snapshots:user:{{ services_service_name }}"
+            - "services:{{ services_service_name }}:backups:snapshots:user"
+      vars:
+        backups_snapshots_user_name: "\
+          {{ services_backups_snapshots_services[services_service_name].user_name }}"
+        backups_snapshots_user_data_dataset: "\
+          {{ services_backups_snapshots_services[services_service_name].data_dataset }}"
+        backups_snapshots_user_backup_dataset: "\
+          {{ services_backups_snapshots_services[services_service_name].backup_dataset }}"
+        backups_snapshots_user_recursive: "\
+          {{ services_backups_snapshots_services[services_service_name].recursive }}"
+        backups_snapshots_user_skip_parent: "\
+          {{ services_backups_snapshots_services[services_service_name].skip_parent }}"
       loop: "{{ services_host_services | dict2items | map(attribute='key') }}"
       loop_control:
         loop_var: "services_service_name"
@@ -98,14 +110,24 @@
   tasks:
     - name: "backups : restic"
       ansible.builtin.include_role:
-        name: "services/backups/restic"
+        name: "backups/restic/user"
         apply:
           tags:
             - "services:{{ services_service_name }}"
             - "services:backups"
-            - "services:backups:restic"
+            - "services:backups:restic:user"
-            - "services:backups:restic:{{ services_service_name }}"
+            - "services:backups:restic:user:{{ services_service_name }}"
-            - "services:{{ services_service_name }}:backups:restic"
+            - "services:{{ services_service_name }}:backups:restic:user"
+      vars:
+        backups_restic_user_name: "\
+          {{ services_backups_restic_services[services_service_name].user_name }}"
+        backups_restic_user_use_dataset: "{{ 'zfs' in group_names }}"
+        backups_restic_user_data_dataset: "\
+          {{ services_backups_restic_services[services_service_name].data_dataset }}"
+        backups_restic_user_data_directory: "\
+          {{ services_backups_restic_services[services_service_name].data_directory }}"
+        backups_restic_user_aws_bucket_prefix: "\
+          {{ services_backups_restic_services[services_service_name].aws_bucket_prefix }}"
       loop: "{{ services_host_services | dict2items | map(attribute='key') }}"
       loop_control:
         loop_var: "services_service_name"