Port zfs snapshot backups

host_vars/yggdrasil/vars.yml

@@ -73,7 +73,8 @@ services_host_services:
 # --------------------------------------------------------------------------------------------------
 # services:backups
 # --------------------------------------------------------------------------------------------------
-services_backups_datasets_root: "rpool/var/lib/yggdrasil/data"
+services_backups_syncoid_data_dataset: "{{ services_data_dataset |
+                                           replace('rpool/var/lib', 'hpool/backup') }}"
 services_backups_datasets: "\
   {% set datasets = {} %}\
   {% for service in services_host_services.keys() %}\

main.yml

@@ -1,4 +1,5 @@
 ---
 - ansible.builtin.import_playbook: "plays/system/main.yml"
 - ansible.builtin.import_playbook: "plays/vpn/main.yml"
+- ansible.builtin.import_playbook: "plays/backups/main.yml"
 - ansible.builtin.import_playbook: "plays/services/main.yml"

playbooks/03-backups.yml

@@ -3,6 +3,5 @@
   hosts: yggdrasil
 
   tasks:
-    - import_tasks: tasks/backups/00-zfs-snapshots.yml
     - import_tasks: tasks/backups/01-restic-setup.yml
     - import_tasks: tasks/backups/02-restic-enable.yml

playbooks/03-test-backups.yml

@@ -1,6 +0,0 @@
----
-- name: Configure yggdrasil backups
-  hosts: yggdrasil
-
-  tasks:
-    - import_tasks: tasks/backups/01-restic-setup.yml

playbooks/tasks/backups/00-zfs-snapshots.d/sanoid.yml

@@ -1,49 +0,0 @@
-- name: Configure sanoid system snapshots
-  blockinfile:
-    path: /etc/sanoid/sanoid.conf
-    create: yes
-    mode: 0644
-    insertbefore: "BOF"
-    marker: "# {mark} ANSIBLE MANAGED BLOCK SYSTEM #"
-    block: |
-      [bpool/BOOT]
-              use_template = system
-              recursive = yes
-              process_children_only = yes
-
-      [rpool/ROOT]
-              use_template = system
-              recursive = yes
-              process_children_only = yes
-
-      [rpool/home]
-              use_template = system,home
-              recursive = yes
-              process_children_only = yes
-
-- name: Configure sanoid service snapshots
-  blockinfile:
-    path: /etc/sanoid/sanoid.conf
-    insertbefore: "# BEGIN ANSIBLE MANAGED BLOCK TEMPLATES #"
-    marker: "# {mark} ANSIBLE MANAGED BLOCK SERVICE {{ service_name }} #"
-    block: |
-      [rpool/var/lib/{{ ansible_hostname }}/data/pod-{{ service_name }}]
-              use_template = production
-              recursive = yes
-              process_children_only = yes
-
-      [hpool/backup/{{ ansible_hostname }}/data/pod-{{ service_name }}]
-              use_template = backup
-              recursive = yes
-              process_children_only = yes
-
-  with_items: "{{ host_services }}"
-  loop_control:
-    loop_var: service_name
-
-- name: Configure sanoid templates
-  blockinfile:
-    path: /etc/sanoid/sanoid.conf
-    insertafter: "EOF"
-    marker: "# {mark} ANSIBLE MANAGED BLOCK TEMPLATES #"
-    block: "{{ lookup('file', './filesystem/{{ ansible_hostname }}/etc/sanoid/sanoid-templates.conf') }}"

playbooks/tasks/backups/00-zfs-snapshots.d/syncoid.yml

@@ -1,58 +0,0 @@
-- name: Configure syncoid unit
-  blockinfile:
-    path: /etc/systemd/system/syncoid-volume-data.service
-    create: yes
-    mode: 0644
-    insertbefore: "BOF"
-    marker: "# {mark} ANSIBLE MANAGED BLOCK UNIT"
-    block: |
-      [Unit]
-      Description=Replicate volume data snapshots
-      Documentation=man:syncoid(8)
-      After=sanoid.service
-      Before=sanoid-prune.service
-      OnFailure=status-mail@%n.service
-  register: systemd_syncoid_volume_data_unit_service_file
-
-- name: Configure syncoid service
-  blockinfile:
-    path: /etc/systemd/system/syncoid-volume-data.service
-    insertafter: "# END ANSIBLE MANAGED BLOCK UNIT"
-    marker: "# {mark} ANSIBLE MANAGED BLOCK SERVICE"
-    block: |
-      [Service]
-      Type=oneshot
-  register: systemd_syncoid_volume_data_service_service_file
-
-- name: Configure syncoid commands
-  blockinfile:
-    path: /etc/systemd/system/syncoid-volume-data.service
-    insertbefore: "# BEGIN ANSIBLE MANAGED BLOCK INSTALL"
-    marker: "# {mark} ANSIBLE MANAGED BLOCK SYNCOID {{ service_name }} #"
-    block: >
-      ExecStart=/usr/sbin/syncoid --recursive --skip-parent --no-sync-snap
-      rpool/var/lib/{{ ansible_hostname }}/data/pod-{{ service_name }}
-      hpool/backup/{{ ansible_hostname }}/data/pod-{{ service_name }}
-  with_items: "{{ host_services }}"
-  loop_control:
-    loop_var: service_name
-  register: systemd_syncoid_volume_data_syncoid_service_file
-
-- name: Configure syncoid install
-  blockinfile:
-    path: /etc/systemd/system/syncoid-volume-data.service
-    insertafter: "EOF"
-    marker: "# {mark} ANSIBLE MANAGED BLOCK INSTALL"
-    block: |
-      [Install]
-      WantedBy=sanoid.service
-  register: systemd_syncoid_volume_data_install_service_file
-
-- name: SystemD daemon reload
-  systemd:
-    daemon_reload: true
-  when:
-    systemd_syncoid_volume_data_unit_service_file is changed or
-    systemd_syncoid_volume_data_service_service_file is changed or
-    systemd_syncoid_volume_data_install_service_file is changed or
-    systemd_syncoid_volume_data_syncoid_service_file is changed

playbooks/tasks/backups/00-zfs-snapshots.yml

@@ -1,24 +0,0 @@
-- name: Install sanoid
-  apt:
-    name: sanoid
-
-- name: Create sanoid directory
-  file:
-    path: /etc/sanoid
-    state: directory
-    mode: 0755
-
-- import_tasks: 00-zfs-snapshots.d/sanoid.yml
-
-- import_tasks: 00-zfs-snapshots.d/syncoid.yml
-
-- name: Enable syncoid service
-  systemd:
-    name: syncoid-volume-data.service
-    enabled: yes
-
-- name: Enable sanoid service
-  systemd:
-    name: sanoid.timer
-    enabled: yes
-    state: started

plays/backups/main.yml

@@ -0,0 +1,10 @@
+---
+- name: "backups : yggdrasil"
+  hosts: "yggdrasil"
+  roles:
+    - role: "snapshots"
+      tags: "backups:snapshots"
+      when: the_nine_worlds_production | bool
+    # - role: "backups"
+    #   tags: "backups:restic"
+    #   when: the_nine_worlds_production | bool

plays/backups/roles/snapshots/files/syncoid-batch

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import subprocess
+import yaml
+
+
+def load_and_validate_config_dir(config_dir):
+    if not os.path.isdir(config_dir):
+        raise ValueError(f"{config_dir} is not a directory")
+
+    return [
+        load_and_validate_config_file(os.path.join(config_dir, file))
+        for file in os.listdir(config_dir)
+    ]
+
+
+def load_and_validate_config_file(config_file_path):
+    if not os.path.isfile(config_file_path):
+        raise ValueError(f"{config_file_path} is not a file")
+
+    with open(config_file_path, encoding="utf-8") as config_file:
+        config = yaml.safe_load(config_file)
+
+    for key in ["dataset", "syncoid_dataset"]:
+        if key not in config:
+            raise KeyError(f"{key} must be present in {config_file_path}")
+
+    return config
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Backup service data using syncoid")
+    parser.add_argument("--config-dir", type=str, default="/etc/syncoid-batch.d",
+                        help="Path to directory with YAML config files")
+    args = parser.parse_args()
+
+    config_list = load_and_validate_config_dir(args.config_dir)
+
+    for config in config_list:
+        syncoid_cmd = ["/usr/sbin/syncoid", "--no-sync-snap"]
+        if config.get("recursive", False):
+            syncoid_cmd.append("--recursive")
+        if config.get("skip_parent", False):
+            syncoid_cmd.append("--skip-parent")
+        syncoid_cmd.append(config["dataset"])
+        syncoid_cmd.append(config["syncoid_dataset"])
+
+        subprocess.run(syncoid_cmd, check=True)

plays/backups/roles/snapshots/files/syncoid-batch.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Replicate volume data snapshots
+Documentation=man:syncoid(8)
+After=sanoid.service
+Before=sanoid-prune.service
+OnFailure=status-mail@%n.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/syncoid-batch --config-dir /etc/syncoid-batch.d
+
+[Install]
+WantedBy=sanoid.service

plays/backups/roles/snapshots/tasks/main.yml

@@ -0,0 +1,69 @@
+- name: "install sanoid and syncoid"
+  ansible.builtin.apt:
+    name:
+      - "sanoid"
+      - "python3-yaml"
+
+- name: "create sanoid configuration directory"
+  ansible.builtin.file:
+    path: "/etc/sanoid"
+    state: "directory"
+    mode: 0755
+
+- name: "configure sanoid templates"
+  ansible.builtin.blockinfile:
+    path: "/etc/sanoid/sanoid.conf"
+    create: true
+    mode: 0644
+    insertafter: "EOF"
+    marker: "# {mark} ANSIBLE MANAGED BLOCK TEMPLATES #"
+    block: "{{ lookup('file', './sanoid-templates.conf') }}"
+
+- name: "configure sanoid system snapshots"
+  ansible.builtin.blockinfile:
+    path: "/etc/sanoid/sanoid.conf"
+    insertbefore: "BOF"
+    marker: "# {mark} ANSIBLE MANAGED BLOCK SYSTEM {{ item.name }} #"
+    # Note that the {{ '' }} is required to force a newline.
+    block: |
+      [{{ item.name }}]
+              use_template = {{ item.templates | join(',') }}
+              recursive = {% if item.recursive %}yes{% else %}no{% endif %}{{ '' }}
+              process_children_only = {% if item.children_only %}yes{% else %}no{% endif %}{{ '' }}
+  loop: "{{ backups_snapshots_sanoid_system_datasets | reverse }}"
+
+- name: "create syncoid-batch config directory"
+  ansible.builtin.file:
+    path: "/etc/syncoid-batch.d"
+    state: "directory"
+    mode: 0755
+
+- name: "install syncoid script"
+  ansible.builtin.copy:
+    src: "./syncoid-batch"
+    dest: "/usr/local/sbin/syncoid-batch"
+    mode: 0755
+
+- name: "install syncoid-batch service"
+  ansible.builtin.copy:
+    src: "./syncoid-batch.service"
+    dest: "/etc/systemd/system/syncoid-batch.service"
+    mode: 0644
+  register: services_backups_snapshots_syncoid_volume_data_service_file
+
+- name: "snapshots : systemd daemon reload"
+  ansible.builtin.systemd:
+    daemon_reload: true
+  when:
+    services_backups_snapshots_syncoid_volume_data_service_file.changed
+
+- name: "enable syncoid-batch service"
+  ansible.builtin.systemd:
+    name: "syncoid-batch.service"
+    enabled: true
+
+- name: "enable sanoid service"
+  ansible.builtin.systemd:
+    name: "sanoid.timer"
+    enabled: true
+    state: "started"

plays/services/main.yml

@@ -56,3 +56,22 @@
       loop_control:
         loop_var: "services_service_name"
       tags: "always"
+
+- name: "services : yggdrasil"
+  hosts: "yggdrasil"
+
+  tasks:
+    - name: "backups"
+      ansible.builtin.include_role:
+        name: "backups"
+        apply:
+          tags:
+            - "services:{{ services_service_name }}"
+            - "services:backups"
+            - "services:backups:{{ services_service_name }}"
+            - "services:{{ services_service_name }}:backups"
+      loop: "{{ services_host_services | dict2items | map(attribute='key') }}"
+      loop_control:
+        loop_var: "services_service_name"
+      tags: "always"
+      when: the_nine_worlds_production | bool

plays/services/roles/backups/tasks/include/snapshots.yml

@@ -0,0 +1,21 @@
+- name: "{{ services_service_name }} : snapshots : configure service sanoid snapshots"
+  ansible.builtin.blockinfile:
+    path: "/etc/sanoid/sanoid.conf"
+    insertbefore: "# BEGIN ANSIBLE MANAGED BLOCK TEMPLATES #"
+    marker: "# {mark} ANSIBLE MANAGED BLOCK SERVICE {{ services_service_name }} #"
+    block: |
+      [{{ services_backups_user_dataset }}]
+              use_template = production
+              recursive = yes
+              process_children_only = yes
+
+      [{{ services_backups_user_syncoid_dataset }}]
+              use_template = backup
+              recursive = yes
+              process_children_only = yes
+
+- name: "{{ services_service_name }} : snapshots : configure service syncoid snapshots"
+  ansible.builtin.template:
+    src: "./snapshots/syncoid-volumes-service.yml.j2"
+    dest: "/etc/syncoid-batch.d/syncoid-volumes-{{ services_service_name }}.yml"
+    mode: 0644

plays/services/roles/backups/tasks/main.yml

@@ -0,0 +1,15 @@
+- name: "play:services : role:backups:{{ services_service_name }} : tasks:vars"
+  ansible.builtin.import_role:
+    name: "include"
+    vars_from: "user"
+  tags:
+    - "services:backups:user"
+    - "services:backups:{{ services_service_name }}:snapshots"
+    - "services:{{ services_service_name }}:backups:snapshots"
+
+- name: "play:services : role:backups : tasks:snapshots"
+  ansible.builtin.import_tasks: "include/snapshots.yml"
+  tags:
+    - "services:backups:snapshots"
+    - "services:backups:{{ services_service_name }}:snapshots"
+    - "services:{{ services_service_name }}:backups:snapshots"

plays/services/roles/backups/templates/snapshots/syncoid-volumes-service.yml.j2

@@ -0,0 +1,4 @@
+dataset: {{ services_backups_user_dataset }}
+syncoid_dataset: {{ services_backups_user_syncoid_dataset }}
+recursive: true
+skip_parent: true

plays/services/roles/backups/vars/main.yml

@@ -0,0 +1,3 @@
+services_backups_user_dataset: "{{ services_data_dataset }}/{{ services_service_user_name }}"
+services_backups_user_syncoid_dataset: "\
+  {{ services_backups_syncoid_data_dataset }}/{{ services_service_user_name }}"