From d155da3414aec4513f789fe58a20f4ef7b78bc73 Mon Sep 17 00:00:00 2001 From: Wojciech Kozlowski Date: Sun, 19 Feb 2023 22:36:16 +0100 Subject: [PATCH] Decouple backups from services --- inventory/group_vars/restic/vars.yml | 31 +++++++------ inventory/host_vars/yggdrasil/vars.yml | 2 + .../restic/user/meta/argument_specs.yml | 46 +++++++++++++++++++ .../roles/backups/restic/user/tasks/main.yml | 18 ++++++++ .../user/templates/restic-aws-keys.yml.j2 | 2 + .../restic/user/templates/restic.password.j2 | 1 + .../restic/user/templates/volumes.yml.j2 | 11 +++++ .../snapshots/user/meta/argument_specs.yml | 22 +++++++++ .../backups/snapshots/user/tasks/main.yml | 22 +++++++++ .../snapshots/user/templates/volumes.yml.j2 | 4 ++ .../backups/include/vars/datasets.yml | 4 -- .../backups/restic/meta/argument_specs.yml | 20 -------- .../services/backups/restic/tasks/main.yml | 28 ----------- .../restic/templates/restic-aws-keys.yml.j2 | 2 - .../restic/templates/restic.password.j2 | 1 - .../backups/restic/templates/volumes.yml.j2 | 11 ----- .../backups/snapshots/meta/argument_specs.yml | 17 ------- .../services/backups/snapshots/tasks/main.yml | 32 ------------- .../snapshots/templates/volumes.yml.j2 | 4 -- playbooks/services.yml | 36 ++++++++++++--- 20 files changed, 174 insertions(+), 140 deletions(-) create mode 100644 playbooks/roles/backups/restic/user/meta/argument_specs.yml create mode 100644 playbooks/roles/backups/restic/user/tasks/main.yml create mode 100644 playbooks/roles/backups/restic/user/templates/restic-aws-keys.yml.j2 create mode 100644 playbooks/roles/backups/restic/user/templates/restic.password.j2 create mode 100644 playbooks/roles/backups/restic/user/templates/volumes.yml.j2 create mode 100644 playbooks/roles/backups/snapshots/user/meta/argument_specs.yml create mode 100644 playbooks/roles/backups/snapshots/user/tasks/main.yml create mode 100644 playbooks/roles/backups/snapshots/user/templates/volumes.yml.j2 delete mode 100644 playbooks/roles/services/backups/include/vars/datasets.yml delete mode 100644 playbooks/roles/services/backups/restic/meta/argument_specs.yml delete mode 100644 playbooks/roles/services/backups/restic/tasks/main.yml delete mode 100644 playbooks/roles/services/backups/restic/templates/restic-aws-keys.yml.j2 delete mode 100644 playbooks/roles/services/backups/restic/templates/restic.password.j2 delete mode 100644 playbooks/roles/services/backups/restic/templates/volumes.yml.j2 delete mode 100644 playbooks/roles/services/backups/snapshots/meta/argument_specs.yml delete mode 100644 playbooks/roles/services/backups/snapshots/tasks/main.yml delete mode 100644 playbooks/roles/services/backups/snapshots/templates/volumes.yml.j2 diff --git a/inventory/group_vars/restic/vars.yml b/inventory/group_vars/restic/vars.yml index d1ec191..db87c9b 100644 --- a/inventory/group_vars/restic/vars.yml +++ b/inventory/group_vars/restic/vars.yml @@ -1,27 +1,30 @@ --- +# -------------------------------------------------------------------------------------------------- +# backups:restic +# -------------------------------------------------------------------------------------------------- +backups_restic_user_aws_access_key_id: "{{ vault_backups_restic_user_aws_access_key_id }}" +backups_restic_user_aws_secret_access_key: "\ + {{ vault_backups_restic_user_aws_secret_access_key }}" +backups_restic_user_aws_keys_file: "/etc/restic-aws-keys.yml" +backups_restic_user_aws_bucket_endpoint: "\ + {{ vault_backups_restic_user_aws_bucket_endpoint }}" +backups_restic_user_restic_password: "{{ vault_backups_restic_user_restic_password }}" +backups_restic_user_restic_password_file: "/etc/restic.password" +backups_restic_user_restic_keep_daily: 30 +backups_restic_user_restic_keep_monthly: 3 + # -------------------------------------------------------------------------------------------------- # services:backups # -------------------------------------------------------------------------------------------------- -services_backups_restic_restic_password: "{{ vault_services_backups_restic_restic_password }}" -services_backups_restic_aws_access_key_id: "{{ vault_services_backups_restic_aws_access_key_id }}" -services_backups_restic_aws_secret_access_key: "\ - {{ vault_services_backups_restic_aws_secret_access_key }}" -services_backups_restic_aws_bucket_endpoint: "\ - {{ vault_services_backups_restic_aws_bucket_endpoint }}" services_backups_restic_services: "\ {% set services_backups_restic_service = {} %}\ {% for service in services_host_services.keys() %}\ {{ services_backups_restic_service.update( { service: { - 'aws_access_key_id': services_backups_restic_aws_access_key_id, - 'aws_secret_access_key': services_backups_restic_aws_secret_access_key, - 'aws_keys_file': '/etc/restic-aws-keys.yml', - 'aws_bucket_endpoint': services_backups_restic_aws_bucket_endpoint, + 'user_name': ( 'pod-' ~ service ), + 'data_dataset': ( services_data_dataset ~ '/pod-' ~ service ), + 'data_directory': ( services_data_directory ~ '/pod-' ~ service ), 'aws_bucket_prefix': ( 'the-nine-worlds---pod-' ~ service ), - 'restic_password': services_backups_restic_restic_password, - 'restic_password_file': '/etc/restic.password', - 'restic_keep_daily': 30, - 'restic_keep_monthly': 3, }} ) }}\ {% endfor %}\ diff --git a/inventory/host_vars/yggdrasil/vars.yml b/inventory/host_vars/yggdrasil/vars.yml index 43e382d..c905483 100644 --- a/inventory/host_vars/yggdrasil/vars.yml +++ b/inventory/host_vars/yggdrasil/vars.yml @@ -133,6 +133,8 @@ services_backups_snapshots_services: "\ {% for service in services_host_services.keys() %}\ {{ services_backups_snapshots_service.update( { service: { + 'user_name': ( 'pod-' ~ service ), + 'data_dataset': ( services_data_dataset ~ '/pod-' ~ service ), 'backup_dataset': ( services_backups_snapshots_data_dataset ~ '/pod-' ~ service ), 'recursive': true, 'skip_parent': true, diff --git a/playbooks/roles/backups/restic/user/meta/argument_specs.yml b/playbooks/roles/backups/restic/user/meta/argument_specs.yml new file mode 100644 index 0000000..1c817eb --- /dev/null +++ b/playbooks/roles/backups/restic/user/meta/argument_specs.yml @@ -0,0 +1,46 @@ +--- +argument_specs: + main: + options: + ansible_hostname: + type: "str" + required: true + backups_restic_user_name: + type: "str" + required: true + backups_restic_user_use_dataset: + type: "bool" + required: true + backups_restic_user_data_dataset: + type: "str" + required: false + backups_restic_user_data_directory: + type: "str" + required: false + backups_restic_user_aws_access_key_id: + type: "str" + required: true + backups_restic_user_aws_secret_access_key: + type: "str" + required: true + backups_restic_user_aws_keys_file: + type: "str" + required: true + backups_restic_user_aws_bucket_endpoint: + type: "str" + required: true + backups_restic_user_aws_bucket_prefix: + type: "str" + required: true + backups_restic_user_restic_password: + type: "str" + required: true + backups_restic_user_restic_password_file: + type: "str" + required: true + backups_restic_user_restic_keep_daily: + type: "int" + required: true + backups_restic_user_restic_keep_monthly: + type: "int" + required: true diff --git a/playbooks/roles/backups/restic/user/tasks/main.yml b/playbooks/roles/backups/restic/user/tasks/main.yml new file mode 100644 index 0000000..ce793b5 --- /dev/null +++ b/playbooks/roles/backups/restic/user/tasks/main.yml @@ -0,0 +1,18 @@ +--- +- name: "create restic password file" + ansible.builtin.template: + src: "./restic.password.j2" + dest: "{{ backups_restic_user_restic_password_file }}" + mode: 0600 + +- name: "create aws key file" + ansible.builtin.template: + src: "./restic-aws-keys.yml.j2" + dest: "{{ backups_restic_user_aws_keys_file }}" + mode: 0600 + +- name: "configure service restic backups" + ansible.builtin.template: + src: "./volumes.yml.j2" + dest: "/etc/restic-batch.d/{{ backups_restic_user_name }}.yml" + mode: 0644 diff --git a/playbooks/roles/backups/restic/user/templates/restic-aws-keys.yml.j2 b/playbooks/roles/backups/restic/user/templates/restic-aws-keys.yml.j2 new file mode 100644 index 0000000..a063fd9 --- /dev/null +++ b/playbooks/roles/backups/restic/user/templates/restic-aws-keys.yml.j2 @@ -0,0 +1,2 @@ +AWS_ACCESS_KEY_ID: {{ backups_restic_user_aws_access_key_id }} +AWS_SECRET_ACCESS_KEY: {{ backups_restic_user_aws_secret_access_key }} diff --git a/playbooks/roles/backups/restic/user/templates/restic.password.j2 b/playbooks/roles/backups/restic/user/templates/restic.password.j2 new file mode 100644 index 0000000..8468fd1 --- /dev/null +++ b/playbooks/roles/backups/restic/user/templates/restic.password.j2 @@ -0,0 +1 @@ +{{ backups_restic_user_restic_password }} diff --git a/playbooks/roles/backups/restic/user/templates/volumes.yml.j2 b/playbooks/roles/backups/restic/user/templates/volumes.yml.j2 new file mode 100644 index 0000000..727c877 --- /dev/null +++ b/playbooks/roles/backups/restic/user/templates/volumes.yml.j2 @@ -0,0 +1,11 @@ +{% if backups_restic_user_use_dataset %} +dataset: {{ backups_restic_user_data_dataset }} +{% else %} +directory: {{ backups_restic_user_data_directory }} +{% endif %} +aws_bucket_keys_file: {{ backups_restic_user_aws_keys_file }} +aws_bucket_endpoint: {{ backups_restic_user_aws_bucket_endpoint }} +aws_bucket_prefix: {{ backups_restic_user_aws_bucket_prefix }} +restic_password_file: {{ backups_restic_user_restic_password_file }} +restic_keep_daily: {{ backups_restic_user_restic_keep_daily }} +restic_keep_monthly: {{ backups_restic_user_restic_keep_monthly }} diff --git a/playbooks/roles/backups/snapshots/user/meta/argument_specs.yml b/playbooks/roles/backups/snapshots/user/meta/argument_specs.yml new file mode 100644 index 0000000..e4b6c59 --- /dev/null +++ b/playbooks/roles/backups/snapshots/user/meta/argument_specs.yml @@ -0,0 +1,22 @@ +--- +argument_specs: + main: + options: + ansible_hostname: + type: "str" + required: true + backups_snapshots_user_name: + type: "str" + required: true + backups_snapshots_user_data_dataset: + type: "str" + required: true + backups_snapshots_user_backup_dataset: + type: "str" + required: true + backups_snapshots_user_recursive: + type: "bool" + required: true + backups_snapshots_user_skip_parent: + type: "bool" + required: true diff --git a/playbooks/roles/backups/snapshots/user/tasks/main.yml b/playbooks/roles/backups/snapshots/user/tasks/main.yml new file mode 100644 index 0000000..c34b603 --- /dev/null +++ b/playbooks/roles/backups/snapshots/user/tasks/main.yml @@ -0,0 +1,22 @@ +--- +- name: "configure service sanoid snapshots" + ansible.builtin.blockinfile: + path: "/etc/sanoid/sanoid.conf" + insertbefore: "# BEGIN ANSIBLE MANAGED BLOCK TEMPLATES #" + marker: "# {mark} ANSIBLE MANAGED BLOCK USER {{ backups_snapshots_user_name }} #" + block: | + [{{ backups_snapshots_user_data_dataset }}] + use_template = production + recursive = yes + process_children_only = yes + + [{{ backups_snapshots_user_backup_dataset }}] + use_template = backup + recursive = yes + process_children_only = yes + +- name: "configure service syncoid snapshots" + ansible.builtin.template: + src: "./volumes.yml.j2" + dest: "/etc/syncoid-batch.d/{{ backups_snapshots_user_name }}.yml" + mode: 0644 diff --git a/playbooks/roles/backups/snapshots/user/templates/volumes.yml.j2 b/playbooks/roles/backups/snapshots/user/templates/volumes.yml.j2 new file mode 100644 index 0000000..e3ef3d1 --- /dev/null +++ b/playbooks/roles/backups/snapshots/user/templates/volumes.yml.j2 @@ -0,0 +1,4 @@ +dataset: {{ backups_snapshots_user_data_dataset }} +backup_dataset: {{ backups_snapshots_user_backup_dataset }} +recursive: {{ backups_snapshots_user_recursive }} +skip_parent: {{ backups_snapshots_user_skip_parent }} diff --git a/playbooks/roles/services/backups/include/vars/datasets.yml b/playbooks/roles/services/backups/include/vars/datasets.yml deleted file mode 100644 index a0aa52c..0000000 --- a/playbooks/roles/services/backups/include/vars/datasets.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -services_backups_user_data_dataset: "{{ services_data_dataset }}/{{ services_service_user_name }}" -services_backups_user_data_directory: "\ - {{ services_data_directory }}/{{ services_service_user_name }}" diff --git a/playbooks/roles/services/backups/restic/meta/argument_specs.yml b/playbooks/roles/services/backups/restic/meta/argument_specs.yml deleted file mode 100644 index 3286b48..0000000 --- a/playbooks/roles/services/backups/restic/meta/argument_specs.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -argument_specs: - main: - options: - ansible_hostname: - type: "str" - required: true - services_service_name: - type: "str" - required: true - services_data_dataset: - type: "str" - required: false - services_data_directory: - type: "str" - required: false - services_backups_restic_services: - type: "dict" - elem: "dict" - required: true diff --git a/playbooks/roles/services/backups/restic/tasks/main.yml b/playbooks/roles/services/backups/restic/tasks/main.yml deleted file mode 100644 index bda1094..0000000 --- a/playbooks/roles/services/backups/restic/tasks/main.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -- name: "{{ services_service_name }} : tasks:vars" - ansible.builtin.import_role: - name: "services/include" - vars_from: "user" - -- name: "{{ services_service_name }} : tasks:vars" - ansible.builtin.import_role: - name: "services/backups/include" - vars_from: "datasets" - -- name: "{{ services_service_name }} : create restic password file" - ansible.builtin.template: - src: "./restic.password.j2" - dest: "{{ services_backups_restic_services[services_service_name].restic_password_file }}" - mode: 0600 - -- name: "{{ services_service_name }} : create aws key file" - ansible.builtin.template: - src: "./restic-aws-keys.yml.j2" - dest: "{{ services_backups_restic_services[services_service_name].aws_keys_file }}" - mode: 0600 - -- name: "{{ services_service_name }} : configure service restic backups" - ansible.builtin.template: - src: "./volumes.yml.j2" - dest: "/etc/restic-batch.d/{{ services_service_user_name }}.yml" - mode: 0644 diff --git a/playbooks/roles/services/backups/restic/templates/restic-aws-keys.yml.j2 b/playbooks/roles/services/backups/restic/templates/restic-aws-keys.yml.j2 deleted file mode 100644 index 73313d2..0000000 --- a/playbooks/roles/services/backups/restic/templates/restic-aws-keys.yml.j2 +++ /dev/null @@ -1,2 +0,0 @@ -AWS_ACCESS_KEY_ID: {{ services_backups_restic_services[services_service_name].aws_access_key_id }} -AWS_SECRET_ACCESS_KEY: {{ services_backups_restic_services[services_service_name].aws_secret_access_key }} diff --git a/playbooks/roles/services/backups/restic/templates/restic.password.j2 b/playbooks/roles/services/backups/restic/templates/restic.password.j2 deleted file mode 100644 index ab2696e..0000000 --- a/playbooks/roles/services/backups/restic/templates/restic.password.j2 +++ /dev/null @@ -1 +0,0 @@ -{{ services_backups_restic_services[services_service_name].restic_password }} diff --git a/playbooks/roles/services/backups/restic/templates/volumes.yml.j2 b/playbooks/roles/services/backups/restic/templates/volumes.yml.j2 deleted file mode 100644 index a620918..0000000 --- a/playbooks/roles/services/backups/restic/templates/volumes.yml.j2 +++ /dev/null @@ -1,11 +0,0 @@ -{% if services_data_dataset is defined %} -dataset: {{ services_backups_user_data_dataset }} -{% else %} -directory: {{ services_backups_user_data_directory }} -{% endif %} -aws_bucket_keys_file: {{ services_backups_restic_services[services_service_name].aws_keys_file }} -aws_bucket_endpoint: {{ services_backups_restic_services[services_service_name].aws_bucket_endpoint }} -aws_bucket_prefix: {{ services_backups_restic_services[services_service_name].aws_bucket_prefix }} -restic_password_file: {{ services_backups_restic_services[services_service_name].restic_password_file }} -restic_keep_daily: {{ services_backups_restic_services[services_service_name].restic_keep_daily }} -restic_keep_monthly: {{ services_backups_restic_services[services_service_name].restic_keep_monthly }} diff --git a/playbooks/roles/services/backups/snapshots/meta/argument_specs.yml b/playbooks/roles/services/backups/snapshots/meta/argument_specs.yml deleted file mode 100644 index 2f9a5c4..0000000 --- a/playbooks/roles/services/backups/snapshots/meta/argument_specs.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -argument_specs: - main: - options: - ansible_hostname: - type: "str" - required: true - services_service_name: - type: "str" - required: true - services_data_dataset: - type: "str" - required: true - services_backups_snapshots_services: - type: "dict" - elem: "dict" - required: true diff --git a/playbooks/roles/services/backups/snapshots/tasks/main.yml b/playbooks/roles/services/backups/snapshots/tasks/main.yml deleted file mode 100644 index 18d4382..0000000 --- a/playbooks/roles/services/backups/snapshots/tasks/main.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -- name: "{{ services_service_name }} : tasks:vars" - ansible.builtin.import_role: - name: "services/include" - vars_from: "user" - -- name: "{{ services_service_name }} : tasks:vars" - ansible.builtin.import_role: - name: "services/backups/include" - vars_from: "datasets" - -- name: "{{ services_service_name }} : configure service sanoid snapshots" - ansible.builtin.blockinfile: - path: "/etc/sanoid/sanoid.conf" - insertbefore: "# BEGIN ANSIBLE MANAGED BLOCK TEMPLATES #" - marker: "# {mark} ANSIBLE MANAGED BLOCK SERVICE {{ services_service_name }} #" - block: | - [{{ services_backups_user_data_dataset }}] - use_template = production - recursive = yes - process_children_only = yes - - [{{ services_backups_snapshots_services[services_service_name].backup_dataset }}] - use_template = backup - recursive = yes - process_children_only = yes - -- name: "{{ services_service_name }} : configure service syncoid snapshots" - ansible.builtin.template: - src: "./volumes.yml.j2" - dest: "/etc/syncoid-batch.d/{{ services_service_user_name }}.yml" - mode: 0644 diff --git a/playbooks/roles/services/backups/snapshots/templates/volumes.yml.j2 b/playbooks/roles/services/backups/snapshots/templates/volumes.yml.j2 deleted file mode 100644 index 22a6ef8..0000000 --- a/playbooks/roles/services/backups/snapshots/templates/volumes.yml.j2 +++ /dev/null @@ -1,4 +0,0 @@ -dataset: {{ services_backups_user_data_dataset }} -backup_dataset: {{ services_backups_snapshots_services[services_service_name].backup_dataset }} -recursive: {{ services_backups_snapshots_services[services_service_name].recursive }} -skip_parent: {{ services_backups_snapshots_services[services_service_name].skip_parent }} diff --git a/playbooks/services.yml b/playbooks/services.yml index c08109b..8c9f4f0 100644 --- a/playbooks/services.yml +++ b/playbooks/services.yml @@ -79,14 +79,26 @@ - name: "backups : snapshots" ansible.builtin.include_role: - name: "services/backups/snapshots" + name: "backups/snapshots/user" apply: tags: - "services:{{ services_service_name }}" - "services:backups" - "services:backups:snapshots" - - "services:backups:snapshots:{{ services_service_name }}" - - "services:{{ services_service_name }}:backups:snapshots" + - "services:backups:snapshots:user" + - "services:backups:snapshots:user:{{ services_service_name }}" + - "services:{{ services_service_name }}:backups:snapshots:user" + vars: + backups_snapshots_user_name: "\ + {{ services_backups_snapshots_services[services_service_name].user_name }}" + backups_snapshots_user_data_dataset: "\ + {{ services_backups_snapshots_services[services_service_name].data_dataset }}" + backups_snapshots_user_backup_dataset: "\ + {{ services_backups_snapshots_services[services_service_name].backup_dataset }}" + backups_snapshots_user_recursive: "\ + {{ services_backups_snapshots_services[services_service_name].recursive }}" + backups_snapshots_user_skip_parent: "\ + {{ services_backups_snapshots_services[services_service_name].skip_parent }}" loop: "{{ services_host_services | dict2items | map(attribute='key') }}" loop_control: loop_var: "services_service_name" @@ -98,14 +110,24 @@ tasks: - name: "backups : restic" ansible.builtin.include_role: - name: "services/backups/restic" + name: "backups/restic/user" apply: tags: - "services:{{ services_service_name }}" - "services:backups" - - "services:backups:restic" - - "services:backups:restic:{{ services_service_name }}" - - "services:{{ services_service_name }}:backups:restic" + - "services:backups:restic:user" + - "services:backups:restic:user:{{ services_service_name }}" + - "services:{{ services_service_name }}:backups:restic:user" + vars: + backups_restic_user_name: "\ + {{ services_backups_restic_services[services_service_name].user_name }}" + backups_restic_user_use_dataset: "{{ 'zfs' in group_names }}" + backups_restic_user_data_dataset: "\ + {{ services_backups_restic_services[services_service_name].data_dataset }}" + backups_restic_user_data_directory: "\ + {{ services_backups_restic_services[services_service_name].data_directory }}" + backups_restic_user_aws_bucket_prefix: "\ + {{ services_backups_restic_services[services_service_name].aws_bucket_prefix }}" loop: "{{ services_host_services | dict2items | map(attribute='key') }}" loop_control: loop_var: "services_service_name"