From 226bd0369f4d92277cdeabfa47cf7f3246258e90 Mon Sep 17 00:00:00 2001 From: Wojciech Kozlowski Date: Sun, 12 Feb 2023 16:37:48 +0100 Subject: [PATCH] Seprate snapshots/restic backups roles --- .../group_vars}/all/vars.yml | 0 .../group_vars}/asgard/vars.yml | 0 .../group_vars}/bifrost/vars.yml | 0 .../group_vars}/home/vars.yml | 0 .../group_vars}/remote/vars.yml | 0 inventory/group_vars/restic/vars.yml | 28 +++++++++++++++++++ .../host_vars}/heimdall/vars.yml | 0 .../host_vars}/valkyrie/vars.yml | 0 .../host_vars}/yggdrasil/vars.yml | 25 ----------------- production => inventory/production | 4 +++ testing => inventory/testing | 4 +++ playbooks/backups.yml | 4 +++ playbooks/backups_start.yml | 4 +++ playbooks/backups_stop.yml | 4 +++ .../backups/{ => include}/vars/main.yml | 0 .../backups/restic/meta/argument_specs.yml | 17 +++++++++++ .../services/backups/restic/tasks/main.yml | 28 +++++++++++++++++++ .../templates}/restic-aws-keys.yml.j2 | 0 .../templates}/restic-volumes-service.yml.j2 | 0 .../templates}/restic.password.j2 | 0 .../{ => snapshots}/meta/argument_specs.yml | 4 --- .../tasks/main.yml} | 16 +++++++++-- .../templates}/syncoid-volumes-service.yml.j2 | 0 .../services/backups/tasks/include/restic.yml | 18 ------------ .../roles/services/backups/tasks/main.yml | 26 ----------------- playbooks/services.yml | 28 ++++++++++++++++--- 26 files changed, 130 insertions(+), 80 deletions(-) rename {group_vars => inventory/group_vars}/all/vars.yml (100%) rename {group_vars => inventory/group_vars}/asgard/vars.yml (100%) rename {group_vars => inventory/group_vars}/bifrost/vars.yml (100%) rename {group_vars => inventory/group_vars}/home/vars.yml (100%) rename {group_vars => inventory/group_vars}/remote/vars.yml (100%) create mode 100644 inventory/group_vars/restic/vars.yml rename {host_vars => inventory/host_vars}/heimdall/vars.yml (100%) rename {host_vars => inventory/host_vars}/valkyrie/vars.yml (100%) rename {host_vars => inventory/host_vars}/yggdrasil/vars.yml (76%) rename production => inventory/production (95%) rename testing => inventory/testing (96%) rename playbooks/roles/services/backups/{ => include}/vars/main.yml (100%) create mode 100644 playbooks/roles/services/backups/restic/meta/argument_specs.yml create mode 100644 playbooks/roles/services/backups/restic/tasks/main.yml rename playbooks/roles/services/backups/{templates/restic => restic/templates}/restic-aws-keys.yml.j2 (100%) rename playbooks/roles/services/backups/{templates/restic => restic/templates}/restic-volumes-service.yml.j2 (100%) rename playbooks/roles/services/backups/{templates/restic => restic/templates}/restic.password.j2 (100%) rename playbooks/roles/services/backups/{ => snapshots}/meta/argument_specs.yml (77%) rename playbooks/roles/services/backups/{tasks/include/snapshots.yml => snapshots/tasks/main.yml} (60%) rename playbooks/roles/services/backups/{templates/snapshots => snapshots/templates}/syncoid-volumes-service.yml.j2 (100%) delete mode 100644 playbooks/roles/services/backups/tasks/include/restic.yml delete mode 100644 playbooks/roles/services/backups/tasks/main.yml diff --git a/group_vars/all/vars.yml b/inventory/group_vars/all/vars.yml similarity index 100% rename from group_vars/all/vars.yml rename to inventory/group_vars/all/vars.yml diff --git a/group_vars/asgard/vars.yml b/inventory/group_vars/asgard/vars.yml similarity index 100% rename from group_vars/asgard/vars.yml rename to inventory/group_vars/asgard/vars.yml diff --git a/group_vars/bifrost/vars.yml b/inventory/group_vars/bifrost/vars.yml similarity index 100% rename from group_vars/bifrost/vars.yml rename to inventory/group_vars/bifrost/vars.yml diff --git a/group_vars/home/vars.yml b/inventory/group_vars/home/vars.yml similarity index 100% rename from group_vars/home/vars.yml rename to inventory/group_vars/home/vars.yml diff --git a/group_vars/remote/vars.yml b/inventory/group_vars/remote/vars.yml similarity index 100% rename from group_vars/remote/vars.yml rename to inventory/group_vars/remote/vars.yml diff --git a/inventory/group_vars/restic/vars.yml b/inventory/group_vars/restic/vars.yml new file mode 100644 index 0000000..d1ec191 --- /dev/null +++ b/inventory/group_vars/restic/vars.yml @@ -0,0 +1,28 @@ +--- +# -------------------------------------------------------------------------------------------------- +# services:backups +# -------------------------------------------------------------------------------------------------- +services_backups_restic_restic_password: "{{ vault_services_backups_restic_restic_password }}" +services_backups_restic_aws_access_key_id: "{{ vault_services_backups_restic_aws_access_key_id }}" +services_backups_restic_aws_secret_access_key: "\ + {{ vault_services_backups_restic_aws_secret_access_key }}" +services_backups_restic_aws_bucket_endpoint: "\ + {{ vault_services_backups_restic_aws_bucket_endpoint }}" +services_backups_restic_services: "\ + {% set services_backups_restic_service = {} %}\ + {% for service in services_host_services.keys() %}\ + {{ services_backups_restic_service.update( + { service: { + 'aws_access_key_id': services_backups_restic_aws_access_key_id, + 'aws_secret_access_key': services_backups_restic_aws_secret_access_key, + 'aws_keys_file': '/etc/restic-aws-keys.yml', + 'aws_bucket_endpoint': services_backups_restic_aws_bucket_endpoint, + 'aws_bucket_prefix': ( 'the-nine-worlds---pod-' ~ service ), + 'restic_password': services_backups_restic_restic_password, + 'restic_password_file': '/etc/restic.password', + 'restic_keep_daily': 30, + 'restic_keep_monthly': 3, + }} + ) }}\ + {% endfor %}\ + {{ services_backups_restic_service }}" diff --git a/host_vars/heimdall/vars.yml b/inventory/host_vars/heimdall/vars.yml similarity index 100% rename from host_vars/heimdall/vars.yml rename to inventory/host_vars/heimdall/vars.yml diff --git a/host_vars/valkyrie/vars.yml b/inventory/host_vars/valkyrie/vars.yml similarity index 100% rename from host_vars/valkyrie/vars.yml rename to inventory/host_vars/valkyrie/vars.yml diff --git a/host_vars/yggdrasil/vars.yml b/inventory/host_vars/yggdrasil/vars.yml similarity index 76% rename from host_vars/yggdrasil/vars.yml rename to inventory/host_vars/yggdrasil/vars.yml index e132463..6b2ef3d 100644 --- a/host_vars/yggdrasil/vars.yml +++ b/inventory/host_vars/yggdrasil/vars.yml @@ -96,28 +96,3 @@ services_backups_snapshots_services: "\ ) }}\ {% endfor %}\ {{ services_backups_snapshots_service }}" - -services_backups_restic_restic_password: "{{ vault_services_backups_restic_restic_password }}" -services_backups_restic_aws_access_key_id: "{{ vault_services_backups_restic_aws_access_key_id }}" -services_backups_restic_aws_secret_access_key: "\ - {{ vault_services_backups_restic_aws_secret_access_key }}" -services_backups_restic_aws_bucket_endpoint: "\ - {{ vault_services_backups_restic_aws_bucket_endpoint }}" -services_backups_restic_services: "\ - {% set services_backups_restic_service = {} %}\ - {% for service in services_host_services.keys() %}\ - {{ services_backups_restic_service.update( - { service: { - 'aws_access_key_id': services_backups_restic_aws_access_key_id, - 'aws_secret_access_key': services_backups_restic_aws_secret_access_key, - 'aws_keys_file': '/etc/restic-aws-keys.yml', - 'aws_bucket_endpoint': services_backups_restic_aws_bucket_endpoint, - 'aws_bucket_prefix': ( 'the-nine-worlds---pod-' ~ service ), - 'restic_password': services_backups_restic_restic_password, - 'restic_password_file': '/etc/restic.password', - 'restic_keep_daily': 30, - 'restic_keep_monthly': 3, - }} - ) }}\ - {% endfor %}\ - {{ services_backups_restic_service }}" diff --git a/production b/inventory/production similarity index 95% rename from production rename to inventory/production index 69f102c..9369555 100644 --- a/production +++ b/inventory/production @@ -14,6 +14,10 @@ heimdall valkyrie yggdrasil +# Hosts with restic backups. +[restic] +yggdrasil + # -------------------------------------------------------------------------------------------------- # Network. # -------------------------------------------------------------------------------------------------- diff --git a/testing b/inventory/testing similarity index 96% rename from testing rename to inventory/testing index 03809cb..840b5c9 100644 --- a/testing +++ b/inventory/testing @@ -18,6 +18,10 @@ heimdall valkyrie yggdrasil +# Hosts with restic backups. +[restic] +yggdrasil + # -------------------------------------------------------------------------------------------------- # Network. # -------------------------------------------------------------------------------------------------- diff --git a/playbooks/backups.yml b/playbooks/backups.yml index 9d0ffa8..7f52f4f 100644 --- a/playbooks/backups.yml +++ b/playbooks/backups.yml @@ -4,5 +4,9 @@ roles: - role: "backups/snapshots/setup" tags: "backups:snapshots:setup" + +- name: "backups : restic" + hosts: "restic" + roles: - role: "backups/restic/setup" tags: "backups:restic:setup" diff --git a/playbooks/backups_start.yml b/playbooks/backups_start.yml index 39d30e1..920df20 100644 --- a/playbooks/backups_start.yml +++ b/playbooks/backups_start.yml @@ -4,5 +4,9 @@ roles: - role: "backups/snapshots/start" tags: "backups:snapshots:start" + +- name: "backups : restic" + hosts: "restic" + roles: - role: "backups/restic/start" tags: "backups:restic:start" diff --git a/playbooks/backups_stop.yml b/playbooks/backups_stop.yml index f51d786..8f7c3a1 100644 --- a/playbooks/backups_stop.yml +++ b/playbooks/backups_stop.yml @@ -4,5 +4,9 @@ roles: - role: "backups/snapshots/stop" tags: "backups:snapshots:stop" + +- name: "backups : restic" + hosts: "restic" + roles: - role: "backups/restic/stop" tags: "backups:restic:stop" diff --git a/playbooks/roles/services/backups/vars/main.yml b/playbooks/roles/services/backups/include/vars/main.yml similarity index 100% rename from playbooks/roles/services/backups/vars/main.yml rename to playbooks/roles/services/backups/include/vars/main.yml diff --git a/playbooks/roles/services/backups/restic/meta/argument_specs.yml b/playbooks/roles/services/backups/restic/meta/argument_specs.yml new file mode 100644 index 0000000..4922206 --- /dev/null +++ b/playbooks/roles/services/backups/restic/meta/argument_specs.yml @@ -0,0 +1,17 @@ +--- +argument_specs: + main: + options: + ansible_hostname: + type: "str" + required: true + services_service_name: + type: "str" + required: true + services_data_dataset: + type: "str" + required: true + services_backups_restic_services: + type: "dict" + elem: "dict" + required: true diff --git a/playbooks/roles/services/backups/restic/tasks/main.yml b/playbooks/roles/services/backups/restic/tasks/main.yml new file mode 100644 index 0000000..48f2134 --- /dev/null +++ b/playbooks/roles/services/backups/restic/tasks/main.yml @@ -0,0 +1,28 @@ +--- +- name: "{{ services_service_name }} : tasks:vars" + ansible.builtin.import_role: + name: "services/include" + vars_from: "user" + +- name: "{{ services_service_name }} : tasks:vars" + ansible.builtin.import_role: + name: "services/backups/include" + vars_from: "main" + +- name: "{{ services_service_name }} : create restic password file" + ansible.builtin.template: + src: "./restic.password.j2" + dest: "{{ services_backups_restic_services[services_service_name].restic_password_file }}" + mode: 0600 + +- name: "{{ services_service_name }} : create aws key file" + ansible.builtin.template: + src: "./restic-aws-keys.yml.j2" + dest: "{{ services_backups_restic_services[services_service_name].aws_keys_file }}" + mode: 0600 + +- name: "{{ services_service_name }} : configure service restic backups" + ansible.builtin.template: + src: "./restic-volumes-service.yml.j2" + dest: "/etc/restic-batch.d/restic-volumes-{{ services_service_name }}.yml" + mode: 0644 diff --git a/playbooks/roles/services/backups/templates/restic/restic-aws-keys.yml.j2 b/playbooks/roles/services/backups/restic/templates/restic-aws-keys.yml.j2 similarity index 100% rename from playbooks/roles/services/backups/templates/restic/restic-aws-keys.yml.j2 rename to playbooks/roles/services/backups/restic/templates/restic-aws-keys.yml.j2 diff --git a/playbooks/roles/services/backups/templates/restic/restic-volumes-service.yml.j2 b/playbooks/roles/services/backups/restic/templates/restic-volumes-service.yml.j2 similarity index 100% rename from playbooks/roles/services/backups/templates/restic/restic-volumes-service.yml.j2 rename to playbooks/roles/services/backups/restic/templates/restic-volumes-service.yml.j2 diff --git a/playbooks/roles/services/backups/templates/restic/restic.password.j2 b/playbooks/roles/services/backups/restic/templates/restic.password.j2 similarity index 100% rename from playbooks/roles/services/backups/templates/restic/restic.password.j2 rename to playbooks/roles/services/backups/restic/templates/restic.password.j2 diff --git a/playbooks/roles/services/backups/meta/argument_specs.yml b/playbooks/roles/services/backups/snapshots/meta/argument_specs.yml similarity index 77% rename from playbooks/roles/services/backups/meta/argument_specs.yml rename to playbooks/roles/services/backups/snapshots/meta/argument_specs.yml index c603d3a..2f9a5c4 100644 --- a/playbooks/roles/services/backups/meta/argument_specs.yml +++ b/playbooks/roles/services/backups/snapshots/meta/argument_specs.yml @@ -15,7 +15,3 @@ argument_specs: type: "dict" elem: "dict" required: true - services_backups_restic_services: - type: "dict" - elem: "dict" - required: true diff --git a/playbooks/roles/services/backups/tasks/include/snapshots.yml b/playbooks/roles/services/backups/snapshots/tasks/main.yml similarity index 60% rename from playbooks/roles/services/backups/tasks/include/snapshots.yml rename to playbooks/roles/services/backups/snapshots/tasks/main.yml index 81a52ba..8d7e897 100644 --- a/playbooks/roles/services/backups/tasks/include/snapshots.yml +++ b/playbooks/roles/services/backups/snapshots/tasks/main.yml @@ -1,5 +1,15 @@ --- -- name: "{{ services_service_name }} : snapshots : configure service sanoid snapshots" +- name: "{{ services_service_name }} : tasks:vars" + ansible.builtin.import_role: + name: "services/include" + vars_from: "user" + +- name: "{{ services_service_name }} : tasks:vars" + ansible.builtin.import_role: + name: "services/backups/include" + vars_from: "main" + +- name: "{{ services_service_name }} : configure service sanoid snapshots" ansible.builtin.blockinfile: path: "/etc/sanoid/sanoid.conf" insertbefore: "# BEGIN ANSIBLE MANAGED BLOCK TEMPLATES #" @@ -15,8 +25,8 @@ recursive = yes process_children_only = yes -- name: "{{ services_service_name }} : snapshots : configure service syncoid snapshots" +- name: "{{ services_service_name }} : configure service syncoid snapshots" ansible.builtin.template: - src: "./snapshots/syncoid-volumes-service.yml.j2" + src: "./syncoid-volumes-service.yml.j2" dest: "/etc/syncoid-batch.d/syncoid-volumes-{{ services_service_name }}.yml" mode: 0644 diff --git a/playbooks/roles/services/backups/templates/snapshots/syncoid-volumes-service.yml.j2 b/playbooks/roles/services/backups/snapshots/templates/syncoid-volumes-service.yml.j2 similarity index 100% rename from playbooks/roles/services/backups/templates/snapshots/syncoid-volumes-service.yml.j2 rename to playbooks/roles/services/backups/snapshots/templates/syncoid-volumes-service.yml.j2 diff --git a/playbooks/roles/services/backups/tasks/include/restic.yml b/playbooks/roles/services/backups/tasks/include/restic.yml deleted file mode 100644 index 817a3ba..0000000 --- a/playbooks/roles/services/backups/tasks/include/restic.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -- name: "{{ services_service_name }} : restic : create restic password file" - ansible.builtin.template: - src: "./restic/restic.password.j2" - dest: "{{ services_backups_restic_services[services_service_name].restic_password_file }}" - mode: 0600 - -- name: "{{ services_service_name }} : restic : create aws key file" - ansible.builtin.template: - src: "./restic/restic-aws-keys.yml.j2" - dest: "{{ services_backups_restic_services[services_service_name].aws_keys_file }}" - mode: 0600 - -- name: "{{ services_service_name }} : restic : configure service restic backups" - ansible.builtin.template: - src: "./restic/restic-volumes-service.yml.j2" - dest: "/etc/restic-batch.d/restic-volumes-{{ services_service_name }}.yml" - mode: 0644 diff --git a/playbooks/roles/services/backups/tasks/main.yml b/playbooks/roles/services/backups/tasks/main.yml deleted file mode 100644 index 63f2121..0000000 --- a/playbooks/roles/services/backups/tasks/main.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -- name: "play:services : role:backups:{{ services_service_name }} : tasks:vars" - ansible.builtin.import_role: - name: "services/include" - vars_from: "user" - tags: - - "services:backups:snapshots" - - "services:backups:{{ services_service_name }}:snapshots" - - "services:{{ services_service_name }}:backups:snapshots" - - "services:backups:restic" - - "services:backups:{{ services_service_name }}:restic" - - "services:{{ services_service_name }}:backups:restic" - -- name: "play:services : role:backups : tasks:snapshots" - ansible.builtin.import_tasks: "include/snapshots.yml" - tags: - - "services:backups:snapshots" - - "services:backups:{{ services_service_name }}:snapshots" - - "services:{{ services_service_name }}:backups:snapshots" - -- name: "play:services : role:backups : tasks:restic" - ansible.builtin.import_tasks: "include/restic.yml" - tags: - - "services:backups:restic" - - "services:backups:{{ services_service_name }}:restic" - - "services:{{ services_service_name }}:backups:restic" diff --git a/playbooks/services.yml b/playbooks/services.yml index 6cef17c..1f4a66b 100644 --- a/playbooks/services.yml +++ b/playbooks/services.yml @@ -91,15 +91,35 @@ loop_var: "services_service_name" tags: "always" - - name: "backups" + - name: "backups : snapshots" ansible.builtin.include_role: - name: "services/backups" + name: "services/backups/snapshots" apply: tags: - "services:{{ services_service_name }}" - "services:backups" - - "services:backups:{{ services_service_name }}" - - "services:{{ services_service_name }}:backups" + - "services:backups:snapshots" + - "services:backups:snapshots:{{ services_service_name }}" + - "services:{{ services_service_name }}:backups:snapshots" + loop: "{{ services_host_services | dict2items | map(attribute='key') }}" + loop_control: + loop_var: "services_service_name" + tags: "always" + +- name: "services : restic" + hosts: "restic" + + tasks: + - name: "backups : restic" + ansible.builtin.include_role: + name: "services/backups/restic" + apply: + tags: + - "services:{{ services_service_name }}" + - "services:backups" + - "services:backups:restic" + - "services:backups:restic:{{ services_service_name }}" + - "services:{{ services_service_name }}:backups:restic" loop: "{{ services_host_services | dict2items | map(attribute='key') }}" loop_control: loop_var: "services_service_name"