the-nine-worlds/ansible-edda
the-nine-worlds/ansible-edda
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Decouple backups from services

Browse Source
This commit is contained in:
Wojciech Kozlowski 2023-02-19 22:36:16 +01:00
parent 350e6514ae
commit d155da3414
20 changed files with 174 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
inventory/group_vars/restic/vars.yml
View File

@ -1,27 +1,30 @@
---
# --------------------------------------------------------------------------------------------------
# backups:restic
# --------------------------------------------------------------------------------------------------
backups_restic_user_aws_access_key_id: "{{ vault_backups_restic_user_aws_access_key_id }}"
backups_restic_user_aws_secret_access_key: "\
{{ vault_backups_restic_user_aws_secret_access_key }}"
backups_restic_user_aws_keys_file: "/etc/restic-aws-keys.yml"
backups_restic_user_aws_bucket_endpoint: "\
{{ vault_backups_restic_user_aws_bucket_endpoint }}"
backups_restic_user_restic_password: "{{ vault_backups_restic_user_restic_password }}"
backups_restic_user_restic_password_file: "/etc/restic.password"
backups_restic_user_restic_keep_daily: 30
backups_restic_user_restic_keep_monthly: 3
# --------------------------------------------------------------------------------------------------
# services:backups
# --------------------------------------------------------------------------------------------------
services_backups_restic_restic_password: "{{ vault_services_backups_restic_restic_password }}"
services_backups_restic_aws_access_key_id: "{{ vault_services_backups_restic_aws_access_key_id }}"
services_backups_restic_aws_secret_access_key: "\
{{ vault_services_backups_restic_aws_secret_access_key }}"
services_backups_restic_aws_bucket_endpoint: "\
{{ vault_services_backups_restic_aws_bucket_endpoint }}"
services_backups_restic_services: "\
{% set services_backups_restic_service = {} %}\
{% for service in services_host_services.keys() %}\
{{ services_backups_restic_service.update(
{ service: {
'aws_access_key_id': services_backups_restic_aws_access_key_id,
'aws_secret_access_key': services_backups_restic_aws_secret_access_key,
'aws_keys_file': '/etc/restic-aws-keys.yml',
'aws_bucket_endpoint': services_backups_restic_aws_bucket_endpoint,
'user_name': ( 'pod-' ~ service ),
'data_dataset': ( services_data_dataset ~ '/pod-' ~ service ),
'data_directory': ( services_data_directory ~ '/pod-' ~ service ),
'aws_bucket_prefix': ( 'the-nine-worlds---pod-' ~ service ),
'restic_password': services_backups_restic_restic_password,
'restic_password_file': '/etc/restic.password',
'restic_keep_daily': 30,
'restic_keep_monthly': 3,
}}
) }}\
{% endfor %}\

2
inventory/host_vars/yggdrasil/vars.yml
View File

@ -133,6 +133,8 @@ services_backups_snapshots_services: "\
{% for service in services_host_services.keys() %}\
{{ services_backups_snapshots_service.update(
{ service: {
'user_name': ( 'pod-' ~ service ),
'data_dataset': ( services_data_dataset ~ '/pod-' ~ service ),
'backup_dataset': ( services_backups_snapshots_data_dataset ~ '/pod-' ~ service ),
'recursive': true,
'skip_parent': true,

46
playbooks/roles/backups/restic/user/meta/argument_specs.yml Normal file
View File

@ -0,0 +1,46 @@
---
argument_specs:
main:
options:
ansible_hostname:
type: "str"
required: true
backups_restic_user_name:
type: "str"
required: true
backups_restic_user_use_dataset:
type: "bool"
required: true
backups_restic_user_data_dataset:
type: "str"
required: false
backups_restic_user_data_directory:
type: "str"
required: false
backups_restic_user_aws_access_key_id:
type: "str"
required: true
backups_restic_user_aws_secret_access_key:
type: "str"
required: true
backups_restic_user_aws_keys_file:
type: "str"
required: true
backups_restic_user_aws_bucket_endpoint:
type: "str"
required: true
backups_restic_user_aws_bucket_prefix:
type: "str"
required: true
backups_restic_user_restic_password:
type: "str"
required: true
backups_restic_user_restic_password_file:
type: "str"
required: true
backups_restic_user_restic_keep_daily:
type: "int"
required: true
backups_restic_user_restic_keep_monthly:
type: "int"
required: true

18
playbooks/roles/backups/restic/user/tasks/main.yml Normal file
View File

@ -0,0 +1,18 @@
---
- name: "create restic password file"
ansible.builtin.template:
src: "./restic.password.j2"
dest: "{{ backups_restic_user_restic_password_file }}"
mode: 0600
- name: "create aws key file"
ansible.builtin.template:
src: "./restic-aws-keys.yml.j2"
dest: "{{ backups_restic_user_aws_keys_file }}"
mode: 0600
- name: "configure service restic backups"
ansible.builtin.template:
src: "./volumes.yml.j2"
dest: "/etc/restic-batch.d/{{ backups_restic_user_name }}.yml"
mode: 0644

2
playbooks/roles/backups/restic/user/templates/restic-aws-keys.yml.j2 Normal file
View File

@ -0,0 +1,2 @@
AWS_ACCESS_KEY_ID: {{ backups_restic_user_aws_access_key_id }}
AWS_SECRET_ACCESS_KEY: {{ backups_restic_user_aws_secret_access_key }}

1
playbooks/roles/backups/restic/user/templates/restic.password.j2 Normal file
View File

@ -0,0 +1 @@
{{ backups_restic_user_restic_password }}

11
playbooks/roles/backups/restic/user/templates/volumes.yml.j2 Normal file
View File

@ -0,0 +1,11 @@
{% if backups_restic_user_use_dataset %}
dataset: {{ backups_restic_user_data_dataset }}
{% else %}
directory: {{ backups_restic_user_data_directory }}
{% endif %}
aws_bucket_keys_file: {{ backups_restic_user_aws_keys_file }}
aws_bucket_endpoint: {{ backups_restic_user_aws_bucket_endpoint }}
aws_bucket_prefix: {{ backups_restic_user_aws_bucket_prefix }}
restic_password_file: {{ backups_restic_user_restic_password_file }}
restic_keep_daily: {{ backups_restic_user_restic_keep_daily }}
restic_keep_monthly: {{ backups_restic_user_restic_keep_monthly }}

22
playbooks/roles/backups/snapshots/user/meta/argument_specs.yml Normal file
View File

@ -0,0 +1,22 @@
---
argument_specs:
main:
options:
ansible_hostname:
type: "str"
required: true
backups_snapshots_user_name:
type: "str"
required: true
backups_snapshots_user_data_dataset:
type: "str"
required: true
backups_snapshots_user_backup_dataset:
type: "str"
required: true
backups_snapshots_user_recursive:
type: "bool"
required: true
backups_snapshots_user_skip_parent:
type: "bool"
required: true

22
playbooks/roles/backups/snapshots/user/tasks/main.yml Normal file
View File

@ -0,0 +1,22 @@
---
- name: "configure service sanoid snapshots"
ansible.builtin.blockinfile:
path: "/etc/sanoid/sanoid.conf"
insertbefore: "# BEGIN ANSIBLE MANAGED BLOCK TEMPLATES #"
marker: "# {mark} ANSIBLE MANAGED BLOCK USER {{ backups_snapshots_user_name }} #"
block: |
[{{ backups_snapshots_user_data_dataset }}]
use_template = production
recursive = yes
process_children_only = yes
[{{ backups_snapshots_user_backup_dataset }}]
use_template = backup
recursive = yes
process_children_only = yes
- name: "configure service syncoid snapshots"
ansible.builtin.template:
src: "./volumes.yml.j2"
dest: "/etc/syncoid-batch.d/{{ backups_snapshots_user_name }}.yml"
mode: 0644

4
playbooks/roles/backups/snapshots/user/templates/volumes.yml.j2 Normal file
View File

@ -0,0 +1,4 @@
dataset: {{ backups_snapshots_user_data_dataset }}
backup_dataset: {{ backups_snapshots_user_backup_dataset }}
recursive: {{ backups_snapshots_user_recursive }}
skip_parent: {{ backups_snapshots_user_skip_parent }}

4
playbooks/roles/services/backups/include/vars/datasets.yml
View File

@ -1,4 +0,0 @@
---
services_backups_user_data_dataset: "{{ services_data_dataset }}/{{ services_service_user_name }}"
services_backups_user_data_directory: "\
{{ services_data_directory }}/{{ services_service_user_name }}"

20
playbooks/roles/services/backups/restic/meta/argument_specs.yml
View File

@ -1,20 +0,0 @@
---
argument_specs:
main:
options:
ansible_hostname:
type: "str"
required: true
services_service_name:
type: "str"
required: true
services_data_dataset:
type: "str"
required: false
services_data_directory:
type: "str"
required: false
services_backups_restic_services:
type: "dict"
elem: "dict"
required: true

28
playbooks/roles/services/backups/restic/tasks/main.yml
View File

@ -1,28 +0,0 @@
---
- name: "{{ services_service_name }} : tasks:vars"
ansible.builtin.import_role:
name: "services/include"
vars_from: "user"
- name: "{{ services_service_name }} : tasks:vars"
ansible.builtin.import_role:
name: "services/backups/include"
vars_from: "datasets"
- name: "{{ services_service_name }} : create restic password file"
ansible.builtin.template:
src: "./restic.password.j2"
dest: "{{ services_backups_restic_services[services_service_name].restic_password_file }}"
mode: 0600
- name: "{{ services_service_name }} : create aws key file"
ansible.builtin.template:
src: "./restic-aws-keys.yml.j2"
dest: "{{ services_backups_restic_services[services_service_name].aws_keys_file }}"
mode: 0600
- name: "{{ services_service_name }} : configure service restic backups"
ansible.builtin.template:
src: "./volumes.yml.j2"
dest: "/etc/restic-batch.d/{{ services_service_user_name }}.yml"
mode: 0644

2
playbooks/roles/services/backups/restic/templates/restic-aws-keys.yml.j2
View File

@ -1,2 +0,0 @@
AWS_ACCESS_KEY_ID: {{ services_backups_restic_services[services_service_name].aws_access_key_id }}
AWS_SECRET_ACCESS_KEY: {{ services_backups_restic_services[services_service_name].aws_secret_access_key }}

1
playbooks/roles/services/backups/restic/templates/restic.password.j2
View File

@ -1 +0,0 @@
{{ services_backups_restic_services[services_service_name].restic_password }}

11
playbooks/roles/services/backups/restic/templates/volumes.yml.j2
View File

@ -1,11 +0,0 @@
{% if services_data_dataset is defined %}
dataset: {{ services_backups_user_data_dataset }}
{% else %}
directory: {{ services_backups_user_data_directory }}
{% endif %}
aws_bucket_keys_file: {{ services_backups_restic_services[services_service_name].aws_keys_file }}
aws_bucket_endpoint: {{ services_backups_restic_services[services_service_name].aws_bucket_endpoint }}
aws_bucket_prefix: {{ services_backups_restic_services[services_service_name].aws_bucket_prefix }}
restic_password_file: {{ services_backups_restic_services[services_service_name].restic_password_file }}
restic_keep_daily: {{ services_backups_restic_services[services_service_name].restic_keep_daily }}
restic_keep_monthly: {{ services_backups_restic_services[services_service_name].restic_keep_monthly }}

17
playbooks/roles/services/backups/snapshots/meta/argument_specs.yml
View File

@ -1,17 +0,0 @@
---
argument_specs:
main:
options:
ansible_hostname:
type: "str"
required: true
services_service_name:
type: "str"
required: true
services_data_dataset:
type: "str"
required: true
services_backups_snapshots_services:
type: "dict"
elem: "dict"
required: true

32
playbooks/roles/services/backups/snapshots/tasks/main.yml
View File

@ -1,32 +0,0 @@
---
- name: "{{ services_service_name }} : tasks:vars"
ansible.builtin.import_role:
name: "services/include"
vars_from: "user"
- name: "{{ services_service_name }} : tasks:vars"
ansible.builtin.import_role:
name: "services/backups/include"
vars_from: "datasets"
- name: "{{ services_service_name }} : configure service sanoid snapshots"
ansible.builtin.blockinfile:
path: "/etc/sanoid/sanoid.conf"
insertbefore: "# BEGIN ANSIBLE MANAGED BLOCK TEMPLATES #"
marker: "# {mark} ANSIBLE MANAGED BLOCK SERVICE {{ services_service_name }} #"
block: |
[{{ services_backups_user_data_dataset }}]
use_template = production
recursive = yes
process_children_only = yes
[{{ services_backups_snapshots_services[services_service_name].backup_dataset }}]
use_template = backup
recursive = yes
process_children_only = yes
- name: "{{ services_service_name }} : configure service syncoid snapshots"
ansible.builtin.template:
src: "./volumes.yml.j2"
dest: "/etc/syncoid-batch.d/{{ services_service_user_name }}.yml"
mode: 0644

4
playbooks/roles/services/backups/snapshots/templates/volumes.yml.j2
View File

@ -1,4 +0,0 @@
dataset: {{ services_backups_user_data_dataset }}
backup_dataset: {{ services_backups_snapshots_services[services_service_name].backup_dataset }}
recursive: {{ services_backups_snapshots_services[services_service_name].recursive }}
skip_parent: {{ services_backups_snapshots_services[services_service_name].skip_parent }}

36
playbooks/services.yml
View File

@ -79,14 +79,26 @@
- name: "backups : snapshots"
ansible.builtin.include_role:
name: "services/backups/snapshots"
name: "backups/snapshots/user"
apply:
tags:
- "services:{{ services_service_name }}"
- "services:backups"
- "services:backups:snapshots"
- "services:backups:snapshots:{{ services_service_name }}"
- "services:{{ services_service_name }}:backups:snapshots"
- "services:backups:snapshots:user"
- "services:backups:snapshots:user:{{ services_service_name }}"
- "services:{{ services_service_name }}:backups:snapshots:user"
vars:
backups_snapshots_user_name: "\
{{ services_backups_snapshots_services[services_service_name].user_name }}"
backups_snapshots_user_data_dataset: "\
{{ services_backups_snapshots_services[services_service_name].data_dataset }}"
backups_snapshots_user_backup_dataset: "\
{{ services_backups_snapshots_services[services_service_name].backup_dataset }}"
backups_snapshots_user_recursive: "\
{{ services_backups_snapshots_services[services_service_name].recursive }}"
backups_snapshots_user_skip_parent: "\
{{ services_backups_snapshots_services[services_service_name].skip_parent }}"
loop: "{{ services_host_services | dict2items | map(attribute='key') }}"
loop_control:
loop_var: "services_service_name"
@ -98,14 +110,24 @@
tasks:
- name: "backups : restic"
ansible.builtin.include_role:
name: "services/backups/restic"
name: "backups/restic/user"
apply:
tags:
- "services:{{ services_service_name }}"
- "services:backups"
- "services:backups:restic"
- "services:backups:restic:{{ services_service_name }}"
- "services:{{ services_service_name }}:backups:restic"
- "services:backups:restic:user"
- "services:backups:restic:user:{{ services_service_name }}"
- "services:{{ services_service_name }}:backups:restic:user"
vars:
backups_restic_user_name: "\
{{ services_backups_restic_services[services_service_name].user_name }}"
backups_restic_user_use_dataset: "{{ 'zfs' in group_names }}"
backups_restic_user_data_dataset: "\
{{ services_backups_restic_services[services_service_name].data_dataset }}"
backups_restic_user_data_directory: "\
{{ services_backups_restic_services[services_service_name].data_directory }}"
backups_restic_user_aws_bucket_prefix: "\
{{ services_backups_restic_services[services_service_name].aws_bucket_prefix }}"
loop: "{{ services_host_services | dict2items | map(attribute='key') }}"
loop_control:
loop_var: "services_service_name"