Initial commit

ansible.cfg

@@ -0,0 +1,14 @@
+[defaults]
+#ask_vault_pass = True
+roles_path = ./roles
+inventory = ./hosts
+ansible_managed = Ansible managed
+remote_user = root
+timeout = 60
+
+[diff]
+always = yes
+
+[ssh_connection]
+pipelining = True
+retries = 3

backups.yml

@@ -0,0 +1,43 @@
+#!/usr/bin/env ansible-playbook
+---
+- hosts:
+    - dodecagon
+  roles:
+    - borgmatic
+  vars:
+    borg__keep_hourly: 24
+    borg__keep_daily: 30
+    borg__keep_weekly: 26
+    borg__keep_monthly: 36
+    borg__backup_dirs:
+      - /etc
+      - /home
+      - /root
+      - /opt
+      - /srv
+      - /var/backups
+      - /var/log
+      - /var/local
+      - /var/lib
+      - /var/mail
+      - /var/www
+    borg__passphrase: UK2mu1faYKQZeBkePbKjxdhuIKBjQriP
+    borg__targets:
+      - name: harpagon.infra.federez.net
+        hostkeys:
+          - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH1qDEAEJZ0qDRUq4yeHar5LKFTtsvHJIt2a54TBB/Lz
+        user: borgmatic
+        path: "/backup/borgmatic/{{ inventory_hostname }}"
+      - name: memoragon.infra.federez.net
+        hostkeys:
+          - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINdqX4I1JyvhC6dySHLnW1IioYk1ZqltFlbDCygozrWx
+        user: borgmatic
+        path: "/backup/borgmatic/{{ inventory_hostname }}"
+    # borg__postgresql:
+    #   - name: all
+    #     username: postgres
+    borg__mysql:
+      - name: all
+        username: root
+        password: KtQ7wHTecNPCMDISEJ1x
+...

group_vars/all.yml

@@ -0,0 +1,8 @@
+---
+ansible_python_interpreter: /usr/bin/python3
+ansible_host: "{{ inventory_hostname }}.federez.net"
+infra__byte_a: "{{ infra__id // 256 }}"
+infra__byte_b: "{{ infra__id % 256 }}"
+infra__addr_v6: "fd0a:66d3:1c19:42::{{ infra__byte_a }}:{{ infra__byte_b }}"
+infra__addr_v4: "10.42.{{ infra__byte_a }}.{{ infra__byte_b }}"
+...

host_vars/carlosgon.yml

@@ -0,0 +1,3 @@
+---
+infra__id: 13
+...

host_vars/dodecagon.yml

@@ -0,0 +1,3 @@
+---
+infra__id: 4
+...

host_vars/dragon.yml

@@ -0,0 +1,3 @@
+---
+infra__id: 16
+...

host_vars/glucagon.yml

@@ -0,0 +1,3 @@
+---
+infra__id: 2
+...

host_vars/harpagon.yml

@@ -0,0 +1,3 @@
+---
+infra__id: 6
+...

host_vars/hendecagon.yml

@@ -0,0 +1,3 @@
+---
+infra__id: 15
+...

host_vars/memoragon.yml

@@ -0,0 +1,3 @@
+---
+infra__id: 14
+...

host_vars/patagon.yml

@@ -0,0 +1,3 @@
+---
+infra__id: 7
+...

host_vars/ronderu.yml

@@ -0,0 +1,3 @@
+---
+infra__id: 3
+...

host_vars/saigon.yml

@@ -0,0 +1,3 @@
+---
+infra__id: 5
+...

hosts

@@ -0,0 +1,13 @@
+[debian]
+dragon
+dodecagon
+saigon
+harpagon
+patagon
+carlosgon
+memoragon
+hendecagon
+
+[pve]
+glucagon
+ronderu

monitoring.yml

@@ -0,0 +1,13 @@
+#!/usr/bin/env ansible-playbook
+---
+- hosts:
+    - debian
+    - pve
+  roles:
+    - prometheus_node_exporter
+  vars:
+    prometheus__listen_addrs:
+      - "{{ infra__addr_v6 }}"
+      #- "{{ infra__addr_v4 }}"
+    prometheus__smart_enabled: "{{ 'pve' in group_names }}"
+...

requirements.yml

@@ -0,0 +1,4 @@
+---
+collections:
+  - name: community.crypto
+...

roles/backports/tasks/main.yml

@@ -0,0 +1,17 @@
+---
+- name: Install python3-debian
+  ansible.builtin.apt:
+    name: python3-debian
+
+- name: Add backports repository
+  ansible.builtin.apt_repository:
+    filename: backports
+    repo: "deb {{ uri }} {{ suite }} {{ components | join(' ') }}"
+    update_cache: true
+  vars:
+    uri: http://deb.debian.org/debian
+    suite: "{{ ansible_distribution_release }}-backports"
+    components:
+      - main
+  when: "ansible_distribution_release in backports__supported_releases"
+...

roles/backports/vars/main.yml

@@ -0,0 +1,4 @@
+---
+backports__supported_releases:
+  - bullseye
+...

roles/borgmatic/defaults/main.yml

@@ -0,0 +1,10 @@
+---
+borg__targets: []
+borg__exclude_patterns: []
+borg__keep_hourly: 0
+borg__keep_daily: 0
+borg__keep_weekly: 0
+borg__keep_monthly: 0
+borg__mysql: []
+borg__postgresql: []
+...

roles/borgmatic/filter_plugins/borgmatic.py

@@ -0,0 +1,11 @@
+from urllib.parse import urlunsplit
+
+
+class FilterModule:
+    def filters(self):
+        return { "borg__to_repo": self.to_repo }
+
+    @staticmethod
+    def to_repo(target):
+        netloc = f"{target['user']}@{target['name']}"
+        return urlunsplit(("ssh", netloc, target["path"], None, None))

roles/borgmatic/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Run daemon-reload
+  ansible.builtin.systemd_service:
+    daemon_reload: true
+...

roles/borgmatic/tasks/main.yml

@@ -0,0 +1,136 @@
+---
+- name: Gather ansible_local facts
+  ansible.builtin.setup:
+    gather_subset: local
+
+- name: Install backports repository
+  ansible.builtin.include_role:
+    name: backports
+  when: "ansible_distribution_release in borg__backports_needed"
+
+- name: Install borgmatic
+  ansible.builtin.apt:
+    name: borgmatic
+    default_release: "{{ (release in borg__backports_needed)
+                        | ternary(release + '-backports', omit) }}"
+  vars:
+    release: "{{ ansible_distribution_release }}"
+
+- name: Install borgmatic
+  ansible.builtin.apt:
+    name: borgmatic
+  when: "ansible_distribution_release not in borg__backports_needed"
+
+- name: Create configuration directory for borgmatic
+  ansible.builtin.file:
+    path: /etc/borgmatic
+    state: directory
+    owner: root
+    group: root
+    mode: u=rwx,g=rx,o=
+
+- name: Create SSH key
+  community.crypto.openssh_keypair:
+    path: "/etc/borgmatic/remote"
+    type: ed25519
+    regenerate: full_idempotence
+    owner: root
+    group: root
+    mode: u=rw,g=,o=
+  register: ssh_key
+
+- name: Add server key to known hosts
+  ansible.builtin.known_hosts:
+    hash_host: true
+    host: "{{ item.0.name }}"
+    key: "{{ item.0.name }} {{ item.1 }}"
+  loop: "{{ borg__targets | subelements('hostkeys')  }}"
+
+- name: Wait for key deployment
+  block:
+    - name: Show the generated public key
+      ansible.builtin.debug:
+        msg: "{{ ssh_key.public_key }}"
+
+    - name: Please deploy the public key on every target
+      ansible.builtin.pause: null
+  when: "borg__targets
+         | map(attribute='name')
+         | difference(ansible_local.borgmatic_deployed_keys
+                      | default([]))
+         | count > 0"
+
+- name: Add borgmatic configuration file
+  ansible.builtin.template:
+    src: config.yaml.j2
+    dest: /etc/borgmatic/config.yaml
+    owner: root
+    group: root
+    mode: u=rw,g=r,o=
+  vars:
+    borg__config:
+      location:
+        source_directories: "{{ borg__backup_dirs }}"
+        exclude_patterns: "{{ borg__exclude_patterns }}"
+        repositories: "{{ borg__targets | map('borg__to_repo') }}"
+        borgmatic_source_directory: /tmp/borgmatic # TODO
+      storage:
+        encryption_passphrase: "{{ borg__passphrase }}"
+        ssh_command: "ssh -i /etc/borgmatic/remote"
+      retention:
+        keep_hourly: "{{ borg__keep_hourly }}"
+        keep_daily: "{{ borg__keep_daily }}"
+        keep_weekly: "{{ borg__keep_weekly }}"
+        keep_monthly: "{{ borg__keep_monthly }}"
+      consistency:
+        checks:
+          - repository
+          - archives
+      hooks:
+        postgresql_databases: "{{ borg__postgresql }}"
+        mysql_databases: "{{ borg__mysql }}"
+
+- name: Init repository
+  ansible.builtin.command: borgmatic init --encryption repokey
+
+- name: Create Ansible facts.d directory
+  ansible.builtin.file:
+    path: /etc/ansible/facts.d
+    state: directory
+    owner: root
+    group: root
+    mode: u=rwx,g=rx,o=
+
+- name: Check deployed keys fact
+  ansible.builtin.copy:
+    dest: /etc/ansible/facts.d/borgmatic_deployed_keys.fact
+    owner: root
+    group: root
+    content: "{{ borg__targets | map(attribute='name') }}"
+    mode: u=rw,g=r,o=
+
+- name: Create override directory
+  ansible.builtin.file:
+    path: /etc/systemd/system/borgmatic.timer.d
+    state: directory
+    owner: root
+    group: root
+    mode: u=rwx,g=rx,o=rx
+
+- name: Override borgmatic.timer
+  ansible.builtin.template:
+    src: override.conf.j2
+    dest: /etc/systemd/system/borgmatic.timer.d/override.conf
+    owner: root
+    group: root
+    mode: u=rw,g=r,o=r
+  notify:
+    - Run daemon-reload
+
+- name: Start and enable borgmatic timer
+  ansible.builtin.systemd_service:
+    name: borgmatic.timer
+    state: started
+    daemon_reload: true
+    enabled: true
+...

roles/borgmatic/templates/config.yaml.j2

@@ -0,0 +1,5 @@
+---
+{{ ansible_managed | comment }}
+
+{{ borg__config | to_nice_yaml }}
+...

roles/borgmatic/templates/override.conf.j2

@@ -0,0 +1,12 @@
+{{ ansible_managed | comment }}
+
+[Timer]
+OnCalendar=
+{% if borg__keep_hourly > 0 %}
+OnCalendar=hourly
+RandomizedDelaySec=20m
+{% else %}
+OnCalendar=daily
+RandomizedDelaySec=3h
+{% endif %}
+FixedRandomDelay=true

roles/borgmatic/vars/main.yml

@@ -0,0 +1,4 @@
+---
+borg__backports_needed:
+  - bullseye
+...

roles/prometheus_node_exporter/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+prometheus__listen_port: 9100
+prometheus__smart_enabled: false
+...

roles/prometheus_node_exporter/handlers/main.yml

@@ -0,0 +1,10 @@
+---
+- name: Run daemon-reload
+  ansible.builtin.systemd_service:
+    daemon_reload: true
+
+- name: Restart prometheus-node-exporter
+  ansible.builtin.systemd_service:
+    name: prometheus-node-exporter.service
+    state: restarted
+...

roles/prometheus_node_exporter/tasks/main.yml

@@ -0,0 +1,69 @@
+---
+- name: Install prometheus-node-exporter
+  ansible.builtin.apt:
+    name:
+      - prometheus-node-exporter
+      - prometheus-node-exporter-collectors
+    install_recommends: false  # Do not install smartmontools
+
+- name: Configure prometheus-node-exporter
+  ansible.builtin.template:
+    src: default.j2
+    dest: /etc/default/prometheus-node-exporter
+    owner: root
+    group: root
+    mode: u=rw,g=r,o=r
+  notify:
+    - Restart prometheus-node-exporter
+
+- name: Install smartmontools
+  ansible.builtin.apt:
+    name: smartmontools
+  when:
+    - "prometheus__smart_enabled"
+
+- name: Disable SMART monitoring
+  block:
+    - name: Get unit info for smartmon timer
+      command: systemctl list-unit-files --output=json prometheus-node-exporter-smartmon.timer
+      register: unit
+
+    - name: Disable prometheus-node-exporter-smartmon.timer
+      ansible.builtin.systemd_service:
+        name: prometheus-node-exporter-smartmon.timer
+        enabled: false
+        state: stopped
+      when: "unit.stdout | from_json | count > 0"
+
+    - name: Remove smartmon.prom
+      ansible.builtin.file:
+        path: /var/lib/prometheus/node-exporter/smartmon.prom
+        state: absent
+  when:
+    - "not prometheus__smart_enabled"
+
+- name: Create override directory
+  ansible.builtin.file:
+    path: /etc/systemd/system/prometheus-node-exporter.service.d
+    state: directory
+    owner: root
+    group: root
+    mode: u=rwx,g=rx,o=rx
+
+- name: Override prometheus-node-exporter.service
+  ansible.builtin.template:
+    src: override.conf.j2
+    dest: /etc/systemd/system/prometheus-node-exporter.service.d/override.conf
+    owner: root
+    group: root
+    mode: u=rw,g=r,o=r
+  notify:
+    - Run daemon-reload
+    - Restart prometheus-node-exporter
+
+- name: Enable prometheus-node-exporter
+  ansible.builtin.systemd_service:
+    name: prometheus-node-exporter
+    enabled: true
+    state: started
+...

roles/prometheus_node_exporter/templates/default.j2

@@ -0,0 +1,13 @@
+{{ ansible_managed | comment }}
+{# https://github.com/prometheus/exporter-toolkit/issues/91 #}
+{%
+  set listen =
+    ["--web.listen-address"]
+    | product(prometheus__listen_addrs
+    	      | ansible.utils.ipwrap
+	      | product([prometheus__listen_port])
+              | map("join", ":"))
+    | map("join", "=")
+%}
+
+ARGS="{{ listen | join(' ') }}"

roles/prometheus_node_exporter/templates/override.conf.j2

@@ -0,0 +1,5 @@
+{{ ansible_managed | comment }}
+
+[Unit]
+After=network-online.target
+Wants=network-online.target