Restore backups from S3 when there is no volume present

ansible/roles/setup_hosted_services/defaults/main.yml

@@ -1,16 +1,22 @@
 ---
 docker_compose_directory: /etc/docker-compose
 services:
+  - name: gitea
+    volumes: ["gitea_data"]
   - name: mealie
-#  - name: linkding
+    volumes: ["mealie_data"]
-#  - name: overseerr
+  - name: linkding
-#  - name: nextcloud
+    volumes: ["linkding_data"]
+  - name: overseerr
+    volumes: ["overseerr_config"]
+  - name: nextcloud
+    volumes: ["nextcloud_data"]
 
 docker_networks:
   - nextcloud_net
 
 aws_s3:
-  s3_url: "https://l8x8.ie11.idrivee2-6.com"
+  s3_url: "l8x8.ie11.idrivee2-6.com"
   aws_access_key: "nyNMQ3fRMSV0bA1xw5uV"
   region: "us-east-1"
   bucket: "backups"

ansible/roles/setup_hosted_services/files/gitea/docker-compose.yml

@@ -0,0 +1,21 @@
+version: "3"
+
+services:
+  gitea:
+    labels:
+      ie.cianhatton.backup.enabled: "true"
+    image: gitea/gitea:1.16.9
+    container_name: gitea
+    environment:
+      - USER_UID=1000
+      - USER_GID=1000
+    restart: unless-stopped
+    volumes:
+      - data:/data
+      - /etc/timezone:/etc/timezone:ro
+      - /etc/localtime:/etc/localtime:ro
+    ports:
+      - "3000:3000"
+      - "222:22"
+volumes:
+  data:

ansible/roles/setup_hosted_services/files/scripts/find-volumes-to-restore.py

@@ -0,0 +1,20 @@
+#!/usr/bin/python
+import os
+
+
+def main():
+    existing = eval(os.getenv("EXISTING_VOLUMES"))
+    services = eval(os.getenv("SERVICES"))
+
+    missing_volumes = []
+    for service in services:
+        for volume_name in service.get("volumes", []):
+            if volume_name not in existing:
+                missing_volumes.append(volume_name)
+
+    for mv in missing_volumes:
+        print(mv)
+
+
+if __name__ == "__main__":
+    main()

ansible/roles/setup_hosted_services/tasks/main.yml

@@ -15,34 +15,44 @@
   ansible.builtin.pip:
     name: requests
 
-- name: Install python dependencies (boto3)
+- name: Find docker volumes
-  ansible.builtin.pip:
+  shell: docker volume ls -f name={{item.name}} --format '{{ '{{' }}  .Name {{ '}}' }}'
-    name: boto3
+  with_items: "{{services}}"
-
+  register: find_volumes
-- name: Find relevant volume(s) in S3
+  changed_when: False
-  amazon.aws.aws_s3:
+
-    bucket: "{{aws_s3.bucket}}"
+- debug: msg="{{find_volumes.results | map(attribute='stdout_lines') | list | flatten }}"
-    mode: list
+
-    region: "{{aws_s3.region}}"
+- name: Find volumes that need to be restored
-    s3_url: "{{aws_s3.s3_url}}"
+  script: scripts/find-volumes-to-restore.py
-    prefix: "mealie"
+  environment:
-    aws_access_key: "{{aws_s3.aws_access_key}}"
+    EXISTING_VOLUMES: "{{ find_volumes.results | map(attribute='stdout_lines') | list | flatten }}"
-    aws_secret_key: "{{aws_s3_secrets.aws_secret_key}}"
+    SERVICES: "{{ services }}"
-  register: s3_list_output
+  args:
-
+    executable: python3
-- debug: msg="{{s3_list_output.s3_keys}}"
+  register: python_output
-
+  changed_when: False
-- name: Download volume(s) from S3
+
-  amazon.aws.aws_s3:
+- debug: msg="{{python_output.stdout_lines | list }}"
-    bucket: "{{aws_s3.bucket}}"
+
-    object: "{{item}}"
+- name: Restore any missing backups from S3
-    dest: "/tmp/{{item}}"
+  docker_container:
-    mode: get
+    command: "restore-volume --s3 --volume {{item}}"
-    region: "{{aws_s3.region}}"
+    image: "ghcr.io/chatton/docker-volume-backup:v0.3.0"
-    s3_url: "{{aws_s3.s3_url}}"
+    name: "s3-restore-{{item}}"
-    aws_access_key: "{{aws_s3.aws_access_key}}"
+    cleanup: true # delete container after it's done.
-    aws_secret_key: "{{aws_s3_secrets.aws_secret_key}}"
+    state: started # container should execute.
-  with_items: "{{s3_list_output.s3_keys}}"
+    detach: no # task fails if container exits.
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /tmp:/tmp # temp s3 archive goes here
+    env:
+      AWS_ACCESS_KEY_ID: "{{aws_s3.aws_access_key}}"
+      AWS_SECRET_ACCESS_KEY: "{{aws_s3_secrets.aws_secret_key}}"
+      AWS_DEFAULT_REGION: "{{aws_s3.region}}"
+      AWS_BUCKET: "{{aws_s3.bucket}}"
+      AWS_ENDPOINT: "{{aws_s3.s3_url}}"
+  with_items: "{{ python_output.stdout_lines  }}"
 
 - name: Create required docker networks
   docker_network:

ansible/setup-homelab.yml

@@ -4,7 +4,7 @@
   vars_files:
     - ../secrets.yml
   roles:
-#    - role: 'roles/setup_users'
+    - role: 'roles/setup_users'
-#    - role: 'roles/setup_docker'
+    - role: 'roles/setup_docker'
-#    - role: 'roles/setup_portainer'
+    - role: 'roles/setup_portainer'
     - role: 'roles/setup_hosted_services'