adding separate deploy_portainer_stack role

host_vars/qnap.yml

@@ -32,6 +32,8 @@ services:
     endpoint_id: 2
   - name: nextcloud
     endpoint_id: 2
+    external_docker_networks:
+      - mariadb_net
   - name: dashboards
     endpoint_id: 2
   - name: nginx-proxy-manager
@@ -44,8 +46,12 @@ services:
     endpoint_id: 2
   - name: mariadb
     endpoint_id: 2
+    external_docker_networks:
+      - mariadb_net
   - name: photoprism
     endpoint_id: 2
+    external_docker_networks:
+      - mariadb_net
   - name: olivetin
     endpoint_id: 2
 

playbooks/setup-homelab.yml

@@ -1,6 +1,6 @@
 ---
 - name: Update packages and ensure users on all hosts
-  tags: [always]
+#  tags: [always]
   hosts: all
   become: true
   pre_tasks:

roles/deploy_portainer_stack/defaults/main.yml

@@ -0,0 +1,12 @@
+---
+# defaults file for chatton.deploy_portainer_stack
+
+portainer_stack_name: ""
+# required directories for this stack
+portainer_stack_directories: []
+
+# variables passed to the template function
+portainer_stack_template_vars: {}
+
+# list of docker networks which should be created
+portainer_stack_external_docker_networks: []

roles/deploy_portainer_stack/files/scripts/find-volumes-to-restore.py

@@ -0,0 +1,40 @@
+#!/usr/bin/python
+import os
+import yaml
+
+from pathlib import Path
+
+
+def main():
+    existing = eval(os.getenv("EXISTING_VOLUMES"))
+    stack_name = os.getenv("STACK_NAME")
+    docker_compose_dir = os.getenv("DOCKER_COMPOSE_DIR")
+
+    full_volume_names = []
+    missing_volumes = []
+    services = [{"name": stack_name}]
+    for service in services:
+        service_name = service["name"]
+        docker_file = f"{docker_compose_dir}/{service_name}/docker-compose.yml"
+        docker_compose_dict = yaml.safe_load(Path(docker_file).read_text())
+
+        # no volumes specified in the compose file
+        if "volumes" not in docker_compose_dict:
+            continue
+
+        volumes = docker_compose_dict["volumes"]
+
+        for v in volumes:
+            full_volume_names.append(f"{service_name}_{v}")
+
+    for volume_name in full_volume_names:
+        if volume_name not in existing:
+            missing_volumes.append(volume_name)
+
+    for mv in missing_volumes:
+        if mv:
+            print(mv)
+
+
+if __name__ == "__main__":
+    main()

roles/deploy_portainer_stack/meta/main.yml

@@ -0,0 +1,12 @@
+galaxy_info:
+  author: Cian Hatton
+  namespace: chatton
+  description: Deploy a single portainer stack.
+  license: MIT
+  min_ansible_version: "2.1"
+  galaxy_tags: []
+  platforms:
+    - name: Debian
+      versions:
+        - all
+dependencies: []

roles/deploy_portainer_stack/tasks/main.yml

@@ -0,0 +1,69 @@
+---
+- name: Docker Compose | Create a directory if it does not exist
+  ansible.builtin.file:
+    path: '{{ directories.docker_compose_directory }}/{{ portainer_stack_name }}'
+    state: directory
+    mode: '0755'
+
+- name: Docker Compose | Template Docker Compose Files
+  ansible.builtin.template:
+    src: '{{ portainer_stack_name }}.j2'
+    dest: '{{ directories.docker_compose_directory }}/{{ portainer_stack_name }}/docker-compose.yml'
+    owner: root
+    group: root
+    mode: 0440
+  vars:
+    template_vars: "{{ portainer_stack_template_vars }}"
+
+- name: Directories | Ensure required directories
+  ansible.builtin.file:
+    path: '{{ item }}'
+    state: directory
+    mode: '0755'
+  with_items: "{{ portainer_stack_directories }}"
+
+- name: Docker | Find docker volumes
+  ansible.builtin.shell: docker volume ls -f name={{ portainer_stack_name }} --format '{{ '{{' }}  .Name {{ '}}' }}'
+  register: find_volumes
+  changed_when: false
+
+- name: Docker | Find volumes that need to be restored
+  ansible.builtin.script: scripts/find-volumes-to-restore.py
+  environment:
+    EXISTING_VOLUMES: "{{ find_volumes.stdout_lines }}"
+    STACK_NAME: '{{ portainer_stack_name }}'
+    DOCKER_COMPOSE_DIR: '{{ directories.docker_compose_directory }}'
+  args:
+    executable: python3
+  register: python_output
+  changed_when: false
+
+- name: Build list of volumes to restore.
+  ansible.builtin.set_fact:
+    restore_volumes: "{{ restore_volumes | default([]) + [{'volume_name':item}] }}"
+  with_items: "{{ python_output.stdout_lines | list }}"
+
+- name: Restore any missing volumes from S3
+  ansible.builtin.include_role:
+    name: chatton.docker_backup.docker_s3_volume_restore
+  when: restore_volumes is defined
+  vars:
+    docker_backup_restore_force: false
+    docker_backup_restore_latest_s3_key: true
+    docker_backup_s3_restores: "{{ restore_volumes }}"
+
+- name: Docker | Create required docker networks
+  docker_network:
+    name: '{{ docker_network }}'
+  loop: '{{ portainer_stack_external_docker_networks }}'
+  loop_control:
+    loop_var: docker_network
+
+- name: Portainer | Update Stack
+  chatton.portainer.portainer_stack:
+    username: admin
+    password: '{{ portainer.password }}'
+    docker_compose_file_path: '{{ directories.docker_compose_directory }}/{{ portainer_stack_name }}/docker-compose.yml'
+    stack_name: '{{ portainer_stack_name }}'
+    endpoint_id: '{{ portainer_stack_endpoint_id }}'
+    state: present

roles/setup_hosted_services/tasks/main.yml

@@ -7,23 +7,6 @@
     - ubuntu
     - busybox
 
-- name: Docker Compose | Create a directory if it does not exist
-  ansible.builtin.file:
-    path: '{{ directories.docker_compose_directory }}/{{ item.name }}'
-    state: directory
-    mode: '0755'
-  with_items: '{{ services }}'
-
-- name: Docker Compose | Template Docker Compose Files
-  ansible.builtin.template:
-    src: '{{ item.name }}.j2'
-    dest: '{{ directories.docker_compose_directory }}/{{ item.name }}/docker-compose.yml'
-    owner: root
-    group: root
-    mode: 0440
-  with_items: '{{ services }}'
-
-
 - name: Directories | Ensure required directories
   ansible.builtin.file:
     path: '{{ item }}'
@@ -54,65 +37,15 @@
   notify:
     - restart-olivetin
 
-- name: Install python dependencies (requests)
-  ansible.builtin.pip:
-    name: requests
-
-- name: Docker | Find docker volumes
-  ansible.builtin.shell: docker volume ls -f name={{ item.name }} --format '{{ '{{' }}  .Name {{ '}}' }}'
-  with_items: '{{ services }}'
-  register: find_volumes
-  changed_when: false
-
-- name: Docker | Find volumes that need to be restored
-  ansible.builtin.script: scripts/find-volumes-to-restore.py
-  environment:
-    EXISTING_VOLUMES: "{{ find_volumes.results | map(attribute='stdout_lines') | list | flatten }}"
-    SERVICES: '{{ services }}'
-    DOCKER_COMPOSE_DIR: '{{ directories.docker_compose_directory }}'
-  args:
-    executable: python3
-  register: python_output
-  changed_when: false
-
-- name: Build list of volumes to restore.
-  ansible.builtin.set_fact:
-    restore_volumes: "{{ restore_volumes | default([]) + [{'volume_name':item}] }}"
-  with_items: "{{ python_output.stdout_lines | list }}"
-
-- name: Restore any missing volumes from S3
+- name: Deploy Portainer stacks.
   ansible.builtin.include_role:
-    name: chatton.docker_backup.docker_s3_volume_restore
-  when: restore_volumes is defined
-  #  ignore_errors: true
+    name: deploy_portainer_stack
   vars:
-    docker_backup_restore_force: false
-    docker_backup_restore_latest_s3_key: true
-    docker_backup_s3_restores: "{{ restore_volumes }}"
-
-- name: Docker | Create required docker networks
-  docker_network:
-    name: '{{ item }}'
-  with_items: '{{ docker_networks }}'
-
-- name: Portainer | Update Stack
-  when: container_deployment_mode == "portainer"
-  chatton.portainer.portainer_stack:
-    username: admin
-    password: '{{ portainer.password }}'
-    docker_compose_file_path: '{{ directories.docker_compose_directory }}/{{ item.name }}/docker-compose.yml'
-    stack_name: '{{ item.name }}'
-    endpoint_id: '{{ item.endpoint_id }}'
-    state: present
-  with_items: '{{ services }}'
-
-- name: Docker compose | Update Stack
-  when: container_deployment_mode == "compose"
-  docker_compose:
-    project_src: '{{ directories.docker_compose_directory }}/{{ item.name }}'
-    state: present
-  with_items: '{{ services }}'
-
+    portainer_stack_name: "{{ item.name }}"
+    portainer_stack_endpoint_id: "{{ item.endpoint_id }}"
+    portainer_stack_external_docker_networks: "{{ item.external_docker_networks | default([]) }}"
+    portainer_stack_template_vars: "{{ item.template_vars | default({}) }}"
+  with_items: "{{ services }}"
 
 - name: Install Ansible pull
   tags: ["cron"]