From d230b4aa8e481b6ad85f2577eb9c8fa9e280616a Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Tue, 6 Sep 2022 21:11:08 +0100 Subject: [PATCH] adding separate deploy_portainer_stack role --- host_vars/qnap.yml | 6 ++ playbooks/setup-homelab.yml | 2 +- .../deploy_portainer_stack/defaults/main.yml | 12 +++ .../files/scripts/find-volumes-to-restore.py | 40 +++++++++ roles/deploy_portainer_stack/meta/main.yml | 12 +++ roles/deploy_portainer_stack/tasks/main.yml | 69 ++++++++++++++++ roles/setup_hosted_services/tasks/main.yml | 81 ++----------------- 7 files changed, 147 insertions(+), 75 deletions(-) create mode 100644 roles/deploy_portainer_stack/defaults/main.yml create mode 100644 roles/deploy_portainer_stack/files/scripts/find-volumes-to-restore.py create mode 100644 roles/deploy_portainer_stack/meta/main.yml create mode 100644 roles/deploy_portainer_stack/tasks/main.yml diff --git a/host_vars/qnap.yml b/host_vars/qnap.yml index 1461c1a..15bcd2e 100644 --- a/host_vars/qnap.yml +++ b/host_vars/qnap.yml @@ -32,6 +32,8 @@ services: endpoint_id: 2 - name: nextcloud endpoint_id: 2 + external_docker_networks: + - mariadb_net - name: dashboards endpoint_id: 2 - name: nginx-proxy-manager @@ -44,8 +46,12 @@ services: endpoint_id: 2 - name: mariadb endpoint_id: 2 + external_docker_networks: + - mariadb_net - name: photoprism endpoint_id: 2 + external_docker_networks: + - mariadb_net - name: olivetin endpoint_id: 2 diff --git a/playbooks/setup-homelab.yml b/playbooks/setup-homelab.yml index ec9dd84..eae1ce6 100644 --- a/playbooks/setup-homelab.yml +++ b/playbooks/setup-homelab.yml @@ -1,6 +1,6 @@ --- - name: Update packages and ensure users on all hosts - tags: [always] +# tags: [always] hosts: all become: true pre_tasks: diff --git a/roles/deploy_portainer_stack/defaults/main.yml b/roles/deploy_portainer_stack/defaults/main.yml new file mode 100644 index 0000000..2326e2a --- /dev/null +++ b/roles/deploy_portainer_stack/defaults/main.yml @@ -0,0 +1,12 @@ +--- +# defaults file for chatton.deploy_portainer_stack + +portainer_stack_name: "" +# required directories for this stack +portainer_stack_directories: [] + +# variables passed to the template function +portainer_stack_template_vars: {} + +# list of docker networks which should be created +portainer_stack_external_docker_networks: [] diff --git a/roles/deploy_portainer_stack/files/scripts/find-volumes-to-restore.py b/roles/deploy_portainer_stack/files/scripts/find-volumes-to-restore.py new file mode 100644 index 0000000..881c90f --- /dev/null +++ b/roles/deploy_portainer_stack/files/scripts/find-volumes-to-restore.py @@ -0,0 +1,40 @@ +#!/usr/bin/python +import os +import yaml + +from pathlib import Path + + +def main(): + existing = eval(os.getenv("EXISTING_VOLUMES")) + stack_name = os.getenv("STACK_NAME") + docker_compose_dir = os.getenv("DOCKER_COMPOSE_DIR") + + full_volume_names = [] + missing_volumes = [] + services = [{"name": stack_name}] + for service in services: + service_name = service["name"] + docker_file = f"{docker_compose_dir}/{service_name}/docker-compose.yml" + docker_compose_dict = yaml.safe_load(Path(docker_file).read_text()) + + # no volumes specified in the compose file + if "volumes" not in docker_compose_dict: + continue + + volumes = docker_compose_dict["volumes"] + + for v in volumes: + full_volume_names.append(f"{service_name}_{v}") + + for volume_name in full_volume_names: + if volume_name not in existing: + missing_volumes.append(volume_name) + + for mv in missing_volumes: + if mv: + print(mv) + + +if __name__ == "__main__": + main() diff --git a/roles/deploy_portainer_stack/meta/main.yml b/roles/deploy_portainer_stack/meta/main.yml new file mode 100644 index 0000000..616666d --- /dev/null +++ b/roles/deploy_portainer_stack/meta/main.yml @@ -0,0 +1,12 @@ +galaxy_info: + author: Cian Hatton + namespace: chatton + description: Deploy a single portainer stack. + license: MIT + min_ansible_version: "2.1" + galaxy_tags: [] + platforms: + - name: Debian + versions: + - all +dependencies: [] diff --git a/roles/deploy_portainer_stack/tasks/main.yml b/roles/deploy_portainer_stack/tasks/main.yml new file mode 100644 index 0000000..754e589 --- /dev/null +++ b/roles/deploy_portainer_stack/tasks/main.yml @@ -0,0 +1,69 @@ +--- +- name: Docker Compose | Create a directory if it does not exist + ansible.builtin.file: + path: '{{ directories.docker_compose_directory }}/{{ portainer_stack_name }}' + state: directory + mode: '0755' + +- name: Docker Compose | Template Docker Compose Files + ansible.builtin.template: + src: '{{ portainer_stack_name }}.j2' + dest: '{{ directories.docker_compose_directory }}/{{ portainer_stack_name }}/docker-compose.yml' + owner: root + group: root + mode: 0440 + vars: + template_vars: "{{ portainer_stack_template_vars }}" + +- name: Directories | Ensure required directories + ansible.builtin.file: + path: '{{ item }}' + state: directory + mode: '0755' + with_items: "{{ portainer_stack_directories }}" + +- name: Docker | Find docker volumes + ansible.builtin.shell: docker volume ls -f name={{ portainer_stack_name }} --format '{{ '{{' }} .Name {{ '}}' }}' + register: find_volumes + changed_when: false + +- name: Docker | Find volumes that need to be restored + ansible.builtin.script: scripts/find-volumes-to-restore.py + environment: + EXISTING_VOLUMES: "{{ find_volumes.stdout_lines }}" + STACK_NAME: '{{ portainer_stack_name }}' + DOCKER_COMPOSE_DIR: '{{ directories.docker_compose_directory }}' + args: + executable: python3 + register: python_output + changed_when: false + +- name: Build list of volumes to restore. + ansible.builtin.set_fact: + restore_volumes: "{{ restore_volumes | default([]) + [{'volume_name':item}] }}" + with_items: "{{ python_output.stdout_lines | list }}" + +- name: Restore any missing volumes from S3 + ansible.builtin.include_role: + name: chatton.docker_backup.docker_s3_volume_restore + when: restore_volumes is defined + vars: + docker_backup_restore_force: false + docker_backup_restore_latest_s3_key: true + docker_backup_s3_restores: "{{ restore_volumes }}" + +- name: Docker | Create required docker networks + docker_network: + name: '{{ docker_network }}' + loop: '{{ portainer_stack_external_docker_networks }}' + loop_control: + loop_var: docker_network + +- name: Portainer | Update Stack + chatton.portainer.portainer_stack: + username: admin + password: '{{ portainer.password }}' + docker_compose_file_path: '{{ directories.docker_compose_directory }}/{{ portainer_stack_name }}/docker-compose.yml' + stack_name: '{{ portainer_stack_name }}' + endpoint_id: '{{ portainer_stack_endpoint_id }}' + state: present diff --git a/roles/setup_hosted_services/tasks/main.yml b/roles/setup_hosted_services/tasks/main.yml index f8e46e6..d6da6e9 100644 --- a/roles/setup_hosted_services/tasks/main.yml +++ b/roles/setup_hosted_services/tasks/main.yml @@ -7,23 +7,6 @@ - ubuntu - busybox -- name: Docker Compose | Create a directory if it does not exist - ansible.builtin.file: - path: '{{ directories.docker_compose_directory }}/{{ item.name }}' - state: directory - mode: '0755' - with_items: '{{ services }}' - -- name: Docker Compose | Template Docker Compose Files - ansible.builtin.template: - src: '{{ item.name }}.j2' - dest: '{{ directories.docker_compose_directory }}/{{ item.name }}/docker-compose.yml' - owner: root - group: root - mode: 0440 - with_items: '{{ services }}' - - - name: Directories | Ensure required directories ansible.builtin.file: path: '{{ item }}' @@ -54,65 +37,15 @@ notify: - restart-olivetin -- name: Install python dependencies (requests) - ansible.builtin.pip: - name: requests - -- name: Docker | Find docker volumes - ansible.builtin.shell: docker volume ls -f name={{ item.name }} --format '{{ '{{' }} .Name {{ '}}' }}' - with_items: '{{ services }}' - register: find_volumes - changed_when: false - -- name: Docker | Find volumes that need to be restored - ansible.builtin.script: scripts/find-volumes-to-restore.py - environment: - EXISTING_VOLUMES: "{{ find_volumes.results | map(attribute='stdout_lines') | list | flatten }}" - SERVICES: '{{ services }}' - DOCKER_COMPOSE_DIR: '{{ directories.docker_compose_directory }}' - args: - executable: python3 - register: python_output - changed_when: false - -- name: Build list of volumes to restore. - ansible.builtin.set_fact: - restore_volumes: "{{ restore_volumes | default([]) + [{'volume_name':item}] }}" - with_items: "{{ python_output.stdout_lines | list }}" - -- name: Restore any missing volumes from S3 +- name: Deploy Portainer stacks. ansible.builtin.include_role: - name: chatton.docker_backup.docker_s3_volume_restore - when: restore_volumes is defined - # ignore_errors: true + name: deploy_portainer_stack vars: - docker_backup_restore_force: false - docker_backup_restore_latest_s3_key: true - docker_backup_s3_restores: "{{ restore_volumes }}" - -- name: Docker | Create required docker networks - docker_network: - name: '{{ item }}' - with_items: '{{ docker_networks }}' - -- name: Portainer | Update Stack - when: container_deployment_mode == "portainer" - chatton.portainer.portainer_stack: - username: admin - password: '{{ portainer.password }}' - docker_compose_file_path: '{{ directories.docker_compose_directory }}/{{ item.name }}/docker-compose.yml' - stack_name: '{{ item.name }}' - endpoint_id: '{{ item.endpoint_id }}' - state: present - with_items: '{{ services }}' - -- name: Docker compose | Update Stack - when: container_deployment_mode == "compose" - docker_compose: - project_src: '{{ directories.docker_compose_directory }}/{{ item.name }}' - state: present - with_items: '{{ services }}' - + portainer_stack_name: "{{ item.name }}" + portainer_stack_endpoint_id: "{{ item.endpoint_id }}" + portainer_stack_external_docker_networks: "{{ item.external_docker_networks | default([]) }}" + portainer_stack_template_vars: "{{ item.template_vars | default({}) }}" + with_items: "{{ services }}" - name: Install Ansible pull tags: ["cron"]