From 099c29e7d4d0677a8198940c35f7ca63fc15ed2f Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Thu, 1 Sep 2022 16:50:33 +0100 Subject: [PATCH] wip --- Makefile | 6 + playbooks/backup-docker-volumes.yml | 11 + playbooks/restore-docker-volumes.yml | 11 + playbooks/setup-homelab.yml | 5 +- playbooks/verify-homelab.yml | 3 +- requirements.yml | 3 + roles/docker_backup/defaults/main.yml | 2 + roles/docker_backup/handlers/main.yml | 2 + roles/docker_backup/meta/main.yml | 52 ++++ roles/docker_backup/tasks/main.yml | 54 ++++ roles/docker_backup/vars/main.yml | 2 + roles/docker_restore/defaults/main.yml | 2 + roles/docker_restore/handlers/main.yml | 2 + roles/docker_restore/meta/main.yml | 52 ++++ roles/docker_restore/tasks/main.yml | 121 +++++++++ roles/docker_restore/vars/main.yml | 2 + roles/sprat.mergerfs/.flake8 | 2 + roles/sprat.mergerfs/.github/workflows/ci.yml | 68 +++++ roles/sprat.mergerfs/.gitignore | 5 + roles/sprat.mergerfs/.yamllint.yml | 9 + roles/sprat.mergerfs/LICENSE | 20 ++ roles/sprat.mergerfs/README.md | 54 ++++ roles/sprat.mergerfs/defaults/main.yml | 23 ++ .../sprat.mergerfs/meta/.galaxy_install_info | 2 + roles/sprat.mergerfs/meta/main.yml | 30 +++ .../molecule/default/converge.yml | 11 + .../molecule/default/molecule.yml | 21 ++ .../molecule/default/prepare.yml | 25 ++ .../molecule/default/tests/test_default.py | 21 ++ roles/sprat.mergerfs/requirements.in | 1 + roles/sprat.mergerfs/requirements.txt | 249 ++++++++++++++++++ .../tasks/install_from_github_releases.yml | 54 ++++ .../tasks/install_from_package_manager.yml | 7 + roles/sprat.mergerfs/tasks/main.yml | 34 +++ roles/sprat.mergerfs/vars/Debian.yml | 12 + roles/sprat.mergerfs/vars/RedHat.yml | 7 + vault_vars/qnap-vault.yml | 105 +++++--- 37 files changed, 1044 insertions(+), 46 deletions(-) create mode 100644 playbooks/backup-docker-volumes.yml create mode 100644 playbooks/restore-docker-volumes.yml create mode 100644 roles/docker_backup/defaults/main.yml create mode 100644 roles/docker_backup/handlers/main.yml create mode 100644 roles/docker_backup/meta/main.yml create mode 100644 roles/docker_backup/tasks/main.yml create mode 100644 roles/docker_backup/vars/main.yml create mode 100644 roles/docker_restore/defaults/main.yml create mode 100644 roles/docker_restore/handlers/main.yml create mode 100644 roles/docker_restore/meta/main.yml create mode 100644 roles/docker_restore/tasks/main.yml create mode 100644 roles/docker_restore/vars/main.yml create mode 100644 roles/sprat.mergerfs/.flake8 create mode 100644 roles/sprat.mergerfs/.github/workflows/ci.yml create mode 100644 roles/sprat.mergerfs/.gitignore create mode 100644 roles/sprat.mergerfs/.yamllint.yml create mode 100644 roles/sprat.mergerfs/LICENSE create mode 100644 roles/sprat.mergerfs/README.md create mode 100644 roles/sprat.mergerfs/defaults/main.yml create mode 100644 roles/sprat.mergerfs/meta/.galaxy_install_info create mode 100644 roles/sprat.mergerfs/meta/main.yml create mode 100644 roles/sprat.mergerfs/molecule/default/converge.yml create mode 100644 roles/sprat.mergerfs/molecule/default/molecule.yml create mode 100644 roles/sprat.mergerfs/molecule/default/prepare.yml create mode 100644 roles/sprat.mergerfs/molecule/default/tests/test_default.py create mode 100644 roles/sprat.mergerfs/requirements.in create mode 100644 roles/sprat.mergerfs/requirements.txt create mode 100644 roles/sprat.mergerfs/tasks/install_from_github_releases.yml create mode 100644 roles/sprat.mergerfs/tasks/install_from_package_manager.yml create mode 100644 roles/sprat.mergerfs/tasks/main.yml create mode 100644 roles/sprat.mergerfs/vars/Debian.yml create mode 100644 roles/sprat.mergerfs/vars/RedHat.yml diff --git a/Makefile b/Makefile index 185c6f4..b59b93e 100644 --- a/Makefile +++ b/Makefile @@ -27,3 +27,9 @@ lint: ansible-lint group_vars ansible-lint roles ansible-lint playbooks + +backup: + ansible-playbook playbooks/backup-docker-volumes.yml + +restore: + ansible-playbook playbooks/restore-docker-volumes.yml diff --git a/playbooks/backup-docker-volumes.yml b/playbooks/backup-docker-volumes.yml new file mode 100644 index 0000000..01a5ffd --- /dev/null +++ b/playbooks/backup-docker-volumes.yml @@ -0,0 +1,11 @@ +--- +- hosts: qnap + become: true + pre_tasks: + - name: Include vault variables. + include_vars: '../{{vault_file}}' + tags: [always] + roles: + - role: docker_backup + vars: + container_backup: linkding diff --git a/playbooks/restore-docker-volumes.yml b/playbooks/restore-docker-volumes.yml new file mode 100644 index 0000000..b6b673a --- /dev/null +++ b/playbooks/restore-docker-volumes.yml @@ -0,0 +1,11 @@ +--- +- hosts: qnap + become: true + pre_tasks: + - name: Include vault variables. + include_vars: '../{{vault_file}}' + tags: [always] + roles: + - role: docker_restore + vars: + container_restore: linkding diff --git a/playbooks/setup-homelab.yml b/playbooks/setup-homelab.yml index 26ad76d..9f97cf4 100644 --- a/playbooks/setup-homelab.yml +++ b/playbooks/setup-homelab.yml @@ -1,3 +1,4 @@ +--- - name: Update packages and ensure users on all hosts tags: [always] hosts: all @@ -22,7 +23,7 @@ become: true pre_tasks: - name: Include vault variables. - include_vars: '{{vault_file}}' + include_vars: '../{{vault_file}}' tags: [always] roles: @@ -41,7 +42,7 @@ pre_tasks: - name: Include vault variables. - include_vars: '{{vault_file}}' + include_vars: '../{{vault_file}}' tags: [always] roles: diff --git a/playbooks/verify-homelab.yml b/playbooks/verify-homelab.yml index 4fd0c83..3a43a72 100644 --- a/playbooks/verify-homelab.yml +++ b/playbooks/verify-homelab.yml @@ -7,8 +7,7 @@ - always tasks: - name: Docker Compose Files Exist - command: >- - stat {{directories.docker_compose_directory}}/{{ item.name }}/docker-compose.yml + command: stat {{directories.docker_compose_directory}}/{{ item.name }}/docker-compose.yml with_items: '{{services}}' changed_when: false register: docker_compose_stat diff --git a/requirements.yml b/requirements.yml index fdebb8e..cc76756 100644 --- a/requirements.yml +++ b/requirements.yml @@ -1,4 +1,7 @@ --- +roles: + - src: sprat.mergerfs + collections: - name: https://github.com/chatton/ansible-portainer.git type: git diff --git a/roles/docker_backup/defaults/main.yml b/roles/docker_backup/defaults/main.yml new file mode 100644 index 0000000..85ffdba --- /dev/null +++ b/roles/docker_backup/defaults/main.yml @@ -0,0 +1,2 @@ +--- +# defaults file for chatton.docker_backup diff --git a/roles/docker_backup/handlers/main.yml b/roles/docker_backup/handlers/main.yml new file mode 100644 index 0000000..7bcee9b --- /dev/null +++ b/roles/docker_backup/handlers/main.yml @@ -0,0 +1,2 @@ +--- +# handlers file for chatton.docker_backup diff --git a/roles/docker_backup/meta/main.yml b/roles/docker_backup/meta/main.yml new file mode 100644 index 0000000..c572acc --- /dev/null +++ b/roles/docker_backup/meta/main.yml @@ -0,0 +1,52 @@ +galaxy_info: + author: your name + description: your role description + company: your company (optional) + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Choose a valid license ID from https://spdx.org - some suggested licenses: + # - BSD-3-Clause (default) + # - MIT + # - GPL-2.0-or-later + # - GPL-3.0-only + # - Apache-2.0 + # - CC-BY-4.0 + license: license (GPL-2.0-or-later, MIT, etc) + + min_ansible_version: 2.1 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + # + # Provide a list of supported platforms, and for each platform a list of versions. + # If you don't wish to enumerate all versions for a particular platform, use 'all'. + # To view available platforms and versions (or releases), visit: + # https://galaxy.ansible.com/api/v1/platforms/ + # + # platforms: + # - name: Fedora + # versions: + # - all + # - 25 + # - name: SomePlatform + # versions: + # - all + # - 1.0 + # - 7 + # - 99.99 + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. diff --git a/roles/docker_backup/tasks/main.yml b/roles/docker_backup/tasks/main.yml new file mode 100644 index 0000000..cecb03b --- /dev/null +++ b/roles/docker_backup/tasks/main.yml @@ -0,0 +1,54 @@ +--- +# tasks file for chatton.docker_backup +# https://docs.ansible.com/ansible/latest/collections/community/docker/docker_container_module.html#ansible-collections-community-docker-docker-container-module +# https://docs.docker.com/storage/volumes/#backup-restore-or-migrate-data-volumes + + +- set_fact: backup_time="{{ ansible_date_time.iso8601 }}" + +- name: Stop a container + community.docker.docker_container: + name: "{{ container_backup }}" + state: stopped + +- name: Get container details + docker_container_info: + name: "{{ container_backup }}" + register: result + +- name: Extract only the volume mounts (not bind mounts) + set_fact: volume_mounts="{{ result.container.Mounts | selectattr("Type", "equalto", "volume")}}" + +- name: Create Backup of Container Volumes + community.docker.docker_container: + name: "backup-container-{{ item.Name }}-{{ 10 | random }}" + image: ubuntu + command: "tar cvf /backups/{{ item.Name }}-{{ backup_time }}.tar.gz {{ item.Destination }}" + auto_remove: true + detach: false # block until this container exists. + state: started + volumes: + - /mnt/mergerfs/backups:/backups + volumes_from: + - "{{ container_backup }}" + with_items: "{{ volume_mounts }}" + +- name: Start the container + community.docker.docker_container: + name: "{{ container_backup }}" + state: started + +- name: Upload backups to S3 + register: upload_result + amazon.aws.aws_s3: + s3_url: "https://{{aws_s3.s3_url}}" + bucket: "{{ aws_s3.bucket }}" + object: "{{ item.Name }}/{{ item.Name }}-{{ backup_time }}.tar.gz" + src: /mnt/mergerfs/backups/{{ item.Name }}-{{ backup_time }}.tar.gz + aws_access_key: "{{ aws_s3.aws_access_key }}" + aws_secret_key: "{{ aws_s3.aws_secret_key }}" + region: "{{ aws_s3.region }}" + mode: put + # empty permissions makes it work with IAM vs ACL, see: https://github.com/ansible/ansible/issues/48050 + permission: [] + with_items: "{{ volume_mounts }}" diff --git a/roles/docker_backup/vars/main.yml b/roles/docker_backup/vars/main.yml new file mode 100644 index 0000000..045ab9b --- /dev/null +++ b/roles/docker_backup/vars/main.yml @@ -0,0 +1,2 @@ +--- +# vars file for chatton.docker_backup diff --git a/roles/docker_restore/defaults/main.yml b/roles/docker_restore/defaults/main.yml new file mode 100644 index 0000000..7377d0b --- /dev/null +++ b/roles/docker_restore/defaults/main.yml @@ -0,0 +1,2 @@ +--- +# defaults file for chatton.docker_restore diff --git a/roles/docker_restore/handlers/main.yml b/roles/docker_restore/handlers/main.yml new file mode 100644 index 0000000..d7dd3db --- /dev/null +++ b/roles/docker_restore/handlers/main.yml @@ -0,0 +1,2 @@ +--- +# handlers file for chatton.docker_restore diff --git a/roles/docker_restore/meta/main.yml b/roles/docker_restore/meta/main.yml new file mode 100644 index 0000000..c572acc --- /dev/null +++ b/roles/docker_restore/meta/main.yml @@ -0,0 +1,52 @@ +galaxy_info: + author: your name + description: your role description + company: your company (optional) + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Choose a valid license ID from https://spdx.org - some suggested licenses: + # - BSD-3-Clause (default) + # - MIT + # - GPL-2.0-or-later + # - GPL-3.0-only + # - Apache-2.0 + # - CC-BY-4.0 + license: license (GPL-2.0-or-later, MIT, etc) + + min_ansible_version: 2.1 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + # + # Provide a list of supported platforms, and for each platform a list of versions. + # If you don't wish to enumerate all versions for a particular platform, use 'all'. + # To view available platforms and versions (or releases), visit: + # https://galaxy.ansible.com/api/v1/platforms/ + # + # platforms: + # - name: Fedora + # versions: + # - all + # - 25 + # - name: SomePlatform + # versions: + # - all + # - 1.0 + # - 7 + # - 99.99 + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. diff --git a/roles/docker_restore/tasks/main.yml b/roles/docker_restore/tasks/main.yml new file mode 100644 index 0000000..8c20bf7 --- /dev/null +++ b/roles/docker_restore/tasks/main.yml @@ -0,0 +1,121 @@ +--- +# tasks file for chatton.docker_backup +# https://docs.ansible.com/ansible/latest/collections/community/docker/docker_container_module.html#ansible-collections-community-docker-docker-container-module +# https://docs.docker.com/storage/volumes/#backup-restore-or-migrate-data-volumes + +- name: Get container details + docker_container_info: + name: "{{ container_restore }}" + register: result + +- name: Fail if container is not present + fail: + msg: Cannot restore volumes for a container when it does not exist. Ensure the container exists and try again. + when: result.exists == false + +- debug: msg="{{ result }}" + +- name: Extract only the volume mounts (not bind mounts) + set_fact: volume_mounts="{{ result.container.Mounts | selectattr("Type", "equalto", "volume")}}" + +- debug: msg="{{ volume_mounts }}" + +- name: Find relevant volume(s) in S3 + amazon.aws.aws_s3: + bucket: "{{ aws_s3.bucket }}" + mode: list + region: "{{ aws_s3.region }}" + s3_url: "https://{{ aws_s3.s3_url }}" + prefix: "{{ item.Name }}/{{ item.Name }}" + aws_access_key: "{{ aws_s3.aws_access_key }}" + aws_secret_key: "{{ aws_s3.aws_secret_key }}" + register: s3_list_output + with_items: "{{ volume_mounts }}" + +- debug: msg="{ {s3_list_output }}" + +- name: Extract s3 keys for container + set_fact: container_s3_keys="{{ container_s3_keys | default([]) + [item.s3_keys | last] }}" + with_items: "{{ s3_list_output.results }}" + +- debug: msg="{{ container_s3_keys }}" + +- name: Create a directory for temporary backups if they do not exist + ansible.builtin.file: + path: "/tmp/{{ item.Name }}" + state: directory + mode: '0755' + with_items: "{{ volume_mounts }}" + +- name: Download archives from S3 + amazon.aws.aws_s3: + bucket: "{{ aws_s3.bucket }}" + object: "{{ item }}" + aws_access_key: "{{ aws_s3.aws_access_key }}" + aws_secret_key: "{{ aws_s3.aws_secret_key }}" + region: "{{ aws_s3.region }}" + s3_url: "https://{{ aws_s3.s3_url }}" + mode: get + dest: "/tmp/{{ item }}" + with_items: "{{ container_s3_keys }}" + register: get_out + +- debug: msg="{{ get_out }}" + +- set_fact: + volume_details: "{{ volume_details |default([]) + [ {'mount': item.0, 's3_key': item.1} ] }}" + with_together: + - "{{ volume_mounts }}" + - "{{ container_s3_keys }}" + +- debug: msg="{{ volume_details }}" + +- name: Stop a container + community.docker.docker_container: + name: "{{ container_restore }}" + state: stopped + +- name: Ensure Volume + docker_volume: + name: "{{ item.mount.Name }}" + state: present + with_items: "{{ volume_details }}" + +- name: Remove contents of volumes + community.docker.docker_container: + name: "restore-container-{{ item.mount.Name }}-{{ 10 | random }}" + image: ubuntu + command: "rm -rf ./* " + auto_remove: true + detach: false # block until this container exists. + state: started + # start inside the directory we want to wipe + working_dir: "{{ item.mount.Destination }}" + volumes: + - /tmp:/tmp + volumes_from: + - "{{ container_restore }}" + with_items: "{{ volume_details }}" + + +- name: Restore contents of volumes + community.docker.docker_container: + name: "restore-container-{{ item.mount.Name }}-{{ 10 | random }}" + image: ubuntu + # extract the tar into the volume. + command: "tar xvf /tmp/{{ item.s3_key }}" + auto_remove: true + detach: false # block until this container exists. + state: started + # the compressed volume contains the directories, so we start from the root + working_dir: "/" + volumes: + - /tmp:/tmp + volumes_from: + - "{{ container_restore }}" + with_items: "{{ volume_details }}" + +- name: Start a container + community.docker.docker_container: + name: "{{ container_restore }}" + state: started diff --git a/roles/docker_restore/vars/main.yml b/roles/docker_restore/vars/main.yml new file mode 100644 index 0000000..2be0913 --- /dev/null +++ b/roles/docker_restore/vars/main.yml @@ -0,0 +1,2 @@ +--- +# vars file for chatton.docker_restore diff --git a/roles/sprat.mergerfs/.flake8 b/roles/sprat.mergerfs/.flake8 new file mode 100644 index 0000000..da5c197 --- /dev/null +++ b/roles/sprat.mergerfs/.flake8 @@ -0,0 +1,2 @@ +[flake8] +exclude = .svn,CVS,.bzr,.hg,.git,__pycache__,.tox,.eggs,*.egg,.*env diff --git a/roles/sprat.mergerfs/.github/workflows/ci.yml b/roles/sprat.mergerfs/.github/workflows/ci.yml new file mode 100644 index 0000000..be487cf --- /dev/null +++ b/roles/sprat.mergerfs/.github/workflows/ci.yml @@ -0,0 +1,68 @@ +--- +name: CI +on: # yamllint disable-line rule:truthy + push: + schedule: + - cron: "0 5 * * 1" + +jobs: + # test the role + test: + runs-on: ubuntu-latest + strategy: + matrix: + config: + - image: geerlingguy/docker-centos8-ansible + mode: github_releases + - image: geerlingguy/docker-centos7-ansible + mode: github_releases + - image: geerlingguy/docker-fedora32-ansible + mode: github_releases + - image: geerlingguy/docker-fedora31-ansible + mode: github_releases + - image: geerlingguy/docker-fedora30-ansible + mode: github_releases + - image: geerlingguy/docker-ubuntu2004-ansible + mode: github_releases + - image: geerlingguy/docker-ubuntu2004-ansible + mode: package_manager + - image: geerlingguy/docker-ubuntu1804-ansible + mode: github_releases + - image: geerlingguy/docker-ubuntu1604-ansible + mode: github_releases + - image: geerlingguy/docker-debian10-ansible + mode: package_manager + - image: geerlingguy/docker-debian10-ansible + mode: github_releases + - image: geerlingguy/docker-debian9-ansible + mode: github_releases + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Python 3 + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.in + + - name: Run molecule tests + env: + IMAGE: ${{ matrix.config.image }} + INSTALL_MODE: ${{ matrix.config.mode }} + run: molecule -v test + + # publish the role on ansible galaxy + publish: + needs: test + runs-on: ubuntu-latest + steps: + - name: Publish + uses: robertdebock/galaxy-action@1.1.0 + with: + galaxy_api_key: ${{ secrets.GALAXY_API_KEY }} diff --git a/roles/sprat.mergerfs/.gitignore b/roles/sprat.mergerfs/.gitignore new file mode 100644 index 0000000..6221012 --- /dev/null +++ b/roles/sprat.mergerfs/.gitignore @@ -0,0 +1,5 @@ +*.retry +*.pyc +__pycache__/ +*env/ +.cache/ diff --git a/roles/sprat.mergerfs/.yamllint.yml b/roles/sprat.mergerfs/.yamllint.yml new file mode 100644 index 0000000..e896dbe --- /dev/null +++ b/roles/sprat.mergerfs/.yamllint.yml @@ -0,0 +1,9 @@ +--- +extends: default + +ignore: | + .*env/ + +rules: + line-length: + max: 120 diff --git a/roles/sprat.mergerfs/LICENSE b/roles/sprat.mergerfs/LICENSE new file mode 100644 index 0000000..6ea29aa --- /dev/null +++ b/roles/sprat.mergerfs/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2020 Sylvain Prat + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/roles/sprat.mergerfs/README.md b/roles/sprat.mergerfs/README.md new file mode 100644 index 0000000..7508ae7 --- /dev/null +++ b/roles/sprat.mergerfs/README.md @@ -0,0 +1,54 @@ +Ansible Role: mergerfs +====================== + +[![Build Status][build_badge]][build_link] +[![Ansible Galaxy][galaxy_badge]][galaxy_link] + +Install and configure Mergerfs — A featureful union filesystem. + +Requirements +------------ + +None. + +Role Variables +-------------- + +See [defaults/main.yml](defaults/main.yml). + +Dependencies +------------ + +None. + +Example Playbook +---------------- + +```yaml +- hosts: server + roles: + - role: sprat.mergerfs + vars: + mergerfs_mounts: + - path: /mnt/data + branches: + - /mnt/data1 + - /mnt/data2 + options: allow_other,use_ino +``` + +License +------- + +MIT + +Author Information +------------------ + +This role was created in 2020 by [Sylvain Prat](https://github.com/sprat). + + +[build_badge]: https://img.shields.io/github/workflow/status/sprat/ansible-role-mergerfs/CI +[build_link]: https://github.com/sprat/ansible-role-mergerfs/actions?query=workflow:CI +[galaxy_badge]: https://img.shields.io/ansible/role/47517 +[galaxy_link]: https://galaxy.ansible.com/sprat/mergerfs diff --git a/roles/sprat.mergerfs/defaults/main.yml b/roles/sprat.mergerfs/defaults/main.yml new file mode 100644 index 0000000..36de9e4 --- /dev/null +++ b/roles/sprat.mergerfs/defaults/main.yml @@ -0,0 +1,23 @@ +--- +# Install mode: defines where to download and install the package from: +# - "github_releases": install from Mergerfs' GitHub releases +# - "package_manager": install from the Linux distribution package manager. +# Note that the mergerfs package does not exists in all distributions, so it +# may not work for you. +mergerfs_install_mode: github_releases + +# Version to install: "latest" version or a specific version number, e.g. "2.28.2" +# This setting only applies in "github_releases" mode +mergerfs_version: latest + +# Mergerfs mountpoints to create. For example: +# mergerfs_mounts: +# - path: /mnt/storage +# branches: +# - /mnt/data* +# - /mnt/other +# options: allow_other,use_ino +mergerfs_mounts: [] + +# Url of the mergerfs GitHub releases page +mergerfs_github_releases_url: https://github.com/trapexit/mergerfs/releases diff --git a/roles/sprat.mergerfs/meta/.galaxy_install_info b/roles/sprat.mergerfs/meta/.galaxy_install_info new file mode 100644 index 0000000..e0802e4 --- /dev/null +++ b/roles/sprat.mergerfs/meta/.galaxy_install_info @@ -0,0 +1,2 @@ +install_date: Thu 1 Sep 15:42:59 2022 +version: master diff --git a/roles/sprat.mergerfs/meta/main.yml b/roles/sprat.mergerfs/meta/main.yml new file mode 100644 index 0000000..498764f --- /dev/null +++ b/roles/sprat.mergerfs/meta/main.yml @@ -0,0 +1,30 @@ +--- +galaxy_info: + author: Sylvain Prat + role_name: mergerfs + namespace: sprat + description: Install and configure Mergerfs — A featureful union filesystem + license: MIT + company: none + min_ansible_version: 2.3 + platforms: + - name: Ubuntu + versions: + - all + - name: Debian + versions: + - all + - name: Fedora + versions: + - all + - name: EL + versions: + - all + galaxy_tags: + - mergerfs + - union + - filesystem + - disk + - mount + +dependencies: [] diff --git a/roles/sprat.mergerfs/molecule/default/converge.yml b/roles/sprat.mergerfs/molecule/default/converge.yml new file mode 100644 index 0000000..e972edd --- /dev/null +++ b/roles/sprat.mergerfs/molecule/default/converge.yml @@ -0,0 +1,11 @@ +--- +- name: Converge + hosts: all + vars: + mergerfs_mounts: + - path: /mnt/storage + branches: + - /mnt/data* + options: allow_other,use_ino + roles: + - role: ansible-role-mergerfs diff --git a/roles/sprat.mergerfs/molecule/default/molecule.yml b/roles/sprat.mergerfs/molecule/default/molecule.yml new file mode 100644 index 0000000..4309a0f --- /dev/null +++ b/roles/sprat.mergerfs/molecule/default/molecule.yml @@ -0,0 +1,21 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: yamllint -s . && ansible-lint . && flake8 +platforms: + - name: instance + image: ${IMAGE:-geerlingguy/docker-ubuntu2004-ansible} + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + pre_build_image: true +provisioner: + name: ansible + inventory: + group_vars: + all: + mergerfs_install_mode: ${INSTALL_MODE:-github_releases} +verifier: + name: testinfra diff --git a/roles/sprat.mergerfs/molecule/default/prepare.yml b/roles/sprat.mergerfs/molecule/default/prepare.yml new file mode 100644 index 0000000..358fc3f --- /dev/null +++ b/roles/sprat.mergerfs/molecule/default/prepare.yml @@ -0,0 +1,25 @@ +--- +- name: Prepare + hosts: all + tasks: + - name: Create directories + become: true + file: + path: "{{ item }}" + state: directory + loop: + - /mnt/data1 + - /mnt/data2 + + - name: Create data files + become: true + copy: + content: "{{ item.content }}\n" + dest: "{{ item.path }}" + loop: + - path: /mnt/data1/file1.txt + content: file1 + - path: /mnt/data2/file2.txt + content: file2 + - path: /mnt/data2/file3.txt + content: file3 diff --git a/roles/sprat.mergerfs/molecule/default/tests/test_default.py b/roles/sprat.mergerfs/molecule/default/tests/test_default.py new file mode 100644 index 0000000..0a89a30 --- /dev/null +++ b/roles/sprat.mergerfs/molecule/default/tests/test_default.py @@ -0,0 +1,21 @@ +import os + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE'] +).get_hosts('all') + + +def test_mount_point(host): + mount_point = host.mount_point('/mnt/storage') + assert mount_point.exists + assert mount_point.filesystem == 'fuse.mergerfs' + assert 'allow_other' in mount_point.options + # assert 'use_ino' in mount_point.options + + +def test_data_files(host): + assert host.file('/mnt/storage/file1.txt').exists + assert host.file('/mnt/storage/file2.txt').exists + assert host.file('/mnt/storage/file3.txt').exists diff --git a/roles/sprat.mergerfs/requirements.in b/roles/sprat.mergerfs/requirements.in new file mode 100644 index 0000000..fb4271c --- /dev/null +++ b/roles/sprat.mergerfs/requirements.in @@ -0,0 +1 @@ +molecule[ansible,docker,test,lint] diff --git a/roles/sprat.mergerfs/requirements.txt b/roles/sprat.mergerfs/requirements.txt new file mode 100644 index 0000000..7740177 --- /dev/null +++ b/roles/sprat.mergerfs/requirements.txt @@ -0,0 +1,249 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile +# +ansi2html==1.6.0 + # via molecule +ansible-base==2.10.7 + # via ansible +ansible-lint==5.0.7 + # via molecule +ansible==3.2.0 + # via molecule +apipkg==1.5 + # via execnet +appdirs==1.4.4 + # via virtualenv +arrow==1.0.3 + # via jinja2-time +attrs==20.3.0 + # via pytest +bcrypt==3.2.0 + # via paramiko +binaryornot==0.4.4 + # via cookiecutter +bracex==2.1.1 + # via wcmatch +cerberus==1.3.2 + # via molecule +certifi==2020.12.5 + # via requests +cffi==1.14.5 + # via + # bcrypt + # cryptography + # pynacl +cfgv==3.2.0 + # via pre-commit +chardet==4.0.0 + # via + # binaryornot + # requests +click-completion==0.5.2 + # via molecule +click-help-colors==0.9 + # via molecule +click==7.1.2 + # via + # click-completion + # click-help-colors + # cookiecutter + # molecule +colorama==0.4.4 + # via rich +commonmark==0.9.1 + # via rich +cookiecutter==1.7.2 + # via molecule +coverage==5.5 + # via pytest-cov +cryptography==3.4.7 + # via + # ansible-base + # paramiko +distlib==0.3.1 + # via virtualenv +distro==1.5.0 + # via selinux +docker==5.0.0 + # via molecule-docker +enrich==1.2.6 + # via + # ansible-lint + # molecule +execnet==1.8.0 + # via pytest-xdist +filelock==3.0.12 + # via virtualenv +flake8==3.9.0 + # via molecule +identify==2.2.3 + # via pre-commit +idna==2.10 + # via requests +iniconfig==1.1.1 + # via pytest +jinja2-time==0.2.0 + # via cookiecutter +jinja2==2.11.3 + # via + # ansible-base + # click-completion + # cookiecutter + # jinja2-time + # molecule +markupsafe==1.1.1 + # via + # cookiecutter + # jinja2 +mccabe==0.6.1 + # via flake8 +molecule-docker==0.2.4 + # via molecule +molecule[ansible,docker,lint,test]==3.3.0 + # via + # -r requirements.in + # molecule-docker +more-itertools==8.7.0 + # via pytest-plus +nodeenv==1.6.0 + # via pre-commit +packaging==20.9 + # via + # ansible-base + # ansible-lint + # molecule + # pytest +paramiko==2.7.2 + # via molecule +pathspec==0.8.1 + # via yamllint +pexpect==4.8.0 + # via molecule +pluggy==0.13.1 + # via + # molecule + # pytest +poyo==0.5.0 + # via cookiecutter +pre-commit==2.12.0 + # via molecule +ptyprocess==0.7.0 + # via pexpect +py==1.10.0 + # via + # pytest + # pytest-forked +pycodestyle==2.7.0 + # via flake8 +pycparser==2.20 + # via cffi +pyflakes==2.3.1 + # via flake8 +pygments==2.8.1 + # via rich +pynacl==1.4.0 + # via paramiko +pyparsing==2.4.7 + # via packaging +pytest-cov==2.11.1 + # via molecule +pytest-forked==1.3.0 + # via pytest-xdist +pytest-helpers-namespace==2021.3.24 + # via molecule +pytest-html==3.1.1 + # via molecule +pytest-metadata==1.11.0 + # via pytest-html +pytest-mock==3.5.1 + # via molecule +pytest-plus==0.2 + # via molecule +pytest-testinfra==6.2.0 + # via molecule +pytest-verbose-parametrize==1.7.0 + # via molecule +pytest-xdist==2.2.1 + # via molecule +pytest==6.2.3 + # via + # molecule + # pytest-cov + # pytest-forked + # pytest-helpers-namespace + # pytest-html + # pytest-metadata + # pytest-mock + # pytest-plus + # pytest-testinfra + # pytest-verbose-parametrize + # pytest-xdist +python-dateutil==2.8.1 + # via arrow +python-slugify==4.0.1 + # via cookiecutter +pyyaml==5.4.1 + # via + # ansible-base + # ansible-lint + # molecule + # pre-commit + # yamllint +requests==2.25.1 + # via + # cookiecutter + # docker +rich==10.1.0 + # via + # ansible-lint + # enrich + # molecule +ruamel.yaml.clib==0.2.2 + # via ruamel.yaml +ruamel.yaml==0.17.4 + # via ansible-lint +selinux==0.2.1 + # via + # molecule + # molecule-docker +shellingham==1.4.0 + # via click-completion +six==1.15.0 + # via + # bcrypt + # click-completion + # cookiecutter + # pynacl + # pytest-verbose-parametrize + # python-dateutil + # tenacity + # virtualenv + # websocket-client +subprocess-tee==0.2.0 + # via molecule +tenacity==7.0.0 + # via ansible-lint +text-unidecode==1.3 + # via python-slugify +toml==0.10.2 + # via + # pre-commit + # pytest +typing-extensions==3.7.4.3 + # via rich +urllib3==1.26.4 + # via requests +virtualenv==20.4.3 + # via pre-commit +wcmatch==8.1.2 + # via ansible-lint +websocket-client==0.58.0 + # via docker +yamllint==1.26.1 + # via molecule + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/roles/sprat.mergerfs/tasks/install_from_github_releases.yml b/roles/sprat.mergerfs/tasks/install_from_github_releases.yml new file mode 100644 index 0000000..de8f333 --- /dev/null +++ b/roles/sprat.mergerfs/tasks/install_from_github_releases.yml @@ -0,0 +1,54 @@ +--- +# Note: we don't use the GitHub API to retrieve the latest version because +# it has rate limits which are hard to avoid in CI (we need a token, authenticate +# with the API, etc.). Instead, we browse the latest release url which redirects +# to the release page, where we can find the version number in the URL. +- become: false + delegate_to: localhost + run_once: true + block: + - name: Get latest release information from GitHub + uri: + url: "{{ mergerfs_github_releases_url }}/latest" + register: mergerfs_github_release_page + - name: Set latest mergerfs version fact + set_fact: + mergerfs_version: "{{ mergerfs_github_release_page['url'].split('/')[-1] }}" + when: mergerfs_version == "latest" + +- name: Determine package download url + set_fact: + mergerfs_package_url: "{{ mergerfs_github_releases_url }}/download/{{ mergerfs_version }}/\ + {{ mergerfs_pkg_prefix }}{{ mergerfs_version }}{{ mergerfs_pkg_suffix }}" + +- name: Install xz-utils package for .deb package installation + become: true + apt: + name: xz-utils + state: present + update_cache: true + when: ansible_pkg_mgr == 'apt' + +- name: Install mergerfs package with apt + become: true + apt: + deb: "{{ mergerfs_package_url }}" + state: present + update_cache: true + when: ansible_pkg_mgr == 'apt' + +- name: Install mergerfs package with yum + become: true + yum: + name: "{{ mergerfs_package_url }}" + state: present + disable_gpg_check: true # the package is not signed + when: ansible_pkg_mgr == 'yum' + +- name: Install mergerfs package with dnf + become: true + dnf: + name: "{{ mergerfs_package_url }}" + state: present + disable_gpg_check: true # the package is not signed + when: ansible_pkg_mgr == 'dnf' diff --git a/roles/sprat.mergerfs/tasks/install_from_package_manager.yml b/roles/sprat.mergerfs/tasks/install_from_package_manager.yml new file mode 100644 index 0000000..5ce2631 --- /dev/null +++ b/roles/sprat.mergerfs/tasks/install_from_package_manager.yml @@ -0,0 +1,7 @@ +--- +- name: Install mergerfs package with package manager + become: true + package: + name: mergerfs + state: present + update_cache: true diff --git a/roles/sprat.mergerfs/tasks/main.yml b/roles/sprat.mergerfs/tasks/main.yml new file mode 100644 index 0000000..1229f1c --- /dev/null +++ b/roles/sprat.mergerfs/tasks/main.yml @@ -0,0 +1,34 @@ +--- +- name: Include OS-specific variables + include_vars: "{{ ansible_os_family }}.yml" + tags: + - mergerfs + +- name: Install mergerfs prerequisites + become: true + package: + name: "{{ mergerfs_prerequisites }}" + state: present + update_cache: true + tags: + - mergerfs + - mergerfs_install + +- name: Include install tasks + import_tasks: install_from_{{ mergerfs_install_mode }}.yml + tags: + - mergerfs + - mergerfs_install + +- name: Mount mergerfs filesystems + become: true + mount: + fstype: fuse.mergerfs + src: "{{ ':'.join(item.branches | mandatory) }}" + path: "{{ item.path | mandatory }}" + opts: "{{ item.options | default('defaults') }}" + state: "{{ item.state | default('mounted') }}" + loop: "{{ mergerfs_mounts }}" + tags: + - mergerfs + - mergerfs_mount diff --git a/roles/sprat.mergerfs/vars/Debian.yml b/roles/sprat.mergerfs/vars/Debian.yml new file mode 100644 index 0000000..ff3d38c --- /dev/null +++ b/roles/sprat.mergerfs/vars/Debian.yml @@ -0,0 +1,12 @@ +--- +mergerfs_prerequisites: + - fuse +mergerfs_dist: "{{ ansible_distribution|lower }}-{{ ansible_distribution_release }}" +mergerfs_arch_map: + x86_64: amd64 + i386: i386 + aarch64: arm64 + armv7l: armhf +mergerfs_arch: "{{ mergerfs_arch_map[ansible_userspace_architecture | default(ansible_architecture) ] }}" +mergerfs_pkg_prefix: "mergerfs_" +mergerfs_pkg_suffix: ".{{ mergerfs_dist }}_{{ mergerfs_arch }}.deb" diff --git a/roles/sprat.mergerfs/vars/RedHat.yml b/roles/sprat.mergerfs/vars/RedHat.yml new file mode 100644 index 0000000..2ab4da1 --- /dev/null +++ b/roles/sprat.mergerfs/vars/RedHat.yml @@ -0,0 +1,7 @@ +--- +mergerfs_prerequisites: + - fuse +mergerfs_dist: "{{ 'fc' if ansible_distribution == 'Fedora' else 'el' }}{{ ansible_distribution_major_version }}" +mergerfs_arch: "{{ ansible_userspace_architecture }}" +mergerfs_pkg_prefix: "mergerfs-" +mergerfs_pkg_suffix: "-1.{{ mergerfs_dist }}.{{ mergerfs_arch }}.rpm" diff --git a/vault_vars/qnap-vault.yml b/vault_vars/qnap-vault.yml index e182d2a..68f83d8 100644 --- a/vault_vars/qnap-vault.yml +++ b/vault_vars/qnap-vault.yml @@ -1,43 +1,64 @@ $ANSIBLE_VAULT;1.1;AES256 -37666165636561303539306466393465653238336365663731616363323164313361633830353730 -3531623965653935303664383061386164383038656439330a323265306137613231313837383335 -31373763633930333536313533356333336235633265326265366337303035333464646462326163 -6632656239626631380a333365653563313139343631306330643638396661393736376239613061 -36616234346663373236666633623231333137316561336362323830643531323934363735323837 -39353363616462343065303538333637623837653633366437646436313963616333653834306334 -30663839616335363162346135393037646330616331323464623631663931623935323563313437 -62323462306463376433663636663033623633343562653834663066323932656230623232666136 -66636264633631393937613535363366626135303939656364623937653763383865303461646536 -65656665643866623363623464656632323261656433663030316333613935336662303763303062 -30623937303738616138633933343438363935616334353837616465643930306435326462313961 -65633131303732343135663635303933313935636533333935633830336439636236363330306337 -36323932613339666631333038323538656334393565343666346133393864643664383662316166 -32636462636165383230353231653261333564646562663035316266653466623565623661376637 -36653935383539313864656165636234636562323664653435633732383737353039323565643537 -34383739616639343831306565323930373961656235383666366139346662626166646462356139 -31336334653864373037353135646538633039623332336635386132396664623737626436303530 -63316532366639643861663433383335626530613437363831623232656339613235383734646466 -39653838326566656439626663363731356134383362333535643736663231323030366565653461 -62343166343762646533653065303531656564363866356366663364363438373731333861316163 -35323430616164323962356635323436306265623335343736623132363138343663613163313862 -37383066393335616666343336643131656431386264386230366434623362663733383334343037 -36616237613233333630323233643630353330343730386464316634633938383030353765366436 -62303962353838643461326565313236336265346234316637626338623031303430623039366562 -64653062333963383865343263343232356366643238393636383139656536613639376135353163 -32306233373533356365393233393165666132336637613862653038373839613036353463306233 -63643564656364353836303665613862316165393263633536623731343137366162663335623066 -66613835306135303563323036313336313632306131643730643931363438666364313864353766 -36643431343265343036313362653262393636653134343339666361383263623936353564393266 -36303836313437656436656634363462386362613361653536373038313263623562653833333735 -62333339626437336464326263393838336135326566346436336637313035333062643662393463 -38633964656361346530326437663233356139373537643130656266653236666437663730356561 -66356662373961623537343063343162303833363130653439613965393363386532633966623537 -66653730613866323933363733633734646437376530396234303161373365376235653132363262 -36643965393164393165363231303336656238326530373531356631313532663864656261653936 -37333365313036363330336133376431313839626633623732316163363632363033306338633030 -34396530313764613465633435323435653161646634336562303064373563373938356132653638 -63643361363763326532363836643433613166636266623933653065633631366234643366383464 -39313465653432663833623030333862363430363036646334366261316161656633646339366231 -37323533633139636363666664353965333637353735633039616337636439636266613962323138 -39333962646233343361363461643936616661323237303030663534336634636239336164653939 -6263396338326466363861343332376264376635653362643764 +66353636383061643861393632363630643262383863653036313663623862616663386534306262 +3466306661356164653730653062393863363163613064660a663732333931323037373834386336 +31383464333930363163646637663631393736386139333430313463356661303033353462313339 +6232336539366234360a663662636662396138636334383737376439346239386466666239313563 +62663864316661306466316561343336613938386239383663656564336166323839383765363761 +38383733323037353961303937623365623064383163373161633733616539373662353439383531 +64393637343035613630356165626666306531353634363966333832353335646262663365313039 +63656331626330366263386466646562393033646165386432386233316235663636643736363237 +66343362663736343136336563663664323030313033326236663132316539346661323532653162 +63336135373835613232656639656161313464663461303034313632303033393233396432613537 +35633239326662333463383735633835616462396239373334373632663361393230373635343366 +33346466613237303137646263626437323964653630323236303964663364626365323162656438 +65323330626536323365613035653332376336343139323235323663623430393534316230346135 +38616333656162393837643564366463626538386633323730313564373561386334643831303063 +31306237333663346130623535636532336234613766316234383135303863653565633939313934 +34656639343763313363653739336163376634376264393133376634373062666137343338663732 +37623463313839393432366334613839663366656633663464656631656334616663636562323536 +65653865646333326631386166333263343031356461666365623532333439326430623632646433 +62373733616162386662393566633333353635373337666234623062626639656536623161343163 +32626164376539626634636561383764306539353666643132363536303361663664346163336266 +31616263663865623036656436326130636337323136633836313037356236633764383934626466 +62613265383261663962356439646135333566356335303139346430353434646661393335353138 +35373037336563326363643339356262366462343463366661386638623530613364663936326239 +64313662653139376164356333303365376366303236643862336465643465383266373335653337 +61623638626633626434343730373930343236653737396161366136373232333034366637336463 +30396436373063666233356636626537626336303163306333343837653163373932663061376566 +61336239306666333764613462663461336138363736383030663534376566303237306639353234 +35353439633830333538363136396336613536396661333337626630326132653736303836613864 +64613531623833343166376238303238633732316438633161383761653664666236633162383531 +37323761376334633535313234376237633938353332353239303863383265393430333264663662 +31613030616139613062326431366666346233383736376161643033356531316130666366333638 +35616336643964613163386264356237383034333634343266303238333864383439383236353766 +63346436393330616639623839363232376236623565666433613137336331353436376165613964 +61306130306130313336303761633037633765386562633464336636613961356430343830383237 +38626164383137613733326162633231346138306338663466396237386631313662353639396461 +62323030386336326662643766663135633231646364343561373134396230373332313561373936 +61303134636565643237633461663261613630363264363439343065653537633961653634346137 +36613766346539653332613537626138636430666432313734363831656666646364656634353163 +66616234366664636466333433383231616638353466333861333430303134646538626466353664 +66613566306135336163376137663462383432656165626663383963306336613034633131393866 +34613132306135613137303033623936393532653231373435363239373830346133643230366264 +66663866613534313831636165376434643031373535393430393863626439633732626338343931 +34376638393962353638663331633434313661643231303866333937633365613137653838646431 +37363865633934356366663431376466656337623466646338386364323761373464616238646337 +36643263383933303361313839643532613135323334623662653432353439316639323238353632 +36353465626462326332386466373062656137383763633430366635653630323031303563333434 +62663533383464363237336661356639393634613531613238343463643934316633656662353137 +31383334373032663936616664646233646164313366396339613937366434613265393162363238 +63646162326465376564623634323062323762343235336138666530333133393462616466636431 +37353736616635323663343832636438306339613563396237326366346563383666303131643264 +34383766623132316139323639626530653033666463323166663734353638393433303739313434 +64643539653631323933373232653737653466346566313930356138363139383465663861636535 +33333063636431623131363237633339303763643931663334656366316137396234643030373735 +64663338316434313731626435353033313333323133386664636531646165383065373937333335 +32336262323764653835663038363934633835313639613130363661333234666535363630326232 +35626366636631386264656162326233376332633333643832333261636634633138353266636264 +37653737303439383564363437393839323364623563376462353531653533396264653730323533 +62656562626263336439626133626663353564646236396461336662623736653037346564653262 +61356233646431646565613532376431363034343830353534386434636638646337303165313739 +62343734326361646637656534636132643438303233313562656366316432346664303364626133 +39663664613236666435646331313365363264316232373831623836323439383632643731303865 +39623766396463643939633666643730386634623230613665653432623662363137663931323363 +656162386137646562313733343631643036