adding mergerfs drive

.old/homelab/hosts

@@ -1,2 +0,0 @@
-[servers]
-localhost

.old/homelab/playbooks/mount-hdds.yml

@@ -1,53 +0,0 @@
----
-- hosts: servers
-  vars_files:
-    - ../variables.yml
-  become: true
-  tasks:
-  - name: Ensure /mnt/hdds directory
-    file:
-      path: /mnt/hdds
-      state: directory
-
-  - name: Unmount HDDS
-    ansible.posix.mount:
-      path: /mnt/hdds/
-      src: "UUID={{hdds_uuid}}"
-      fstype: ext4
-      state: unmounted
-      opts: defaults,nofail
-
-  - name: Make /mnt/hdds immutable
-    ansible.builtin.shell:  chattr +i /mnt/hdds
-
-  - name: Mount HDDS
-    ansible.posix.mount:
-      path: /mnt/hdds
-      src: "UUID={{hdds_uuid}}"
-      fstype: ext4
-      state: mounted
-      opts: defaults,nofail
-
-  - name: Ensure /mnt/ssd0 directory
-    file:
-      path: /mnt/ssd0
-      state: directory
-
-  - name: Unmount SSD0
-    ansible.posix.mount:
-      path: /mnt/ssd0/
-      src: "UUID={{ssd0_uuid}}"
-      fstype: ext4
-      state: unmounted
-      opts: defaults,nofail
-
-  - name: Make /mnt/ssd0 immutable
-    ansible.builtin.shell:  chattr +i /mnt/ssd0
-
-  - name: Mount SSD0
-    ansible.posix.mount:
-      path: /mnt/ssd0/
-      src: "UUID={{ssd0_uuid}}"
-      fstype: ext4
-      state: mounted
-      opts: defaults,nofail

.old/homelab/playbooks/redeploy-dashy.yml

@@ -1,11 +0,0 @@
----
-- hosts: servers
-  vars_files:
-    - ../variables.yml
-  become: true
-  tasks:
-  - name: Checkout Git Repos
-    import_tasks: ../tasks/checkout-git-repos-tasks.yml
-
-  - name: Restart Dashy
-    ansible.builtin.command: docker restart dashy

.old/homelab/playbooks/restore-all.yml

@@ -1,163 +0,0 @@
-###
-# restore-all restores a manually specified list of volumes.
-# This is intended for user when initially setting up a home lab environment
-# and want to restore all applications from existing backups
-#
-# Restores will be skipped if there is no archive in the backup directory.
-#
-# If a container exists, it will be stopped, restored and started
-# If a container does not exist, the volume will be still be created.
-###
-
-############################################
-# TODO: add a command which lists all the stack, and each stack can list all the containers. From there we can iterate
-# through each container in the stack calling restore-volume-tasks.yml and then call create-stack-tasks.yml
-############################################
-
----
-- name: Redeploy Portainer and All Volumes
-  hosts: servers
-  vars_files:
-    - ../variables.yml
-    - ~/.homelab/variables.yml
-  tasks:
-    - name: Create next cloud a network
-      docker_network:
-        name: nextcloud_net
-
-    # The docker volume backup image utilizes these images
-    - name: Pull ubuntu
-      docker_image:
-        name: ubuntu
-        source: pull
-    - name: Pull busybox
-      docker_image:
-        name: busybox
-        source: pull
-    - name: Pull docker-volume-backup
-      docker_image:
-        name: "{{docker_volume_backup_image}}"
-        source: pull
-    - name: Pull start-portainer-stack
-      docker_image:
-        name: ghcr.io/chatton/start-portainer-stack:master
-        source: pull
-
-    - name: Restore Portainer
-      import_tasks: ../tasks/restore-volume-tasks.yml
-      vars:
-        container_name: "portainer"
-
-    - name: "Restore {{item}}"
-      import_tasks: ../tasks/restore-volume-tasks.yml
-      vars:
-        container_name: "{{item}}"
-      with_items: {{stacks}}
-
-    - name: "Start {{item}} Stack"
-      import_tasks: ../tasks/create-stack-tasks.yml
-      vars:
-        stack_name: "{{item}}"
-      with_items: {{stacks}}
-
-
-#    - name: Restore Linkding
-#      import_tasks: ../tasks/restore-volume-tasks.yml
-#      vars:
-#        container_name: "linkding"
-#
-#    - name: Start Linkding Stack
-#      import_tasks: ../tasks/create-stack-tasks.yml
-#      vars:
-#        stack_name: "linkding"
-#
-#    - name: Restore Nginx Proxy Manager
-#      import_tasks: ../tasks/restore-volume-tasks.yml
-#      vars:
-#        container_name: "nginx-proxy-manager"
-#
-#    - name: Start Nginx Proxy Manager
-#      import_tasks: ../tasks/create-stack-tasks.yml
-#      vars:
-#        stack_name: "nginx-proxy-manager"
-#
-#    - name: Restore Vpn Stack
-#      import_tasks: ../tasks/restore-volume-tasks.yml
-#      vars:
-#        container_name: "vpn-stack"
-#
-#    - name: Start Vpn Stack
-#      import_tasks: ../tasks/create-stack-tasks.yml
-#      vars:
-#        stack_name: "vpn-stack"
-#
-#
-#    - name: Restore MariaDB
-#      import_tasks: ../tasks/restore-volume-tasks.yml
-#      vars:
-#        container_name: "mariadb"
-#
-#    - name: Start MariaDB
-#      import_tasks: ../tasks/create-stack-tasks.yml
-#      vars:
-#        stack_name: "mariadb"
-#
-#    - name: Restore Nextcloud
-#      import_tasks: ../tasks/restore-volume-tasks.yml
-#      vars:
-#        container_name: "nextcloud"
-#
-#    - name: Start Nextcloud
-#      import_tasks: ../tasks/create-stack-tasks.yml
-#      vars:
-#        stack_name: "nextcloud"
-#
-#    - name: Restore Photoprism
-#      import_tasks: ../tasks/restore-volume-tasks.yml
-#      vars:
-#        container_name: "photoprism"
-#
-#    - name: Start Photoprism
-#      import_tasks: ../tasks/create-stack-tasks.yml
-#      vars:
-#        stack_name: "photoprism"
-#
-#    - name: Restore Plex
-#      import_tasks: ../tasks/restore-volume-tasks.yml
-#      vars:
-#        container_name: "plex"
-#
-#    - name: Start Plex
-#      import_tasks: ../tasks/create-stack-tasks.yml
-#      vars:
-#        stack_name: "plex"
-#
-#    - name: Restore Mealie
-#      import_tasks: ../tasks/restore-volume-tasks.yml
-#      vars:
-#        container_name: "mealie"
-#
-#    - name: Start Mealie
-#      import_tasks: ../tasks/create-stack-tasks.yml
-#      vars:
-#        stack_name: "mealie"
-#
-#    - name: Restore Overseerr
-#      import_tasks: ../tasks/restore-volume-tasks.yml
-#      vars:
-#        container_name: "overseerr"
-#
-#    - name: Start Overseerr
-#      import_tasks: ../tasks/create-stack-tasks.yml
-#      vars:
-#        stack_name: "overseerr"
-#
-#    - name: Restore Uptime Kuma
-#      import_tasks: ../tasks/restore-volume-tasks.yml
-#      vars:
-#        container_name: "uptime-kuma"
-#
-#    - name: Start Uptime Kuma
-#      import_tasks: ../tasks/create-stack-tasks.yml
-#      vars:
-#        stack_name: "uptime-kuma"

.old/homelab/playbooks/setup-home-lab.yml

@@ -1,34 +0,0 @@
----
-- hosts: servers
-  become: true
-  vars_files:
-    - ../variables.yml
-  tasks:
-  - name: Update and upgrade apt packages
-    apt:
-      upgrade: yes
-      update_cache: yes
-      cache_valid_time: 86400
-
-  - name: Install aptitude
-    apt:
-      name: aptitude
-      state: latest
-      update_cache: true
-
-  - name: Install Dev Tooling
-    import_tasks: ../tasks/install-dev-tooling-tasks.yml
-
-  - name: Install & Configure Sudo
-    import_tasks: ../tasks/install-sudo-tasks.yml
-
-  - name: Checkout Git Repos
-    import_tasks: ../tasks/checkout-git-repos-tasks.yml
-
-  - name: Install Docker
-    import_tasks: ../tasks/install-docker-tasks.yml
-
-  - name: Create and Portainer
-    tags: "skip-e2e" # we can't mount volumes on test runners.
-    community.docker.docker_compose:
-      project_src: "{{home_lab_checkout_dir}}/docker-compose/portainer"

.old/homelab/playbooks/volume-restore.yml

@@ -1,11 +0,0 @@
-### Example:
-# ansible-playbook -v ansible/playbooks/volume-restore.yml -e "container_name=mariadb"
----
-- hosts: servers
-  vars_files:
-    - ../variables.yml
-  tasks:
-    - name: Restore Volume
-      import_tasks: ../tasks/restore-volume-tasks.yml
-      vars:
-        container_name: "{{ container_name }}"

.old/homelab/tasks/checkout-git-repos-tasks.yml

@@ -1,13 +0,0 @@
----
-- name: Git checkout HomeLab repository
-  ansible.builtin.git:
-    repo: 'https://github.com/chatton/HomeLab.git'
-    dest: "{{home_lab_checkout_dir}}"
-    force: true
-    version: master
-
-- name: Change Ownership
-  file:
-    path: "{{home_lab_checkout_dir}}"
-    owner: "{{homelab_user}}"
-    group: "{{homelab_group}}"

.old/homelab/tasks/create-stack-tasks.yml

@@ -1,7 +0,0 @@
----
-- name: Start stack
-  ansible.builtin.command: docker run --rm
-    -e PORTAINER_USER_NAME=admin 
-    -e PORTAINER_PASSWORD="{{portainer_password}}"
-    -e PORTAINER_BASE_URL="http://{{ansible_eth0.ipv4.address}}:9000"
-    ghcr.io/chatton/start-portainer-stack:master "{{stack_name}}"

.old/homelab/tasks/install-dev-tooling-tasks.yml

@@ -1,11 +0,0 @@
----
-- name: Install Dev Tooling
-  ansible.builtin.apt:
-    pkg:
-    - jq
-    - fzf
-    - git
-    - mdadm
-    - parted
-    - ncdu
-    - tree

.old/homelab/tasks/install-docker-tasks.yml

@@ -1,74 +0,0 @@
----
-- name: Install required system packages
-  apt:
-    pkg:
-      - apt-transport-https
-      - ca-certificates
-      - curl
-      - software-properties-common
-      - python3-pip
-      - virtualenv
-      - python3-setuptools
-    state: latest
-    update_cache: true
-
-- name: Add Docker GPG apt Key
-  apt_key:
-    url: https://download.docker.com/linux/ubuntu/gpg
-    state: present
-
-- name: Add Docker Repository
-  apt_repository:
-    repo: deb https://download.docker.com/linux/ubuntu focal stable
-    state: present
-
-- name: Update apt and install docker-ce
-  apt:
-    name: docker-ce
-    state: latest
-    update_cache: true
-
-- name: Install Docker Module for Python
-  pip:
-    name: docker
-
-- name: Install Docker Compose Module for Python
-  pip:
-    name: docker-compose
-
-- name: Install docker-compose
-  get_url:
-    url : https://github.com/docker/compose/releases/download/1.25.1-rc1/docker-compose-Linux-x86_64
-    dest: /usr/local/bin/docker-compose
-    mode: 'u+x,g+x'
-#
-## Change the docker dir.
-#- name: Stop docker service
-#  ansible.builtin.service:
-#    name: docker
-#    state: stopped
-#
-#- name: Change volume path
-#  tags: "skip-e2e"
-#  copy:
-#    dest: "/etc/docker/daemon.json"
-#    content: |
-#      {
-#        "data-root": "{{docker_daemon_data_root}}"
-#      }
-#
-#- name: Backup old files
-#  tags: "skip-e2e"
-#  ansible.builtin.shell: cp -r /var/lib/docker /var/lib/docker.bak
-#
-#- name: Delete old docker directory
-#  tags: "skip-e2e"
-#  file:
-#    state: absent
-#    path: /var/lib/docker
-#
-#- name: Start docker service
-#  tags: "skip-e2e"
-#  ansible.builtin.service:
-#    name: docker
-#    state: started

.old/homelab/tasks/install-sudo-tasks.yml

@@ -1,27 +0,0 @@
----
-# https://stackoverflow.com/questions/37333305/ansible-create-a-user-with-sudo-privileges
-- name: Install Sudo
-  apt:
-    pkg:
-      - sudo
-    state: latest
-    update_cache: true
-
-- name: Make sure we have a '{{homelab_group}}' group
-  group:
-    name: "{{homelab_group}}"
-    state: present
-- name: Allow '{{homelab_user}}' group to have passwordless sudo
-  lineinfile:
-    dest: /etc/sudoers
-    state: present
-    regexp: '^%{{homelab_user}}'
-    line: '%{{homelab_user}} ALL=(ALL) NOPASSWD: ALL'
-    validate: 'visudo -cf %s'
-
-- name: Add User
-  ansible.builtin.user:
-    name: "{{homelab_user}}"
-    comment: "{{homelab_user}} user"
-    uid: 1000
-    group: "{{homelab_group}}"

.old/homelab/tasks/restore-volume-tasks.yml

@@ -1,40 +0,0 @@
----
-# https://stackoverflow.com/questions/45237632/ansible-w-docker-show-current-container-state
-- name: Get container info
-  docker_container_info:
-    name: "{{container_name}}"
-  register: result
-# We find the volumes
-- name: Find volumes
-  ansible.builtin.shell: docker run --rm
-    -v /var/run/docker.sock:/var/run/docker.sock
-    -v "{{backup_dir}}:{{backup_dir}}"
-    "{{docker_volume_backup_image}}"
-    list-backups
-    --host-path "{{backup_dir}}"
-    --volume-name-filter {{container_name}}
-    --newest-only
-  register: out
-
-- name: Stop the container
-  # not out.stdout is search("null") occurs when there is no volume in the backups dir
-  when: result.exists and not out.stdout is search("null")
-  docker_container:
-    name: "{{container_name}}"
-    state: stopped
-- name: Restore Volume
-  ansible.builtin.command: docker run --rm
-    -v /var/run/docker.sock:/var/run/docker.sock
-    {{docker_volume_backup_image}}
-    restore-volume
-    --volume {{item.volumeName}}
-    --archive {{item.absoluteFilePath}}
-  with_items: "{{ out.stdout }}"
-  # the output will be "null" if there are no backups. We simply skip if this is the case
-  when: not out.stdout is search("null")
-- name: Start the container
-  # not out.stdout is search("null") occurs when there is no volume in the backups dir
-  when: result.exists and not out.stdout is search("null")
-  docker_container:
-    name: "{{container_name}}"
-    state: started

.old/homelab/variables.yml

@@ -1,23 +0,0 @@
----
-homelab_user: cianhatton
-homelab_group: cianhatton
-backup_dir: /mnt/hdds/backups/
-docker_volume_backup_image: ghcr.io/chatton/docker-volume-backup:master
-home_lab_checkout_dir: /home/cianhatton/checkouts/HomeLab
-hdds_uuid: 8d6e8b33-7d7f-456b-9976-7baea0cf4260
-ssd0_uuid: c528bf82-61ab-4f3d-87e0-d1e6e02ef7ec
-#docker_daemon_data_root: /mnt/docker
-
-networks:
-  - nextcloud_net
-stacks:
-  - linkding
-  - nginx-proxy-manager
-  - vpn-stack
-  - mariadb
-  - nextcloud
-  - photoprism
-  - plex
-  - mealie
-  - overseerr
-  - uptime-kuma

roles/setup_hosted_services/defaults/main.yml

@@ -24,15 +24,15 @@ qnap:
   # path on qnap where plex transcoding happens
   transcoding_dir: /mnt/ssd0/transcoding
   # path on qnap where movies are stored
-  movies_dir: /mnt/test/media/movies
+  movies_dir: /mnt/mergerfs/media/movies
   # path on qnap where tv shows are stored
-  tv_dir: /mnt/test/media/tv
+  tv_dir: /mnt/mergerfs/media/tv
   # path on qnap where docker compose files are stored
   docker_compose_directory: /etc/docker-compose
   # path on qnap where backups are stored
-  backups_dir: /mnt/test/backups
+  backups_dir: /mnt/mergerfs/backups
 
-  photoprism_dir: /mnt/test/photoprism
+  photoprism_dir: /mnt/mergerfs/photoprism
 
 # dashy related config
 dashy:

roles/setup_mergerfs/defaults/main.yml

@@ -0,0 +1,21 @@
+---
+# defaults file for setup_mergerfs
+mounts:
+- path: /mnt/mergerfs
+  state: mounted
+  branches:
+    - "/mnt/data/device0"
+    - "/mnt/data/device1"
+    - "/mnt/data/device2"
+  options: allow_other,use_ino
+
+devices:
+  - uuid: "a54c1bde-1400-4975-bf24-08c603ca3a11" # /dev/sdc1
+    path: "/mnt/data/device0"
+  - uuid: "727dddaa-f7a1-439a-995f-5f4d35322e08" # /dev/sdd1
+    path: "/mnt/data/device1"
+  - uuid: "f3cff115-9adc-4761-b1e9-e81055f3e0af" # /dev/sda1
+    path: "/mnt/data/device2"
+    # SSD for downloads / transcoding
+  - uuid: "c528bf82-61ab-4f3d-87e0-d1e6e02ef7ec" # /dev/sdf
+    path: "/mnt/ssd0/"

roles/setup_mergerfs/handlers/main.yml

@@ -0,0 +1,2 @@
+---
+# handlers file for setup_mergerfs

roles/setup_mergerfs/meta/main.yml

@@ -0,0 +1,53 @@
+galaxy_info:
+  author: your name
+  namespace: chatton
+  description: your role description
+  company: your company (optional)
+
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+
+  # Choose a valid license ID from https://spdx.org - some suggested licenses:
+  # - BSD-3-Clause (default)
+  # - MIT
+  # - GPL-2.0-or-later
+  # - GPL-3.0-only
+  # - Apache-2.0
+  # - CC-BY-4.0
+  license: license (GPL-2.0-or-later, MIT, etc)
+
+  min_ansible_version: 2.1
+
+  # If this a Container Enabled role, provide the minimum Ansible Container version.
+  # min_ansible_container_version:
+
+  #
+  # Provide a list of supported platforms, and for each platform a list of versions.
+  # If you don't wish to enumerate all versions for a particular platform, use 'all'.
+  # To view available platforms and versions (or releases), visit:
+  # https://galaxy.ansible.com/api/v1/platforms/
+  #
+  # platforms:
+  # - name: Fedora
+  #   versions:
+  #   - all
+  #   - 25
+  # - name: SomePlatform
+  #   versions:
+  #   - all
+  #   - 1.0
+  #   - 7
+  #   - 99.99
+
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is a keyword that describes
+    # and categorizes the role. Users find roles by searching for tags. Be sure to
+    # remove the '[]' above, if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
+    #       Maximum 20 tags per role.
+
+dependencies: []
+  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
+  # if you add dependencies to this list.

roles/setup_mergerfs/tasks/main.yml

@@ -0,0 +1,41 @@
+---
+# dangerous to use this as the device name changes when there are new drives.
+# can this be done with an ID?
+#- name: Create a ext4 filesystem on /dev/sdb1
+#  filesystem:
+#    fstype: ext4
+#    dev: /dev/sdb1
+#    state: present
+
+- name: Mount Volumes
+  mount:
+    path: "{{item.path}}"
+    src: "UUID={{item.uuid}}"
+    fstype: ext4
+    state: mounted
+    opts: defaults,nofail
+  loop: "{{ devices }}"
+  register: volume_out
+
+- debug: msg="{{volume_out}}"
+
+# tasks file for setup_mergerfs
+- name: Create Mergefs directory
+  file:
+    path: /mnt/mergerfs
+    state: directory
+    owner: cianhatton
+    group: cianhatton
+    mode: 0775
+  register: directory_creation
+
+# Ensure the actual folder can't be written to only when it has just been created.
+- name: Make /mnt/mergerfs immutable
+  command: chattr +i /mnt/mergerfs
+  when: directory_creation.changed == true
+
+- name: Create Mergerfs pool
+  include_role:
+    name: sprat.mergerfs
+  vars:
+    mergerfs_mounts: "{{mounts}}"

roles/setup_mergerfs/vars/main.yml

@@ -0,0 +1,2 @@
+---
+# vars file for setup_mergerfs

setup-homelab.yml

@@ -4,6 +4,8 @@
   vars_files:
     - secrets-vault.yml
   roles:
+    - role: 'roles/setup_mergerfs'
+      tags: ["mergerfs"]
     - role: 'roles/setup_users'
       tags: ["users"]
     - role: 'roles/setup_samba'