diff --git a/.old/homelab/hosts b/.old/homelab/hosts deleted file mode 100644 index b345d20..0000000 --- a/.old/homelab/hosts +++ /dev/null @@ -1,2 +0,0 @@ -[servers] -localhost \ No newline at end of file diff --git a/.old/homelab/playbooks/mount-hdds.yml b/.old/homelab/playbooks/mount-hdds.yml deleted file mode 100644 index 43252a1..0000000 --- a/.old/homelab/playbooks/mount-hdds.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -- hosts: servers - vars_files: - - ../variables.yml - become: true - tasks: - - name: Ensure /mnt/hdds directory - file: - path: /mnt/hdds - state: directory - - - name: Unmount HDDS - ansible.posix.mount: - path: /mnt/hdds/ - src: "UUID={{hdds_uuid}}" - fstype: ext4 - state: unmounted - opts: defaults,nofail - - - name: Make /mnt/hdds immutable - ansible.builtin.shell: chattr +i /mnt/hdds - - - name: Mount HDDS - ansible.posix.mount: - path: /mnt/hdds - src: "UUID={{hdds_uuid}}" - fstype: ext4 - state: mounted - opts: defaults,nofail - - - name: Ensure /mnt/ssd0 directory - file: - path: /mnt/ssd0 - state: directory - - - name: Unmount SSD0 - ansible.posix.mount: - path: /mnt/ssd0/ - src: "UUID={{ssd0_uuid}}" - fstype: ext4 - state: unmounted - opts: defaults,nofail - - - name: Make /mnt/ssd0 immutable - ansible.builtin.shell: chattr +i /mnt/ssd0 - - - name: Mount SSD0 - ansible.posix.mount: - path: /mnt/ssd0/ - src: "UUID={{ssd0_uuid}}" - fstype: ext4 - state: mounted - opts: defaults,nofail diff --git a/.old/homelab/playbooks/redeploy-dashy.yml b/.old/homelab/playbooks/redeploy-dashy.yml deleted file mode 100644 index f408c42..0000000 --- a/.old/homelab/playbooks/redeploy-dashy.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- hosts: servers - vars_files: - - ../variables.yml - become: true - tasks: - - name: Checkout Git Repos - import_tasks: ../tasks/checkout-git-repos-tasks.yml - - - name: Restart Dashy - ansible.builtin.command: docker restart dashy diff --git a/.old/homelab/playbooks/restore-all.yml b/.old/homelab/playbooks/restore-all.yml deleted file mode 100644 index b0b4787..0000000 --- a/.old/homelab/playbooks/restore-all.yml +++ /dev/null @@ -1,163 +0,0 @@ -### -# restore-all restores a manually specified list of volumes. -# This is intended for user when initially setting up a home lab environment -# and want to restore all applications from existing backups -# -# Restores will be skipped if there is no archive in the backup directory. -# -# If a container exists, it will be stopped, restored and started -# If a container does not exist, the volume will be still be created. -### - -############################################ -# TODO: add a command which lists all the stack, and each stack can list all the containers. From there we can iterate -# through each container in the stack calling restore-volume-tasks.yml and then call create-stack-tasks.yml -############################################ - ---- -- name: Redeploy Portainer and All Volumes - hosts: servers - vars_files: - - ../variables.yml - - ~/.homelab/variables.yml - tasks: - - name: Create next cloud a network - docker_network: - name: nextcloud_net - - # The docker volume backup image utilizes these images - - name: Pull ubuntu - docker_image: - name: ubuntu - source: pull - - name: Pull busybox - docker_image: - name: busybox - source: pull - - name: Pull docker-volume-backup - docker_image: - name: "{{docker_volume_backup_image}}" - source: pull - - name: Pull start-portainer-stack - docker_image: - name: ghcr.io/chatton/start-portainer-stack:master - source: pull - - - name: Restore Portainer - import_tasks: ../tasks/restore-volume-tasks.yml - vars: - container_name: "portainer" - - - name: "Restore {{item}}" - import_tasks: ../tasks/restore-volume-tasks.yml - vars: - container_name: "{{item}}" - with_items: {{stacks}} - - - name: "Start {{item}} Stack" - import_tasks: ../tasks/create-stack-tasks.yml - vars: - stack_name: "{{item}}" - with_items: {{stacks}} - - -# - name: Restore Linkding -# import_tasks: ../tasks/restore-volume-tasks.yml -# vars: -# container_name: "linkding" -# -# - name: Start Linkding Stack -# import_tasks: ../tasks/create-stack-tasks.yml -# vars: -# stack_name: "linkding" -# -# - name: Restore Nginx Proxy Manager -# import_tasks: ../tasks/restore-volume-tasks.yml -# vars: -# container_name: "nginx-proxy-manager" -# -# - name: Start Nginx Proxy Manager -# import_tasks: ../tasks/create-stack-tasks.yml -# vars: -# stack_name: "nginx-proxy-manager" -# -# - name: Restore Vpn Stack -# import_tasks: ../tasks/restore-volume-tasks.yml -# vars: -# container_name: "vpn-stack" -# -# - name: Start Vpn Stack -# import_tasks: ../tasks/create-stack-tasks.yml -# vars: -# stack_name: "vpn-stack" -# -# -# - name: Restore MariaDB -# import_tasks: ../tasks/restore-volume-tasks.yml -# vars: -# container_name: "mariadb" -# -# - name: Start MariaDB -# import_tasks: ../tasks/create-stack-tasks.yml -# vars: -# stack_name: "mariadb" -# -# - name: Restore Nextcloud -# import_tasks: ../tasks/restore-volume-tasks.yml -# vars: -# container_name: "nextcloud" -# -# - name: Start Nextcloud -# import_tasks: ../tasks/create-stack-tasks.yml -# vars: -# stack_name: "nextcloud" -# -# - name: Restore Photoprism -# import_tasks: ../tasks/restore-volume-tasks.yml -# vars: -# container_name: "photoprism" -# -# - name: Start Photoprism -# import_tasks: ../tasks/create-stack-tasks.yml -# vars: -# stack_name: "photoprism" -# -# - name: Restore Plex -# import_tasks: ../tasks/restore-volume-tasks.yml -# vars: -# container_name: "plex" -# -# - name: Start Plex -# import_tasks: ../tasks/create-stack-tasks.yml -# vars: -# stack_name: "plex" -# -# - name: Restore Mealie -# import_tasks: ../tasks/restore-volume-tasks.yml -# vars: -# container_name: "mealie" -# -# - name: Start Mealie -# import_tasks: ../tasks/create-stack-tasks.yml -# vars: -# stack_name: "mealie" -# -# - name: Restore Overseerr -# import_tasks: ../tasks/restore-volume-tasks.yml -# vars: -# container_name: "overseerr" -# -# - name: Start Overseerr -# import_tasks: ../tasks/create-stack-tasks.yml -# vars: -# stack_name: "overseerr" -# -# - name: Restore Uptime Kuma -# import_tasks: ../tasks/restore-volume-tasks.yml -# vars: -# container_name: "uptime-kuma" -# -# - name: Start Uptime Kuma -# import_tasks: ../tasks/create-stack-tasks.yml -# vars: -# stack_name: "uptime-kuma" diff --git a/.old/homelab/playbooks/setup-home-lab.yml b/.old/homelab/playbooks/setup-home-lab.yml deleted file mode 100644 index 92b5c5c..0000000 --- a/.old/homelab/playbooks/setup-home-lab.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -- hosts: servers - become: true - vars_files: - - ../variables.yml - tasks: - - name: Update and upgrade apt packages - apt: - upgrade: yes - update_cache: yes - cache_valid_time: 86400 - - - name: Install aptitude - apt: - name: aptitude - state: latest - update_cache: true - - - name: Install Dev Tooling - import_tasks: ../tasks/install-dev-tooling-tasks.yml - - - name: Install & Configure Sudo - import_tasks: ../tasks/install-sudo-tasks.yml - - - name: Checkout Git Repos - import_tasks: ../tasks/checkout-git-repos-tasks.yml - - - name: Install Docker - import_tasks: ../tasks/install-docker-tasks.yml - - - name: Create and Portainer - tags: "skip-e2e" # we can't mount volumes on test runners. - community.docker.docker_compose: - project_src: "{{home_lab_checkout_dir}}/docker-compose/portainer" diff --git a/.old/homelab/playbooks/volume-restore.yml b/.old/homelab/playbooks/volume-restore.yml deleted file mode 100644 index 6f6685a..0000000 --- a/.old/homelab/playbooks/volume-restore.yml +++ /dev/null @@ -1,11 +0,0 @@ -### Example: -# ansible-playbook -v ansible/playbooks/volume-restore.yml -e "container_name=mariadb" ---- -- hosts: servers - vars_files: - - ../variables.yml - tasks: - - name: Restore Volume - import_tasks: ../tasks/restore-volume-tasks.yml - vars: - container_name: "{{ container_name }}" diff --git a/.old/homelab/tasks/checkout-git-repos-tasks.yml b/.old/homelab/tasks/checkout-git-repos-tasks.yml deleted file mode 100644 index 099a7ae..0000000 --- a/.old/homelab/tasks/checkout-git-repos-tasks.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- name: Git checkout HomeLab repository - ansible.builtin.git: - repo: 'https://github.com/chatton/HomeLab.git' - dest: "{{home_lab_checkout_dir}}" - force: true - version: master - -- name: Change Ownership - file: - path: "{{home_lab_checkout_dir}}" - owner: "{{homelab_user}}" - group: "{{homelab_group}}" \ No newline at end of file diff --git a/.old/homelab/tasks/create-stack-tasks.yml b/.old/homelab/tasks/create-stack-tasks.yml deleted file mode 100644 index 9a1fa07..0000000 --- a/.old/homelab/tasks/create-stack-tasks.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Start stack - ansible.builtin.command: docker run --rm - -e PORTAINER_USER_NAME=admin - -e PORTAINER_PASSWORD="{{portainer_password}}" - -e PORTAINER_BASE_URL="http://{{ansible_eth0.ipv4.address}}:9000" - ghcr.io/chatton/start-portainer-stack:master "{{stack_name}}" diff --git a/.old/homelab/tasks/install-dev-tooling-tasks.yml b/.old/homelab/tasks/install-dev-tooling-tasks.yml deleted file mode 100644 index 6c94d8c..0000000 --- a/.old/homelab/tasks/install-dev-tooling-tasks.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- name: Install Dev Tooling - ansible.builtin.apt: - pkg: - - jq - - fzf - - git - - mdadm - - parted - - ncdu - - tree diff --git a/.old/homelab/tasks/install-docker-tasks.yml b/.old/homelab/tasks/install-docker-tasks.yml deleted file mode 100644 index 22d50c0..0000000 --- a/.old/homelab/tasks/install-docker-tasks.yml +++ /dev/null @@ -1,74 +0,0 @@ ---- -- name: Install required system packages - apt: - pkg: - - apt-transport-https - - ca-certificates - - curl - - software-properties-common - - python3-pip - - virtualenv - - python3-setuptools - state: latest - update_cache: true - -- name: Add Docker GPG apt Key - apt_key: - url: https://download.docker.com/linux/ubuntu/gpg - state: present - -- name: Add Docker Repository - apt_repository: - repo: deb https://download.docker.com/linux/ubuntu focal stable - state: present - -- name: Update apt and install docker-ce - apt: - name: docker-ce - state: latest - update_cache: true - -- name: Install Docker Module for Python - pip: - name: docker - -- name: Install Docker Compose Module for Python - pip: - name: docker-compose - -- name: Install docker-compose - get_url: - url : https://github.com/docker/compose/releases/download/1.25.1-rc1/docker-compose-Linux-x86_64 - dest: /usr/local/bin/docker-compose - mode: 'u+x,g+x' -# -## Change the docker dir. -#- name: Stop docker service -# ansible.builtin.service: -# name: docker -# state: stopped -# -#- name: Change volume path -# tags: "skip-e2e" -# copy: -# dest: "/etc/docker/daemon.json" -# content: | -# { -# "data-root": "{{docker_daemon_data_root}}" -# } -# -#- name: Backup old files -# tags: "skip-e2e" -# ansible.builtin.shell: cp -r /var/lib/docker /var/lib/docker.bak -# -#- name: Delete old docker directory -# tags: "skip-e2e" -# file: -# state: absent -# path: /var/lib/docker -# -#- name: Start docker service -# tags: "skip-e2e" -# ansible.builtin.service: -# name: docker -# state: started diff --git a/.old/homelab/tasks/install-sudo-tasks.yml b/.old/homelab/tasks/install-sudo-tasks.yml deleted file mode 100644 index b4cc57b..0000000 --- a/.old/homelab/tasks/install-sudo-tasks.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -# https://stackoverflow.com/questions/37333305/ansible-create-a-user-with-sudo-privileges -- name: Install Sudo - apt: - pkg: - - sudo - state: latest - update_cache: true - -- name: Make sure we have a '{{homelab_group}}' group - group: - name: "{{homelab_group}}" - state: present -- name: Allow '{{homelab_user}}' group to have passwordless sudo - lineinfile: - dest: /etc/sudoers - state: present - regexp: '^%{{homelab_user}}' - line: '%{{homelab_user}} ALL=(ALL) NOPASSWD: ALL' - validate: 'visudo -cf %s' - -- name: Add User - ansible.builtin.user: - name: "{{homelab_user}}" - comment: "{{homelab_user}} user" - uid: 1000 - group: "{{homelab_group}}" diff --git a/.old/homelab/tasks/restore-volume-tasks.yml b/.old/homelab/tasks/restore-volume-tasks.yml deleted file mode 100644 index 20bd136..0000000 --- a/.old/homelab/tasks/restore-volume-tasks.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -# https://stackoverflow.com/questions/45237632/ansible-w-docker-show-current-container-state -- name: Get container info - docker_container_info: - name: "{{container_name}}" - register: result -# We find the volumes -- name: Find volumes - ansible.builtin.shell: docker run --rm - -v /var/run/docker.sock:/var/run/docker.sock - -v "{{backup_dir}}:{{backup_dir}}" - "{{docker_volume_backup_image}}" - list-backups - --host-path "{{backup_dir}}" - --volume-name-filter {{container_name}} - --newest-only - register: out - -- name: Stop the container - # not out.stdout is search("null") occurs when there is no volume in the backups dir - when: result.exists and not out.stdout is search("null") - docker_container: - name: "{{container_name}}" - state: stopped -- name: Restore Volume - ansible.builtin.command: docker run --rm - -v /var/run/docker.sock:/var/run/docker.sock - {{docker_volume_backup_image}} - restore-volume - --volume {{item.volumeName}} - --archive {{item.absoluteFilePath}} - with_items: "{{ out.stdout }}" - # the output will be "null" if there are no backups. We simply skip if this is the case - when: not out.stdout is search("null") -- name: Start the container - # not out.stdout is search("null") occurs when there is no volume in the backups dir - when: result.exists and not out.stdout is search("null") - docker_container: - name: "{{container_name}}" - state: started diff --git a/.old/homelab/variables.yml b/.old/homelab/variables.yml deleted file mode 100644 index 2506e82..0000000 --- a/.old/homelab/variables.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -homelab_user: cianhatton -homelab_group: cianhatton -backup_dir: /mnt/hdds/backups/ -docker_volume_backup_image: ghcr.io/chatton/docker-volume-backup:master -home_lab_checkout_dir: /home/cianhatton/checkouts/HomeLab -hdds_uuid: 8d6e8b33-7d7f-456b-9976-7baea0cf4260 -ssd0_uuid: c528bf82-61ab-4f3d-87e0-d1e6e02ef7ec -#docker_daemon_data_root: /mnt/docker - -networks: - - nextcloud_net -stacks: - - linkding - - nginx-proxy-manager - - vpn-stack - - mariadb - - nextcloud - - photoprism - - plex - - mealie - - overseerr - - uptime-kuma diff --git a/roles/setup_hosted_services/defaults/main.yml b/roles/setup_hosted_services/defaults/main.yml index 2808b17..f41ae9a 100644 --- a/roles/setup_hosted_services/defaults/main.yml +++ b/roles/setup_hosted_services/defaults/main.yml @@ -24,15 +24,15 @@ qnap: # path on qnap where plex transcoding happens transcoding_dir: /mnt/ssd0/transcoding # path on qnap where movies are stored - movies_dir: /mnt/test/media/movies + movies_dir: /mnt/mergerfs/media/movies # path on qnap where tv shows are stored - tv_dir: /mnt/test/media/tv + tv_dir: /mnt/mergerfs/media/tv # path on qnap where docker compose files are stored docker_compose_directory: /etc/docker-compose # path on qnap where backups are stored - backups_dir: /mnt/test/backups + backups_dir: /mnt/mergerfs/backups - photoprism_dir: /mnt/test/photoprism + photoprism_dir: /mnt/mergerfs/photoprism # dashy related config dashy: diff --git a/roles/setup_mergerfs/defaults/main.yml b/roles/setup_mergerfs/defaults/main.yml new file mode 100644 index 0000000..c29ec3a --- /dev/null +++ b/roles/setup_mergerfs/defaults/main.yml @@ -0,0 +1,21 @@ +--- +# defaults file for setup_mergerfs +mounts: +- path: /mnt/mergerfs + state: mounted + branches: + - "/mnt/data/device0" + - "/mnt/data/device1" + - "/mnt/data/device2" + options: allow_other,use_ino + +devices: + - uuid: "a54c1bde-1400-4975-bf24-08c603ca3a11" # /dev/sdc1 + path: "/mnt/data/device0" + - uuid: "727dddaa-f7a1-439a-995f-5f4d35322e08" # /dev/sdd1 + path: "/mnt/data/device1" + - uuid: "f3cff115-9adc-4761-b1e9-e81055f3e0af" # /dev/sda1 + path: "/mnt/data/device2" + # SSD for downloads / transcoding + - uuid: "c528bf82-61ab-4f3d-87e0-d1e6e02ef7ec" # /dev/sdf + path: "/mnt/ssd0/" diff --git a/roles/setup_mergerfs/handlers/main.yml b/roles/setup_mergerfs/handlers/main.yml new file mode 100644 index 0000000..7d12f81 --- /dev/null +++ b/roles/setup_mergerfs/handlers/main.yml @@ -0,0 +1,2 @@ +--- +# handlers file for setup_mergerfs diff --git a/roles/setup_mergerfs/meta/main.yml b/roles/setup_mergerfs/meta/main.yml new file mode 100644 index 0000000..fe3deea --- /dev/null +++ b/roles/setup_mergerfs/meta/main.yml @@ -0,0 +1,53 @@ +galaxy_info: + author: your name + namespace: chatton + description: your role description + company: your company (optional) + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Choose a valid license ID from https://spdx.org - some suggested licenses: + # - BSD-3-Clause (default) + # - MIT + # - GPL-2.0-or-later + # - GPL-3.0-only + # - Apache-2.0 + # - CC-BY-4.0 + license: license (GPL-2.0-or-later, MIT, etc) + + min_ansible_version: 2.1 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + # + # Provide a list of supported platforms, and for each platform a list of versions. + # If you don't wish to enumerate all versions for a particular platform, use 'all'. + # To view available platforms and versions (or releases), visit: + # https://galaxy.ansible.com/api/v1/platforms/ + # + # platforms: + # - name: Fedora + # versions: + # - all + # - 25 + # - name: SomePlatform + # versions: + # - all + # - 1.0 + # - 7 + # - 99.99 + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. diff --git a/roles/setup_mergerfs/tasks/main.yml b/roles/setup_mergerfs/tasks/main.yml new file mode 100644 index 0000000..7a7c21d --- /dev/null +++ b/roles/setup_mergerfs/tasks/main.yml @@ -0,0 +1,41 @@ +--- +# dangerous to use this as the device name changes when there are new drives. +# can this be done with an ID? +#- name: Create a ext4 filesystem on /dev/sdb1 +# filesystem: +# fstype: ext4 +# dev: /dev/sdb1 +# state: present + +- name: Mount Volumes + mount: + path: "{{item.path}}" + src: "UUID={{item.uuid}}" + fstype: ext4 + state: mounted + opts: defaults,nofail + loop: "{{ devices }}" + register: volume_out + +- debug: msg="{{volume_out}}" + +# tasks file for setup_mergerfs +- name: Create Mergefs directory + file: + path: /mnt/mergerfs + state: directory + owner: cianhatton + group: cianhatton + mode: 0775 + register: directory_creation + +# Ensure the actual folder can't be written to only when it has just been created. +- name: Make /mnt/mergerfs immutable + command: chattr +i /mnt/mergerfs + when: directory_creation.changed == true + +- name: Create Mergerfs pool + include_role: + name: sprat.mergerfs + vars: + mergerfs_mounts: "{{mounts}}" diff --git a/roles/setup_mergerfs/vars/main.yml b/roles/setup_mergerfs/vars/main.yml new file mode 100644 index 0000000..d6020ae --- /dev/null +++ b/roles/setup_mergerfs/vars/main.yml @@ -0,0 +1,2 @@ +--- +# vars file for setup_mergerfs diff --git a/setup-homelab.yml b/setup-homelab.yml index 055e326..b9bad5c 100644 --- a/setup-homelab.yml +++ b/setup-homelab.yml @@ -4,6 +4,8 @@ vars_files: - secrets-vault.yml roles: + - role: 'roles/setup_mergerfs' + tags: ["mergerfs"] - role: 'roles/setup_users' tags: ["users"] - role: 'roles/setup_samba'