From cf61f8c04b73bff20013423dab438c2eb890e558 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Sun, 28 Aug 2022 01:14:46 +0100 Subject: [PATCH] Add CI (#18) --- .github/workflows/ci.yml | 57 +++++++++++++++ group_vars/linode.yml | 16 +++++ group_vars/qnap.yml | 2 + group_vars/servers.yml | 10 +-- hosts.ini | 5 +- requirements.txt | 44 ++++++++++++ roles/setup_hosted_services/handlers/main.yml | 1 + roles/setup_hosted_services/tasks/main.yml | 1 + .../templates/dashboards.j2 | 2 - roles/setup_linode/defaults/main.yml | 4 ++ roles/setup_linode/tasks/main.yml | 5 +- roles/setup_portainer/tasks/main.yml | 2 +- setup-linode.yml | 2 - tests/.vault-pass.sh | 2 + tests/ansible.cfg | 6 ++ tests/hosts.ini | 6 ++ vault_vars/linode-vault.yml | 72 +++++++++++-------- verify-homelab.yml | 19 +++++ 18 files changed, 210 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 requirements.txt create mode 100755 tests/.vault-pass.sh create mode 100644 tests/ansible.cfg create mode 100644 tests/hosts.ini create mode 100644 verify-homelab.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..8ac09e0 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,57 @@ +--- +name: CI +on: + push: + +env: + VAULT_PASSWORD: "${{ secrets.VAULT_PASSWORD }}" + SSH_PUB: "${{ secrets.SSH_PUB }}" + +jobs: + tests: + name: Tests + runs-on: ubuntu-latest + steps: + - name: Create SSH pub + run: | + echo "creating ~/.ssh/id_rsa.pub file" + mkdir -p ~/.ssh/ + echo "$SSH_PUB" > ~/.ssh/id_rsa.pub + - name: Install SSH key + uses: shimataro/ssh-key-action@v2 + with: + key: ${{ secrets.SSH_KEY }} + known_hosts: ${{ secrets.KNOWN_HOSTS }} + - name: Check out the codebase. + uses: actions/checkout@v3 + - name: Prepare Test Files + run: | + mv tests/.vault-pass.sh ./.vault-pass.sh + mv tests/ansible.cfg ./ansible.cfg + mv tests/hosts.ini ./hosts.ini + - uses: actions/setup-python@v4 + with: + python-version: 3.8.9 + + - name: Install test dependencies. + run: | + pip install -r requirements.txt + - name: Syntax Check + run: | + ansible-playbook setup-linode.yml --syntax-check + ansible-playbook setup-homelab.yml --syntax-check + - name: Setup Linode Instance + run: | + ansible-playbook setup-linode.yml + env: + ANSIBLE_FORCE_COLOR: '1' + - name: Setup Home Lab + run: | + ansible-playbook setup-homelab.yml + env: + ANSIBLE_FORCE_COLOR: '1' + - name: Verify Home Lab + run: | + ansible-playbook verify-homelab.yml + env: + ANSIBLE_FORCE_COLOR: '1' diff --git a/group_vars/linode.yml b/group_vars/linode.yml index 40194b5..c41ab43 100644 --- a/group_vars/linode.yml +++ b/group_vars/linode.yml @@ -6,8 +6,24 @@ configure_mergefs: false configure_samba: false services: - name: gitea + - name: mealie + - name: linkding + - name: overseerr + - name: nextcloud +# - name: dashboards + - name: nginx-proxy-manager + - name: uptime-kuma +# - name: vpn-stack - name: docker-volume-backup + - name: mariadb + - name: photoprism + - name: olivetin + +# any additional docker networks that should be created +docker_networks: + - mariadb_net # use raw docker compose instead of portainer use_docker_compose: true use_portainer: false +restore_from_s3: false diff --git a/group_vars/qnap.yml b/group_vars/qnap.yml index 9015ad5..77cca77 100644 --- a/group_vars/qnap.yml +++ b/group_vars/qnap.yml @@ -50,6 +50,7 @@ services: - name: docker-volume-backup - name: mariadb - name: photoprism + - name: olivetin extra_hosts: - "qnap:192.168.178.42" @@ -60,3 +61,4 @@ docker_networks: use_portainer: true use_docker_compose: false +restore_from_s3: true diff --git a/group_vars/servers.yml b/group_vars/servers.yml index 4fb26d1..951d6e1 100644 --- a/group_vars/servers.yml +++ b/group_vars/servers.yml @@ -4,7 +4,7 @@ backups: monthly: "monthly" nightly: "nightly" -docker_compose_directory: /etc/docker-compose/portainer +docker_compose_directory: /etc/docker-compose # TODO: docker_volume_backup exists in vault. Clean this up. dockervolumebackup: @@ -14,15 +14,15 @@ dockervolumebackup: # dashy related config dashy: - config_file: dashy-config.yml source_file: dashboards/dashy-config.yml - config_directory: /etc/config/olivetin + config_file: dashy-config.yml + config_directory: /etc/config/dashy # olivetin related config olivetin: - config_file: config.yml source_file: olivetin/config.yml - config_directory: /etc/config/dashy + config_file: config.yml + config_directory: /etc/config/olivetin docker_networks: [] diff --git a/hosts.ini b/hosts.ini index 165ce6a..5e7cb1f 100644 --- a/hosts.ini +++ b/hosts.ini @@ -7,10 +7,7 @@ linode [qnap] cianhatton@qnap -[localhost] -localhost ansible_connection=local ansible_python_interpreter=/Users/chatton/checkouts/ansible-homelab/venv/bin/python - # BEGIN ANSIBLE MANAGED BLOCK [linode] -root@45.79.165.198 +root@198.74.59.177 # END ANSIBLE MANAGED BLOCK diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..04e9da6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,44 @@ +ansible==6.3.0 +ansible-compat==2.2.0 +ansible-core==2.13.3 +arrow==1.2.2 +attrs==22.1.0 +binaryornot==0.4.4 +Cerberus==1.3.2 +certifi==2022.6.15 +cffi==1.15.1 +chardet==5.0.0 +charset-normalizer==2.1.1 +click==8.1.3 +click-help-colors==0.9.1 +commonmark==0.9.1 +cookiecutter==2.1.1 +cryptography==37.0.4 +enrich==1.2.7 +idna==3.3 +importlib-resources==5.9.0 +Jinja2==3.1.2 +jinja2-time==0.2.0 +jsonschema==4.14.0 +linode-api4==5.2.1 +MarkupSafe==2.1.1 +molecule==4.0.1 +packaging==21.3 +pkgutil_resolve_name==1.3.10 +pluggy==1.0.0 +pycparser==2.21 +Pygments==2.13.0 +pyparsing==3.0.9 +pyrsistent==0.18.1 +python-dateutil==2.8.2 +python-slugify==6.1.2 +PyYAML==6.0 +requests==2.28.1 +resolvelib==0.8.1 +rich==12.5.1 +six==1.16.0 +subprocess-tee==0.3.5 +text-unidecode==1.3 +typing_extensions==4.3.0 +urllib3==1.26.12 +zipp==3.8.1 diff --git a/roles/setup_hosted_services/handlers/main.yml b/roles/setup_hosted_services/handlers/main.yml index f3e7048..91f8f4c 100644 --- a/roles/setup_hosted_services/handlers/main.yml +++ b/roles/setup_hosted_services/handlers/main.yml @@ -2,6 +2,7 @@ # handlers file for setup_hosted_services - name: restart-dashy shell: docker restart dashy + ignore_errors: True - name: restart-olivetin shell: docker restart olivetin diff --git a/roles/setup_hosted_services/tasks/main.yml b/roles/setup_hosted_services/tasks/main.yml index 173f6ee..380dcc4 100644 --- a/roles/setup_hosted_services/tasks/main.yml +++ b/roles/setup_hosted_services/tasks/main.yml @@ -78,6 +78,7 @@ - debug: msg="{{python_output.stdout_lines | list }}" - name: Docker Volume Backup | Restore any missing backups from S3 + when: restore_from_s3 docker_container: command: "restore-volume --s3 --volume {{item}}" image: "ghcr.io/chatton/docker-volume-backup:v0.3.0" diff --git a/roles/setup_hosted_services/templates/dashboards.j2 b/roles/setup_hosted_services/templates/dashboards.j2 index e5e4962..ecee342 100644 --- a/roles/setup_hosted_services/templates/dashboards.j2 +++ b/roles/setup_hosted_services/templates/dashboards.j2 @@ -36,8 +36,6 @@ services: retries: 3 start_period: 40s - extra_hosts: {{ extra_hosts }} - glances: image: nicolargo/glances:latest-alpine restart: always diff --git a/roles/setup_linode/defaults/main.yml b/roles/setup_linode/defaults/main.yml index ea79b65..bba11e6 100644 --- a/roles/setup_linode/defaults/main.yml +++ b/roles/setup_linode/defaults/main.yml @@ -8,3 +8,7 @@ label: simple-linode hosts: - user: root ip: "{{my_linode.instance.ipv4[0]}}" + +# https://www.linode.com/community/questions/17190/obtain-a-list-of-image-and-plan-types-using-linode-apicli +type: g6-standard-2 +region: us-east diff --git a/roles/setup_linode/tasks/main.yml b/roles/setup_linode/tasks/main.yml index 43f90f2..e2f0a32 100644 --- a/roles/setup_linode/tasks/main.yml +++ b/roles/setup_linode/tasks/main.yml @@ -6,8 +6,8 @@ linode_v4: label: "{{ label }}" access_token: "{{ token }}" - type: g6-nanode-1 - region: us-east + type: "{{ type }}" + region: "{{ region }}" image: linode/debian11 root_pass: "{{ password }}" authorized_keys: "{{ ssh_keys }}" @@ -15,7 +15,6 @@ state: present register: my_linode - - name: Wait for SSH to come up local_action: module: wait_for diff --git a/roles/setup_portainer/tasks/main.yml b/roles/setup_portainer/tasks/main.yml index 3c7d606..9dc58ab 100644 --- a/roles/setup_portainer/tasks/main.yml +++ b/roles/setup_portainer/tasks/main.yml @@ -45,7 +45,7 @@ - name: Portainer | Docker compose up community.docker.docker_compose: - project_src: "{{docker_compose_directory}}" + project_src: "{{docker_compose_directory}}/portainer" # Don't really need this as long as there is an S3 backup. #- name: Portainer | Register Admin User diff --git a/setup-linode.yml b/setup-linode.yml index 03cd371..13bf16a 100644 --- a/setup-linode.yml +++ b/setup-linode.yml @@ -1,7 +1,5 @@ --- - hosts: localhost become: true - vars: - ansible_python_interpreter: /Users/chatton/checkouts/ansible-homelab/venv/bin/python roles: - role: 'roles/setup_linode' diff --git a/tests/.vault-pass.sh b/tests/.vault-pass.sh new file mode 100755 index 0000000..d913f4e --- /dev/null +++ b/tests/.vault-pass.sh @@ -0,0 +1,2 @@ +#!/bin/bash +echo "${VAULT_PASSWORD}" diff --git a/tests/ansible.cfg b/tests/ansible.cfg new file mode 100644 index 0000000..992fb32 --- /dev/null +++ b/tests/ansible.cfg @@ -0,0 +1,6 @@ +[defaults] +default_module_path=library +inventory=hosts.ini +vault_password_file=.vault-pass.sh +host_key_checking = False +enable_plugins = linode diff --git a/tests/hosts.ini b/tests/hosts.ini new file mode 100644 index 0000000..a516c87 --- /dev/null +++ b/tests/hosts.ini @@ -0,0 +1,6 @@ +[servers:children] +linode + +# BEGIN ANSIBLE MANAGED BLOCK +[linode] +# END ANSIBLE MANAGED BLOCK diff --git a/vault_vars/linode-vault.yml b/vault_vars/linode-vault.yml index aff0131..3e8f4f2 100644 --- a/vault_vars/linode-vault.yml +++ b/vault_vars/linode-vault.yml @@ -1,30 +1,44 @@ $ANSIBLE_VAULT;1.1;AES256 -30366332636539663938343535343632386164326137373937363336623136336436663831666166 -6237326538393061383033633864666165333136666366630a326664636638303566316531346430 -63363363656461313162663935613065616236386332326133363331663937396232353865633834 -6137653864623962300a303035366132313939623937393863316261386238616138643866663632 -38383562613339333564393438323938343331633935643864633332626662323231323465303938 -38626633313264376464326431363562663932343263613637383939663966383230373561393465 -33396430343864326163663433376662636565643839303734383566323666356330366562626637 -36343835666334303863323839643032363635626366626339323333396164643835646364353037 -65323562313636396566363234656566383561396564383265356630633366373930376665323038 -65333631353836623333323765363866323062303236333030323564383332306466666639306332 -33393434626239363861343130363362333734363662656236303833636165633231316233316132 -65353930656230326461326436396533393433316331633837616531303861326138353831366436 -36356266613236393165333339653334356338636636346166363736653063313531653339646565 -35383233343965316630613236643530636430333966633466636135663931633031343634313535 -35383564663631663431313536336231663337663537346131653534393163633266353430666531 -34643362363239613561623064626331353639333766623066666638313866663863396636623331 -38313437623261626131636261646331376366646635326332376332633338653136633732616237 -31323531383130343436623838366439316539396263323864396266646630666334343237303961 -64333430336265316565303264666636646431666137323133663035346430346331313634656637 -39663165356263343263393863653338303366343830306139353033393463616666316136653532 -64666361353536386162653934663936336465393730653431336437393161303166623231336562 -38306131336537386362323730656234373839643538616638326332633735626639343633346365 -38356234616165363837306634383937653963333936633030666435333534313235313962363263 -63353932613939363166613336336266333064336262383062303761616535656534653064393066 -34333832623831653965393166633065663965386162373238663632656636326265333464303530 -33633037653730613135313335343264343730396236626563353235663037366339303638623930 -35373765343436613531656537656636613663366438653233346465396330393034623537653733 -30366466326362636464313963613866306565616630363366616263306264346561613332393835 -30386561383334383361643834376536643931616533656330353166396335333731 +63353635333236643065633063323665363937346666333930363433326561643064323366336535 +3439323961653433373738333633363666646136303631330a366534393237643331623763376561 +33636436626666303663663134346161613561323561656139633165333764363336316235613632 +3738383662663137350a613230313936663635626666373530656361633437623336303061316266 +64643333366562386564393839383339643932636234653065373530636466616561653161356665 +39386137633866333061633037323432353833306261303638303139663632336330346432393262 +37373235656135396537373865306530616664663866353033623237623932396361373633393339 +33623737343838636634333435383864356634336363336262376134623735383161316430643231 +63323562346137636132316464643439656339326635383166323161646635313333616235623761 +32356630323732633432396631316466323834343032376139623865303337636566666632656530 +61353534633135396666336535366631653666616237313430356438386165323339346531393862 +66383737363261383164626437373931643739666635626135656364363435386532323161616439 +61376239653234313964656339313866636435656164396333616231643533643964613166633865 +30346538343733373762343439383662633736323865353637343939626638383732346462303863 +65313530306663353332396538623638363737393639643536396639386133373364336437343135 +38333431633635643661346139663265613538653932633262316331323439383233666236613361 +62326634646335393235313765353739363765333865333435336331636633643866653634363566 +36316337623131633439383465643937383966333836396165636562316233646361666566656265 +66633766623434636663643966633335663831623232346637663538373063663832323562616330 +31643634653161626563616564326563396633306462663932633838366332376161333437336438 +61343363356633643835306262376662613739613132333062646263353836346332636136653930 +34616464343231623664663164613463646563616162333033383663366438643436613732323639 +63386133366530353862353537643763303162616666313363663834386235323561666561396131 +36653563633535633532653734386637373832663137323638373236326332303562643637303938 +33313638343462303465663934313236626163663665353762663064333562636533343233343637 +31666432303365366137373839393630653162613261323839396235653531653937383064373765 +35343566623735356138333837383661306533343334396436623336316261306666636233336239 +31313636306161326266653930343566643432363664363236323365356133663765313530353065 +33363833353964396234326636306136366465613338363665366164383764396438636464396631 +64656262376161323837326632366135663535383265393939333361323631333338653130313966 +62336537373037656638363230303531363463376235373931333332373430643365633362623736 +34393531353132366139346338343636366437666334643563346330343661626434353337653861 +63346261653063363933316133633431343539373734343064663766663065313663356662386532 +37343238666539313564643237396333613062396433653936323537313436313932393566656133 +61626432313664366563396463633866343063383164616263363766326537653830363863373532 +61386434363839663238616564356437336438626564316236303630656137303034633830326338 +33616333346566366631303235623762666437396131373961613234313239333062353836613736 +62366363343032393630616636336661386631643635313465663535353231326237633530323464 +33366530626463313031316436323931333134613263316461306162333366663538333531343462 +61383462376430363831376661393666343136333136353233373763323138653237353937623937 +31396433636365643562653937336265383861303139666136346135383431343134363365363761 +62643231306263656534333836613930623831636235386230643632386366613965386166666465 +646230326535356537626131396132643235 diff --git a/verify-homelab.yml b/verify-homelab.yml new file mode 100644 index 0000000..a42eede --- /dev/null +++ b/verify-homelab.yml @@ -0,0 +1,19 @@ +--- +- hosts: linode + tasks: + # TODO + # Verify sudoers + # Verify ssh authorized key + # Verify docker installed + # Verify docker images pulled + # Verify app config files + + - name: Docker Compose Files Exist + command: "stat {{docker_compose_directory}}/{{ item.name }}/docker-compose.yml" + with_items: "{{services}}" + changed_when: false + register: stat + - name: Assert all Docker Compose files were created + assert: + that: item.rc == 0 + with_items: "{{stat.results}}"