Portainer stacks ansible (#3)

.gitignore

@@ -1,2 +1,3 @@
 .idea
+secrets.yml
 venv

ansible/ansible.cfg

@@ -0,0 +1,5 @@
+[defaults]
+DEFAULT_MODULE_PATH=/Users/chatton/checkouts/HomeLab/ansible/library
+# this isn't working? I had to use
+# export ANSIBLE_LIBRARY="/Users/chatton/checkouts/HomeLab/ansible/library"
+# instead

ansible/library/portainer.py

@@ -0,0 +1,265 @@
+#!/usr/bin/python
+
+from __future__ import (absolute_import, division, print_function)
+
+__metaclass__ = type
+
+import requests
+
+DOCUMENTATION = r'''
+---
+module: my_test
+
+short_description: This is my test module
+
+# If this is part of a collection, you need to use semantic versioning,
+# i.e. the version is of the form "2.5.0" and not "2.4".
+version_added: "1.0.0"
+
+description: This is my longer description explaining my test module.
+
+options:
+    name:
+        description: This is the message to send to the test module.
+        required: true
+        type: str
+    new:
+        description:
+            - Control to demo if the result of this module is changed or not.
+            - Parameter description can be a list as well.
+        required: false
+        type: bool
+# Specify this value according to your collection
+# in format of namespace.collection.doc_fragment_name
+extends_documentation_fragment:
+    - my_namespace.my_collection.my_doc_fragment_name
+
+author:
+    - Your Name (@chatton)
+'''
+
+EXAMPLES = r'''
+# Pass in a message
+- name: Test with a message
+  my_namespace.my_collection.my_test:
+    name: hello world
+
+# pass in a message and have changed true
+- name: Test with a message and changed output
+  my_namespace.my_collection.my_test:
+    name: hello world
+    new: true
+
+# fail the module
+- name: Test failure of the module
+  my_namespace.my_collection.my_test:
+    name: fail me
+'''
+
+RETURN = r'''
+# These are examples of possible return values, and in general should use other names for return values.
+original_message:
+    description: The original name param that was passed in.
+    type: str
+    returned: always
+    sample: 'hello world'
+message:
+    description: The output message that the test module generates.
+    type: str
+    returned: always
+    sample: 'goodbye'
+'''
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def _extract_creds(module):
+    return {
+        "username": module.params["username"],
+        "password": module.params["password"],
+        "base_url": module.params["base_url"],
+    }
+
+
+def _get_jwt_token(creds):
+    payload = {
+        "Username": creds["username"],
+        "Password": creds["password"],
+    }
+
+    base_url = creds["base_url"]
+    auth_url = f"{base_url}/api/auth"
+    resp = requests.post(auth_url, json=payload)
+    resp.raise_for_status()
+    return resp.json()["jwt"]
+
+
+COMPOSE_STACK = 2
+STRING_METHOD = "string"
+
+
+def _query_params_to_string(params):
+    s = "?"
+    for k, v in params.items():
+        s += f"&{k}={v}"
+    return s
+
+
+class PortainerClient:
+    def __init__(self, creds):
+        self.base_url = creds["base_url"]
+        self.token = _get_jwt_token(creds)
+        self.headers = {
+            "Authorization": f"Bearer {self.token}"
+        }
+
+    def get(self, endpoint):
+        url = f"{self.base_url}/api/{endpoint}"
+        res = requests.get(url, headers=self.headers)
+        res.raise_for_status()
+        return res.json()
+
+    def delete(self, endpoint):
+        url = f"{self.base_url}/api/{endpoint}"
+        try:
+            # TODO: deletion works, but the request fails?
+            res = requests.delete(url, headers=self.headers)
+            res.raise_for_status()
+        except Exception:
+            pass
+        return {}
+
+    def put(self, endpoint, body):
+        url = f"{self.base_url}/api/{endpoint}"
+        res = requests.put(url, json=body, headers=self.headers)
+        res.raise_for_status()
+        return res.json()
+
+    def post(self, endpoint, body, query_params=None):
+        url = f"{self.base_url}/api/{endpoint}" + _query_params_to_string(query_params)
+
+        res = requests.post(url, json=body, headers=self.headers)
+        res.raise_for_status()
+        return res.json()
+
+
+def _create_stack(client, module):
+    target_stack_name = module.params["stack_name"]
+    with open(module.params["docker_compose_file_path"]) as f:
+        file_contents = f.read()
+    body = {
+        "env": [],
+        "name": target_stack_name,
+        "stackFileContent": file_contents,
+    }
+
+    query_params = {
+        "type": COMPOSE_STACK,
+        "method": STRING_METHOD,
+        "endpointId": 2,
+    }
+    return client.post("stacks", body=body, query_params=query_params)
+
+
+def _update_stack(client, module, stack_id):
+    target_stack_name = module.params["stack_name"]
+    with open(module.params["docker_compose_file_path"]) as f:
+        file_contents = f.read()
+    return client.put(f"stacks/{stack_id}?&endpointId=2", body={
+        "env": [],
+        "name": target_stack_name,
+        "stackFileContent": file_contents,
+    })
+
+
+def handle_state_present(client, module):
+    result = dict(
+        changed=False,
+        stack_name=module.params["stack_name"]
+    )
+
+    already_exists = False
+    stacks = client.get("stacks")
+    result["stacks"] = stacks
+
+    target_stack_name = module.params["stack_name"]
+    for stack in stacks:
+        if stack["Name"] == target_stack_name:
+            already_exists = True
+            result["stack_id"] = stack["Id"]
+            break
+
+    if not already_exists:
+        stack = _create_stack(client, module)
+        result["changed"] = True
+        result["stack_id"] = stack["Id"]
+        module.exit_json(**result)
+        return
+
+    # TODO: is it possible to know if we've changed the stack?
+    # the stack exists, we just want to update it.
+    _update_stack(client, module, result["stack_id"])
+    result["changed"] = True
+    module.exit_json(**result)
+
+
+def handle_state_absent(client, module):
+    result = dict(
+        changed=False,
+        stack_name=module.params["stack_name"]
+    )
+    already_exists = False
+    target_stack_name = module.params["stack_name"]
+    stacks = client.get("stacks")
+    for stack in stacks:
+        if stack["Name"] == target_stack_name:
+            already_exists = True
+            result["stack_id"] = stack["Id"]
+            break
+
+    if not already_exists:
+        module.exit_json(**result)
+        return
+
+    stack_id = result['stack_id']
+    client.delete(f"stacks/{stack_id}" + _query_params_to_string({"endpointId": 2}))
+    result["changed"] = True
+    module.exit_json(**result)
+
+
+def run_module():
+    # define available arguments/parameters a user can pass to the module
+    module_args = dict(
+        stack_name=dict(type='str', required=True),
+        docker_compose_file_path=dict(type='str', required=True),
+        env_file_path=dict(type='str', required=False),
+        username=dict(type='str', default='admin'),
+        password=dict(type='str', required=True),
+        base_url=dict(type='str', default="http://localhost:9000"),
+        state=dict(type='str', default="present")
+    )
+
+    state_fns = {
+        "present": handle_state_present,
+        "absent": handle_state_absent
+    }
+
+    # the AnsibleModule object will be our abstraction working with Ansible
+    # this includes instantiation, a couple of common attr would be the
+    # args/params passed to the execution, as well as if the module
+    # supports check mode
+    module = AnsibleModule(
+        argument_spec=module_args,
+        supports_check_mode=False
+    )
+
+    client = PortainerClient(creds=_extract_creds(module))
+    state_fns[module.params["state"]](client, module)
+
+
+def main():
+    run_module()
+
+
+if __name__ == '__main__':
+    main()

ansible/roles/setup_hosted_services/.yamllint

@@ -0,0 +1,33 @@
+---
+# Based on ansible-lint config
+extends: default
+
+rules:
+  braces:
+    max-spaces-inside: 1
+    level: error
+  brackets:
+    max-spaces-inside: 1
+    level: error
+  colons:
+    max-spaces-after: -1
+    level: error
+  commas:
+    max-spaces-after: -1
+    level: error
+  comments: disable
+  comments-indentation: disable
+  document-start: disable
+  empty-lines:
+    max: 3
+    level: error
+  hyphens:
+    level: error
+  indentation: disable
+  key-duplicates: enable
+  line-length: disable
+  new-line-at-end-of-file: disable
+  new-lines:
+    type: unix
+  trailing-spaces: disable
+  truthy: disable

ansible/roles/setup_hosted_services/README.md

@@ -0,0 +1,38 @@
+Role Name
+=========
+
+A brief description of the role goes here.
+
+Requirements
+------------
+
+Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
+
+Role Variables
+--------------
+
+A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
+
+Dependencies
+------------
+
+A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
+
+Example Playbook
+----------------
+
+Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
+
+    - hosts: servers
+      roles:
+         - { role: username.rolename, x: 42 }
+
+License
+-------
+
+BSD
+
+Author Information
+------------------
+
+An optional section for the role authors to include contact information, or a website (HTML is not allowed).

ansible/roles/setup_hosted_services/defaults/main.yml

@@ -0,0 +1,10 @@
+---
+docker_compose_directory: /etc/docker-compose
+services:
+  - name: mealie
+  - name: linkding
+  - name: overseerr
+  - name: nextcloud
+
+docker_networks:
+  - nextcloud_net

ansible/roles/setup_hosted_services/files/linkding/docker-compose.yml

@@ -0,0 +1,14 @@
+version: '3'
+services:
+  linkding:
+    labels:
+      ie.cianhatton.backup.enabled: "true"
+    container_name: "linkding"
+    image: sissbruecker/linkding:latest
+    ports:
+      - "9090:9090"
+    volumes:
+      - "data:/etc/linkding/data"
+    restart: unless-stopped
+volumes:
+  data:

ansible/roles/setup_hosted_services/files/mealie/docker-compose.yml

@@ -0,0 +1,26 @@
+---
+version: "3.1"
+services:
+  mealie:
+    labels:
+      ie.cianhatton.backup.enabled: "true"
+    container_name: mealie
+    image: hkotel/mealie:latest
+    restart: unless-stopped
+    ports:
+      - 9925:80
+    environment:
+      PUID: 1000
+      PGID: 1000
+
+      # Default Recipe Settings
+      RECIPE_PUBLIC: 'true'
+      RECIPE_SHOW_NUTRITION: 'true'
+      RECIPE_SHOW_ASSETS: 'true'
+      RECIPE_LANDSCAPE_VIEW: 'true'
+      RECIPE_DISABLE_COMMENTS: 'false'
+      RECIPE_DISABLE_AMOUNT: 'false'
+    volumes:
+      - data:/app/data
+volumes:
+  data:

ansible/roles/setup_hosted_services/files/nextcloud/docker-compose.yml

@@ -0,0 +1,19 @@
+version: '3.2'
+services:
+  nextcloud:
+    labels:
+      ie.cianhatton.backup.enabled: "true"
+    image: nextcloud
+    container_name: nextcloud
+    restart: unless-stopped
+    ports:
+      - 8081:80
+    volumes:
+      - data:/var/www/html
+volumes:
+  data:
+
+networks:
+  default:
+    name: nextcloud_net
+    external: true

ansible/roles/setup_hosted_services/files/overseerr/docker-compose.yml

@@ -0,0 +1,18 @@
+version: "3"
+services:
+  overseerr:
+    labels:
+      ie.cianhatton.backup.enabled: "true"
+    image: sctx/overseerr:latest
+    container_name: overseerr
+    ports:
+      - 5055:5055
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=Europe/London
+    volumes:
+      - "config:/app/config"
+    restart: unless-stopped
+volumes:
+  config:

ansible/roles/setup_hosted_services/handlers/main.yml

@@ -0,0 +1,2 @@
+---
+# handlers file for setup_hosted_services

ansible/roles/setup_hosted_services/meta/main.yml

@@ -0,0 +1,53 @@
+galaxy_info:
+  author: your name
+  namespace: chatton
+  description: your role description
+  company: your company (optional)
+
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+
+  # Choose a valid license ID from https://spdx.org - some suggested licenses:
+  # - BSD-3-Clause (default)
+  # - MIT
+  # - GPL-2.0-or-later
+  # - GPL-3.0-only
+  # - Apache-2.0
+  # - CC-BY-4.0
+  license: license (GPL-2.0-or-later, MIT, etc)
+
+  min_ansible_version: 2.1
+
+  # If this a Container Enabled role, provide the minimum Ansible Container version.
+  # min_ansible_container_version:
+
+  #
+  # Provide a list of supported platforms, and for each platform a list of versions.
+  # If you don't wish to enumerate all versions for a particular platform, use 'all'.
+  # To view available platforms and versions (or releases), visit:
+  # https://galaxy.ansible.com/api/v1/platforms/
+  #
+  # platforms:
+  # - name: Fedora
+  #   versions:
+  #   - all
+  #   - 25
+  # - name: SomePlatform
+  #   versions:
+  #   - all
+  #   - 1.0
+  #   - 7
+  #   - 99.99
+
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is a keyword that describes
+    # and categorizes the role. Users find roles by searching for tags. Be sure to
+    # remove the '[]' above, if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
+    #       Maximum 20 tags per role.
+
+dependencies: []
+  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
+  # if you add dependencies to this list.

ansible/roles/setup_hosted_services/molecule/default/converge.yml

@@ -0,0 +1,7 @@
+---
+- name: Converge
+  hosts: all
+  tasks:
+    - name: "Include chatton.setup_hosted_services"
+      include_role:
+        name: "chatton.setup_hosted_services"

ansible/roles/setup_hosted_services/molecule/default/molecule.yml

@@ -0,0 +1,29 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+platforms:
+  - name: instance
+    image: geerlingguy/docker-debian10-ansible:latest
+    privileged: true
+    pre_build_image: true
+provisioner:
+  name: ansible
+verifier:
+  name: ansible
+scenario:
+  test_sequence:
+    #    - dependency
+    #    - lint
+    #    - cleanup
+    - destroy
+    - syntax
+    - create
+    #    - prepare
+    - converge
+    - idempotence
+    #    - side_effect
+    - verify
+    #    - cleanup
+    - destroy

ansible/roles/setup_hosted_services/molecule/default/verify.yml

@@ -0,0 +1,10 @@
+---
+# This is an example playbook to execute Ansible tests.
+
+- name: Verify
+  hosts: all
+  gather_facts: false
+  tasks:
+  - name: Example assertion
+    ansible.builtin.assert:
+      that: true

ansible/roles/setup_hosted_services/tasks/main.yml

@@ -0,0 +1,33 @@
+- name: Create a directory if it does not exist
+  ansible.builtin.file:
+    path: "{{docker_compose_directory}}/{{item.name}}"
+    state: directory
+    mode: '0755'
+  with_items: "{{services}}"
+
+- name: Copy Docker Compose Files
+  copy:
+    src: "{{item.name}}/docker-compose.yml"
+    dest: "{{docker_compose_directory}}/{{item.name}}/docker-compose.yml"
+  with_items: "{{services}}"
+
+- name: Install python dependencies (requests)
+  ansible.builtin.pip:
+    name: requests
+
+- name: Install python dependencies (boto3)
+  ansible.builtin.pip:
+    name: boto3
+
+- name: Create required docker networks
+  docker_network:
+    name: "{{item}}"
+  with_items: "{{ docker_networks }}"
+
+- name: Update Portainer Stack
+  portainer:
+    username: admin
+    password: "{{portainer.password}}"
+    docker_compose_file_path: "{{docker_compose_directory}}/{{ item.name }}/docker-compose.yml"
+    stack_name: "{{ item.name }}"
+  with_items: "{{services}}"

ansible/roles/setup_hosted_services/tests/inventory

@@ -0,0 +1,2 @@
+localhost
+

ansible/roles/setup_hosted_services/tests/test.yml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  remote_user: root
+  roles:
+    - setup_hosted_services

ansible/roles/setup_hosted_services/vars/main.yml

@@ -0,0 +1,2 @@
+---
+# vars file for setup_hosted_services

ansible/setup-homelab.yml

@@ -1,7 +1,10 @@
 ---
 - hosts: servers
   become: true
+  vars_files:
+    - ../secrets.yml
   roles:
     - role: 'roles/setup_users'
     - role: 'roles/setup_docker'
     - role: 'roles/setup_portainer'
+    - role: 'roles/setup_hosted_services'

docker-compose/bitwarden/docker-compose.yml

@@ -4,11 +4,11 @@ services:
     image: bitwardenrs/server
     restart: always
     ports:
-      - ${HOST_PORT}:80
+      - 80:80
     volumes:
       - data:/data
     environment:
       WEBSOCKET_ENABLED: 'true' # Required to use websockets
-      SIGNUPS_ALLOWED: 'true'   # set to false to disable signups
+      SIGNUPS_ALLOWED: 'false'   # set to false to disable signups
 volumes:
   data:

secrets-example.yml

@@ -0,0 +1,2 @@
+portainer:
+  password: ""