From e22e07c82bd6a42010ae7532caf90fda5ba6b5fd Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Fri, 2 Sep 2022 22:16:16 +0100 Subject: [PATCH] updating to use same s3 fields (#11) --- roles/docker_s3_backup/defaults/main.yml | 14 ++-- roles/docker_s3_backup/tasks/main.yml | 16 ++--- .../defaults/main.yml | 26 ++++---- roles/docker_s3_volume_restore/tasks/main.yml | 52 +++++++-------- tests/host_vars/localhost.yml | 65 +++++++------------ tests/playbook.yml | 4 +- 6 files changed, 78 insertions(+), 99 deletions(-) diff --git a/roles/docker_s3_backup/defaults/main.yml b/roles/docker_s3_backup/defaults/main.yml index 8d0dd64..dcc83ee 100644 --- a/roles/docker_s3_backup/defaults/main.yml +++ b/roles/docker_s3_backup/defaults/main.yml @@ -3,10 +3,10 @@ # the backup directory where backups are stored on the host machine. # these will be uploaded to S3. -docker_s3_backup_aws_s3_region: "us-east-1" -docker_s3_backup_aws_s3_bucket: "backups" -docker_s3_backup_host_backup_directory: "" -docker_s3_backup_aws_s3_url: "" -docker_s3_backup_aws_s3_aws_access_key: "" -docker_s3_backup_aws_s3_aws_secret_key: "" -docker_s3_backup_aws_s3_permissions: [] +docker_backup_aws_s3_region: "us-east-1" +docker_backup_aws_s3_bucket: "backups" +docker_backup_host_backup_directory: "" +docker_backup_aws_s3_url: "" +docker_backup_aws_s3_aws_access_key: "" +docker_backup_aws_s3_aws_secret_key: "" +docker_backup_aws_s3_permissions: [] diff --git a/roles/docker_s3_backup/tasks/main.yml b/roles/docker_s3_backup/tasks/main.yml index a22efe9..bc5dc02 100644 --- a/roles/docker_s3_backup/tasks/main.yml +++ b/roles/docker_s3_backup/tasks/main.yml @@ -34,7 +34,7 @@ state: started volumes: - "{{ item.Name }}:/data" - - "{{ docker_s3_backup_host_backup_directory }}:/backups" + - "{{ docker_backup_host_backup_directory }}:/backups" with_items: "{{ volume_mounts }}" - name: Start the container @@ -45,13 +45,13 @@ - name: Upload backups to S3 register: upload_result amazon.aws.aws_s3: - s3_url: "{{ docker_s3_backup_aws_s3_url }}" - bucket: "{{ docker_s3_backup_aws_s3_bucket }}" + s3_url: "{{ docker_backup_aws_s3_url }}" + bucket: "{{ docker_backup_aws_s3_bucket }}" object: "{{ item.Name }}/{{ item.Name }}-{{ backup_time }}.tar.gz" - src: "{{ docker_s3_backup_host_backup_directory }}/{{ item.Name }}-{{ backup_time }}.tar.gz" - aws_access_key: "{{ docker_s3_backup_aws_s3_aws_access_key }}" - aws_secret_key: "{{ docker_s3_backup_aws_s3_aws_secret_key }}" - region: "{{ docker_s3_backup_aws_s3_region }}" + src: "{{ docker_backup_host_backup_directory }}/{{ item.Name }}-{{ backup_time }}.tar.gz" + aws_access_key: "{{ docker_backup_aws_s3_aws_access_key }}" + aws_secret_key: "{{ docker_backup_aws_s3_aws_secret_key }}" + region: "{{ docker_backup_aws_s3_region }}" mode: put - permission: "{{ docker_s3_backup_aws_s3_permissions }}" + permission: "{{ docker_backup_aws_s3_permissions }}" with_items: "{{ volume_mounts }}" diff --git a/roles/docker_s3_volume_restore/defaults/main.yml b/roles/docker_s3_volume_restore/defaults/main.yml index a95ae63..5d0d1f5 100644 --- a/roles/docker_s3_volume_restore/defaults/main.yml +++ b/roles/docker_s3_volume_restore/defaults/main.yml @@ -2,24 +2,24 @@ # defaults file for docker_s3_volume_restore # forces a revert to the volume. -docker_s3_volume_restore_force: false -# specify docker_s3_volume_restore_latest_s3_key true to automatically determine the latest +docker_backup_restore_force: false +# specify docker_backup_restore_latest_s3_key true to automatically determine the latest # backup in the s3 backup. The format which is expected has the prefix of volume_name/volume_name_* # this is the format the the "docker_s3_backup" role updloads them with. -docker_s3_volume_restore_latest_s3_key: false -docker_volume_s3_restores: [] -# docker_volume_s3_restores: +docker_backup_restore_latest_s3_key: false +docker_backup_s3_restores: [] +# docker_backup_s3_restores: # - volume_name: "linkding_data" # s3_key: "linkding_data/linkding_data-2022-09-01T21:32:54Z.tar.gz" # dynamically find the latest linkding_data backup. -# docker_s3_volume_restore_latest_s3_key: true -# docker_volume_s3_restores: +# docker_backup_restore_latest_s3_key: true +# docker_backup_s3_restores: # - volume_name: "linkding_data" -docker_s3_volume_restore_aws_s3_region: "us-east-1" -docker_s3_volume_restore_aws_s3_bucket: "backups" -docker_s3_volume_restore_aws_s3_url: "" -docker_s3_volume_restore_aws_s3_aws_access_key: "" -docker_s3_volume_restore_aws_s3_aws_secret_key: "" -docker_s3_volume_restore_aws_s3_permissions: [] +docker_backup_aws_s3_region: "us-east-1" +docker_backup_aws_s3_bucket: "backups" +docker_backup_aws_s3_url: "" +docker_backup_aws_s3_aws_access_key: "" +docker_backup_aws_s3_aws_secret_key: "" +docker_backup_aws_s3_permissions: [] diff --git a/roles/docker_s3_volume_restore/tasks/main.yml b/roles/docker_s3_volume_restore/tasks/main.yml index 711dcaa..46e7c9d 100644 --- a/roles/docker_s3_volume_restore/tasks/main.yml +++ b/roles/docker_s3_volume_restore/tasks/main.yml @@ -7,10 +7,10 @@ name: "{{ item.volume_name }}" state: present register: volume_out - with_items: "{{ docker_volume_s3_restores }}" + with_items: "{{ docker_backup_s3_restores }}" - name: Determine if backup is needed. - ansible.builtin.set_fact: should_perform_backup="{{ docker_s3_volume_restore_force == true or volume_out.changed == true }}" + ansible.builtin.set_fact: should_perform_backup="{{ docker_backup_restore_force == true or volume_out.changed == true }}" - name: End play as no backup is needed. ansible.builtin.meta: end_play @@ -18,59 +18,59 @@ # try and find latest volume based on the name. - name: Find latest s3 version. - when: docker_s3_volume_restore_latest_s3_key + when: docker_backup_restore_latest_s3_key amazon.aws.aws_s3: - bucket: "{{ docker_s3_volume_restore_aws_s3_bucket }}" + bucket: "{{ docker_backup_aws_s3_bucket }}" mode: list prefix: "{{ item.volume_name }}/{{ item.volume_name }}" - aws_access_key: "{{ docker_s3_volume_restore_aws_s3_aws_access_key }}" - aws_secret_key: "{{ docker_s3_volume_restore_aws_s3_aws_secret_key }}" - region: "{{ docker_s3_volume_restore_aws_s3_region }}" - s3_url: "{{ docker_s3_volume_restore_aws_s3_url }}" + aws_access_key: "{{ docker_backup_aws_s3_aws_access_key }}" + aws_secret_key: "{{ docker_backup_aws_s3_aws_secret_key }}" + region: "{{ docker_backup_aws_s3_region }}" + s3_url: "{{ docker_backup_aws_s3_url }}" register: s3_list_output - with_items: "{{ docker_volume_s3_restores }}" + with_items: "{{ docker_backup_s3_restores }}" - name: Extract S3 keys for container. - when: docker_s3_volume_restore_latest_s3_key + when: docker_backup_restore_latest_s3_key ansible.builtin.set_fact: container_s3_keys="{{ container_s3_keys | default([]) + [item.s3_keys | last] }}" with_items: "{{ s3_list_output.results }}" - name: Extract volume names. - ansible.builtin.set_fact: volume_names="{{ docker_volume_s3_restores | map(attribute='volume_name') }}" - when: docker_s3_volume_restore_latest_s3_key + ansible.builtin.set_fact: volume_names="{{ docker_backup_s3_restores | map(attribute='volume_name') }}" + when: docker_backup_restore_latest_s3_key - name: Merge volume names and S3 keys. ansible.builtin.set_fact: - docker_volume_s3_restores_new: "{{ docker_volume_s3_restores_new | default([]) + [ {'volume_name': item.0, 's3_key': item.1} ] }}" - when: docker_s3_volume_restore_latest_s3_key + docker_backup_s3_restores_new: "{{ docker_backup_s3_restores_new | default([]) + [ {'volume_name': item.0, 's3_key': item.1} ] }}" + when: docker_backup_restore_latest_s3_key with_together: - "{{ volume_names }}" - "{{ container_s3_keys }}" - name: Set volumes to restore. ansible.builtin.set_fact: - docker_volume_s3_restores_new: "{{ docker_volume_s3_restores }}" - when: not docker_s3_volume_restore_latest_s3_key + docker_backup_s3_restores_new: "{{ docker_backup_s3_restores }}" + when: not docker_backup_restore_latest_s3_key - name: Create directories for /tmp file. ansible.builtin.file: path: '/tmp/{{ item.s3_key | dirname }}' state: directory mode: '0755' - with_items: '{{ docker_volume_s3_restores_new }}' + with_items: '{{ docker_backup_s3_restores_new }}' - name: Download archive from S3. amazon.aws.aws_s3: - bucket: "{{ docker_s3_volume_restore_aws_s3_bucket }}" + bucket: "{{ docker_backup_aws_s3_bucket }}" object: "{{ item.s3_key }}" - aws_access_key: "{{ docker_s3_volume_restore_aws_s3_aws_access_key }}" - aws_secret_key: "{{ docker_s3_volume_restore_aws_s3_aws_secret_key }}" - region: "{{ docker_s3_volume_restore_aws_s3_region }}" - s3_url: "{{ docker_s3_volume_restore_aws_s3_url }}" + aws_access_key: "{{ docker_backup_aws_s3_aws_access_key }}" + aws_secret_key: "{{ docker_backup_aws_s3_aws_secret_key }}" + region: "{{ docker_backup_aws_s3_region }}" + s3_url: "{{ docker_backup_aws_s3_url }}" mode: get dest: "/tmp/{{ item.s3_key }}" register: get_out - with_items: "{{ docker_volume_s3_restores_new }}" + with_items: "{{ docker_backup_s3_restores_new }}" - name: Remove contents of volumes. community.docker.docker_container: @@ -84,7 +84,7 @@ working_dir: "/data" volumes: - "{{ item.volume_name }}:/data" - with_items: "{{ docker_volume_s3_restores_new }}" + with_items: "{{ docker_backup_s3_restores_new }}" - name: Restore contents of volumes community.docker.docker_container: @@ -98,10 +98,10 @@ volumes: - "{{ item.volume_name }}:/data" - /tmp:/tmp - with_items: "{{ docker_volume_s3_restores_new }}" + with_items: "{{ docker_backup_s3_restores_new }}" - name: Remove uploaded files from /tmp ansible.builtin.file: path: '/tmp/{{ item.s3_key }}' state: absent - with_items: '{{ docker_volume_s3_restores_new }}' + with_items: '{{ docker_backup_s3_restores_new }}' diff --git a/tests/host_vars/localhost.yml b/tests/host_vars/localhost.yml index 43235ad..d2ffd79 100644 --- a/tests/host_vars/localhost.yml +++ b/tests/host_vars/localhost.yml @@ -1,44 +1,23 @@ $ANSIBLE_VAULT;1.1;AES256 -61396462383161346163353864613039623137313932386635643566333036626662353166303336 -3134363137336332663437363465353037643033663434350a316238663232353139623936343637 -65653862376431336435396539643266363735636339663662316439613637653935613566626338 -3338666636313234380a366166663233363032666430323864323032653830656233386330643233 -37613334663739373764633862373235623335303864383930316463633835663930363266343365 -38303666306338393836336134366139386165343764353938613339303566336365636232663330 -63366662343666396431653932373064653038666238373236613464613462343764383830653365 -64636330646162663939333762633538343833343563666563363330653435333665643035386133 -63356561316434613336353639356639383566343637656532663230353131376265363931376635 -64616565623362346264333564633335623933636435636433643163623166333765346235663136 -32653234653239386238653739366336643034336634363762346462363638646230653164313138 -36663836616366366461633762336133366235633165343363323861353934373530653363643233 -34663263303263373737643632663036666632306234313261333163376163656131356138306464 -63373836316637623062383461623266346164646364313062346133623031663738633133343564 -30363636383264636636336637396432396432623766623561396164656638353233303661396464 -39306337323932363839373337316531653166373837646233303235353631616663366333616532 -30306662633066316230383636643862633734623235306565343937376132323965346166386261 -61383163313539663063323161353331626638343063396433336437346437396265633439343532 -31386636303131626331353761323337643062393837373737613335643432643032323835643466 -31353166386131323034623738303037646433333466396162346163663933313462616336303265 -63643465643337396237386662393433633034613938623834613830373135633134633166633265 -63323330353830613832646433393439383739616465616635623464306432373864313133306338 -31666139313838396663656361363661643138366134336539356130333862626365343637363063 -36336462643866666262636466633730376562643630643937643130353930646231323138323366 -34663038623937336337366461393935366630346264306430626434643365656562386633393265 -34366465376263303036333934386362346234356335363835643361333366376361383962383464 -30623066653937303535316433393261373563363632393433653862356536336337333934316435 -39373230313235356437303637343266633063653639323936396661633535373532666432613766 -64336130616461656132353365343832646639653663623434323337326537663663306565643835 -30336262636239666339616437363061323435613238376661353435326436373063613463353561 -30396366636165386131616533326631323738373161313265383630363866663666663066343661 -31633331326634323463333839363662623430326566333764376130643238613761663332383039 -37653235393732313231336334373730626430363437646362363365646161336230313236316335 -36613835336632353266653234353839316438346565656637333833313136313663643764353535 -65316163363939656638663636376535663239626365366337346132376161353636366465383038 -35353930316166346163656234386338613037646261366462333231386137646165393465653864 -39666366623631326663386235653565343336613263333136393838303061643564633234643565 -30643631613737376665653736656663633233653661613164626163323535633836316462343661 -34643039653765383266326130326565396630306261313461353137333665656130343833303839 -34303937656439663661323861656334633566653834383438636133636130306366633732356430 -66393832316639343439336562356634383061616163646465626162666364656138653232666534 -63653533633032646632613836636333306331376639316264663332333362373434323864343561 -353937383136613266333838653236623462 +31343237643365393335643262646139363037616365623238316536313238386237353436643738 +3332376365393138633962373132633562363035396161650a323039646238373162333366353533 +37333238643437623538343630393065313463313038616538613838366638636433313637313630 +3733623838373864320a373636303030373737336265353165373463306233333033393266343838 +31323634313531343262323561636438326166343461373133393166303630303865316661356165 +64366435343933396131313665336437363234393239663831363033356433383330323964613833 +36346436383739623163343130376465333465316139303437666333386633313132336234383936 +65353437336637353739373837343766333266396263396562366463626332363061383435353132 +34616134396136333266323930343866663332373864623537623765303435366331613466376137 +63346337303461623036306362306235366365646137316165376634316230396239623132363337 +35363932663361313533663436633532313732646564663463393233316231623361336332336135 +39356338363336336231643661313062303734316539653031313630363866303464643438653035 +37393039623961386539303235636562653130343237336332643639346631326633363366373466 +62356536356664353466383131306664653132393837663635366466613665626535323930366637 +38646263326264313363386634363834626638383563346361386165323430383266646631626362 +64303263383138353739656534623734623638653438353666623464656461316636626564326536 +34616163626539383265353963333734313363343162663434356337393266313637323732346231 +33373835373465666637663330653337373130373732303632326530336132333236313466653239 +30383632363337333833666132363563363361623865616134613538373439353836346366353065 +34356134633038653839333430313738613531653634333430373635373239653362393461306330 +35326532303432316636666134353534626139316331333538356165313965613739653665616237 +3165353731626130666639346263333865316362623134373463 diff --git a/tests/playbook.yml b/tests/playbook.yml index 6a112a7..c91d143 100644 --- a/tests/playbook.yml +++ b/tests/playbook.yml @@ -78,8 +78,8 @@ include_role: name: docker_s3_volume_restore vars: - docker_s3_volume_restore_latest_s3_key: true - docker_volume_s3_restores: + docker_backup_restore_latest_s3_key: true + docker_backup_s3_restores: - volume_name: portainer_data - name: Deploy Portainer