From e25891308df3c6ebf60014e516b8a6361357276e Mon Sep 17 00:00:00 2001 From: Floris van der Grinten Date: Thu, 20 May 2021 11:27:09 +0200 Subject: [PATCH 01/14] Create env-export action --- action.yml | 12 ++++++ entrypoint.sh | 108 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 action.yml create mode 100755 entrypoint.sh diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..247ca13 --- /dev/null +++ b/action.yml @@ -0,0 +1,12 @@ +name: env-export-action +inputs: + unset-previous: + description: Whether to unset environment variables populated by 1Password in earlier job steps + default: false +runs: + using: composite + steps: + - run: | + export INPUT_UNSET_PREVIOUS=${{ inputs.unset-previous }} + ${{ github.action_path }}/entrypoint.sh + shell: bash diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000..dbff759 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,108 @@ +#!/bin/bash +set -e + +managed_by_statement="Managed by 1Password" + +# Unset all secrets managed by 1Password if `unset-previous` is set. +if [ "$INPUT_UNSET_PREVIOUS" == "true" ]; then + echo "Unsetting previous values..." + + # Iterate over 'Managed by 1Password' comments in environment. + printenv | grep "$managed_by_statement" | while read -r comment; do + # Extract env var name and heredoc identifier from comment. + env_var=$(echo "$comment" | sed -e "s/.*$managed_by_statement: \(.*\)=.*/\1/") + + echo "Unsetting $env_var" + unset $env_var + + echo "$env_var=" >> $GITHUB_ENV + + # Keep the masks, just in case. + done +fi + +# Iterate over environment varables to find 1Password references, load the secret values, +# and make them available as environment variables in the next steps. +printenv | grep "=op://" | grep -v "^#" | while read -r possible_ref; do + env_var=$(echo "$possible_ref" | cut -d '=' -f1) + ref=$(printenv $env_var) + + if [[ ! $ref == "op://"* ]]; then + echo "Not really a reference: $ref" + continue + fi + + path=$(echo $ref | sed -e "s/^op:\/\///") + if [ $(echo "$path" | tr -cd '/' | wc -c) -lt 2 ]; then + echo "Expected path to be in format op:///[/
]/: $ref" + continue + fi + + echo "Populating variable: $env_var" + + vault="" + item="" + section="" + field="" + i=0 + IFS="/" + for component in $path; do + ((i+=1)) + case "$i" in + 1) vault=$component ;; + 2) item=$component ;; + 3) section=$component ;; + 4) field=$component ;; + esac + done + unset IFS + + # If field is not set, it may have wrongfully been interpreted as the section. + if [ -z "$field" ]; then + field="$section" + section="" + fi + + echo "Loading item $item from vault $vault..." + item_json=$(curl -sSf -H "Content-Type: application/json" -H "Authorization: Bearer $OP_CONNECT_TOKEN" "$OP_CONNECT_HOST/v1/vaults/$vault/items/$item") + jq_field_selector=".id == \"$field\" or .label == \"$field\"" + + # If the reference contains a section, edit the jq selector to take that into account. + if [ -n "$section" ]; then + echo "Looking for section: $section" + section_id=$(echo "$item_json" | jq -r ".sections[] | select(.id == \"$section\" or .label == \"$section\") | .id") + jq_field_selector=".section.id == \"$section_id\" and ($jq_field_selector)" + else + jq_field_selector=".section == null" + fi + + echo "Looking for field: $field" + secret_value=$(echo "$item_json" | jq -r "first(.fields[] | select($jq_field_selector) | .value)") + + # Register a mask for the secret to prevent accidental log exposure. + # To support multiline secrets, escape percent signs and add a mask per line. + escaped_mask_value=$(echo "$secret_value" | sed -e 's/%/%25/g') + IFS=$'\n' + for line in $escaped_mask_value; do + if [ "${#line}" -lt 3 ]; then + # To avoid false positives and unreadable logs, omit mask for lines that are too short. + continue + fi + echo "::add-mask::$line" + done + unset IFS + + # To support multiline secrets, we'll use the heredoc syntax to populate the environment variables. + # As the heredoc identifier, we'll use a randomly generated 64-character string, + # so that collisions are practically impossible. + random_heredoc_identifier=$(openssl rand -hex 16) + + { + # Add 'Managed by 1Password' comment, so in a later step it the secret can be unset again. + echo "# $managed_by_statement: $env_var=$ref" + # Populate env var, using heredoc syntax with generated identifier + echo "$env_var<<${random_heredoc_identifier}" + echo "$secret_value" + echo "${random_heredoc_identifier}" + } >> $GITHUB_ENV +done From 4c749feaf1a5eef8083280909ad7b2947af43c72 Mon Sep 17 00:00:00 2001 From: Floris van der Grinten Date: Wed, 19 May 2021 15:01:33 +0200 Subject: [PATCH 02/14] Add test workflow --- .github/workflows/test.yml | 77 +++++++++++++++++++++++++++++++ tests/fixtures/docker-compose.yml | 20 ++++++++ 2 files changed, 97 insertions(+) create mode 100644 .github/workflows/test.yml create mode 100644 tests/fixtures/docker-compose.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..a254ba8 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,77 @@ +on: push +name: Run acceptance tests + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Launch 1Password Connect instance + env: + OP_CONNECT_CREDENTIALS: ${{ secrets.OP_CONNECT_CREDENTIALS }} + run: | + echo "$OP_CONNECT_CREDENTIALS" > 1password-credentials.json + docker-compose -f tests/fixtures/docker-compose.yml up -d && sleep 10 + - name: Load secrets + uses: ./ + env: + OP_CONNECT_HOST: http://localhost:8080 + OP_CONNECT_TOKEN: ${{ secrets.OP_CONNECT_TOKEN }} + SECRET: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/password + MULTILINE_SECRET: op://v5pz6venw4roosmkzdq2nhpv6u/ghtz3jvcc6dqmzc53d3r3eskge/notesPlain + - name: Print environment variables with masked secrets + run: printenv + - name: Assert test secret values + env: + EXPECTED_SECRET: RGVhciBzZWN1cml0eSByZXNlYXJjaGVyLCB0aGlzIGlzIGp1c3QgYSBkdW1teSBzZWNyZXQuIFBsZWFzZSBkb24ndCByZXBvcnQgaXQu + EXPECTED_MULTILINE_SECRET: |- + -----BEGIN PRIVATE KEY----- + RGVhciBzZWN1cml0eSByZXNlYXJjaGVyLApXaGls + ZSB3ZSBkZWVwbHkgYXBwcmVjaWF0ZSB5b3VyIHZp + Z2lsYW5jZSBhbmQgZWZmb3J0cyB0byBtYWtlIHRo + ZSB3b3JsZCBtb3JlIHNlY3VyZSwgSSdtIGFmcmFp + ZCBJIG11c3QgdGVsbCB5b3UgdGhhdCB0aGlzIHZh + bHVlIGlzIG5vdCBhIGFjdHVhbCBwcml2YXRlIGtl + eS4gCkl0J3MgYSBqdXN0IGEgZHVtbXkgc2VjcmV0 + IHRoYXQgd2UgdXNlIHRvIHRlc3QgdmFyaW91cyAx + UGFzc3dvcmQgc2VjcmV0cyBpbnRlZ3JhdGlvbnMu + IApTbyBwbGVhc2UgZG9uJ3QgcmVwb3J0IGl0IQo= + -----END PRIVATE KEY----- + run: | + if [ "$SECRET" != "$EXPECTED_SECRET" ]; then + echo -e "Expected test SECRET to be set to:\n$EXPECTED_SECRET\nBut got:\n$SECRET" + exit 1 + fi + + if [ "$MULTILINE_SECRET" != "$EXPECTED_MULTILINE_SECRET" ]; then + echo -e "Expected MULTILINE_SECRET to be set to:\n$EXPECTED_MULTILINE_SECRET\nBut got:\n$MULTILINE_SECRET" + exit 1 + fi + - name: Remove secrets + uses: ./ + with: + unset-previous: true + - name: Print environment variables with secrets removed + run: printenv + - name: Assert removed secrets + run: | + if [ -n "$SECRET" ] || [ -n "$MULTILINE_SECRET" ]; then + echo "Expected secrets from 1Password to be unset" + exit 1 + fi + - name: Load secret again + uses: ./ + env: + OP_CONNECT_HOST: http://localhost:8080 + OP_CONNECT_TOKEN: ${{ secrets.OP_CONNECT_TOKEN }} + SECRET: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/password + - name: Print environment variables with masked secrets + run: printenv + - name: Assert test secret value + env: + EXPECTED_SECRET: RGVhciBzZWN1cml0eSByZXNlYXJjaGVyLCB0aGlzIGlzIGp1c3QgYSBkdW1teSBzZWNyZXQuIFBsZWFzZSBkb24ndCByZXBvcnQgaXQu + run: | + if [ "$SECRET" != "$EXPECTED_SECRET" ]; then + echo -e "Expected test SECRET to be set to:\n$EXPECTED_SECRET\nBut got:\n$SECRET" + exit 1 + fi diff --git a/tests/fixtures/docker-compose.yml b/tests/fixtures/docker-compose.yml new file mode 100644 index 0000000..cd2f518 --- /dev/null +++ b/tests/fixtures/docker-compose.yml @@ -0,0 +1,20 @@ +version: "3.4" + +services: + op-connect-api: + image: 1password/connect-api:latest + ports: + - "8080:8080" + volumes: + - "$PWD/1password-credentials.json:/home/opuser/.op/1password-credentials.json" + - "data:/home/opuser/.op/data" + op-connect-sync: + image: 1password/connect-sync:latest + ports: + - "8081:8080" + volumes: + - "$PWD/1password-credentials.json:/home/opuser/.op/1password-credentials.json" + - "data:/home/opuser/.op/data" + +volumes: + data: From a361a0c7847aa2c6807de7a770924c146aca34ce Mon Sep 17 00:00:00 2001 From: Floris van der Grinten Date: Wed, 19 May 2021 15:12:12 +0200 Subject: [PATCH 03/14] Move test assertions to separate file --- .github/workflows/test.yml | 43 +++++--------------------------------- tests/assert-env-set.sh | 25 ++++++++++++++++++++++ tests/assert-env-unset.sh | 10 +++++++++ 3 files changed, 40 insertions(+), 38 deletions(-) create mode 100755 tests/assert-env-set.sh create mode 100755 tests/assert-env-unset.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a254ba8..cf010c1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,31 +22,7 @@ jobs: - name: Print environment variables with masked secrets run: printenv - name: Assert test secret values - env: - EXPECTED_SECRET: RGVhciBzZWN1cml0eSByZXNlYXJjaGVyLCB0aGlzIGlzIGp1c3QgYSBkdW1teSBzZWNyZXQuIFBsZWFzZSBkb24ndCByZXBvcnQgaXQu - EXPECTED_MULTILINE_SECRET: |- - -----BEGIN PRIVATE KEY----- - RGVhciBzZWN1cml0eSByZXNlYXJjaGVyLApXaGls - ZSB3ZSBkZWVwbHkgYXBwcmVjaWF0ZSB5b3VyIHZp - Z2lsYW5jZSBhbmQgZWZmb3J0cyB0byBtYWtlIHRo - ZSB3b3JsZCBtb3JlIHNlY3VyZSwgSSdtIGFmcmFp - ZCBJIG11c3QgdGVsbCB5b3UgdGhhdCB0aGlzIHZh - bHVlIGlzIG5vdCBhIGFjdHVhbCBwcml2YXRlIGtl - eS4gCkl0J3MgYSBqdXN0IGEgZHVtbXkgc2VjcmV0 - IHRoYXQgd2UgdXNlIHRvIHRlc3QgdmFyaW91cyAx - UGFzc3dvcmQgc2VjcmV0cyBpbnRlZ3JhdGlvbnMu - IApTbyBwbGVhc2UgZG9uJ3QgcmVwb3J0IGl0IQo= - -----END PRIVATE KEY----- - run: | - if [ "$SECRET" != "$EXPECTED_SECRET" ]; then - echo -e "Expected test SECRET to be set to:\n$EXPECTED_SECRET\nBut got:\n$SECRET" - exit 1 - fi - - if [ "$MULTILINE_SECRET" != "$EXPECTED_MULTILINE_SECRET" ]; then - echo -e "Expected MULTILINE_SECRET to be set to:\n$EXPECTED_MULTILINE_SECRET\nBut got:\n$MULTILINE_SECRET" - exit 1 - fi + run: ./tests/assert-env-set.sh - name: Remove secrets uses: ./ with: @@ -54,24 +30,15 @@ jobs: - name: Print environment variables with secrets removed run: printenv - name: Assert removed secrets - run: | - if [ -n "$SECRET" ] || [ -n "$MULTILINE_SECRET" ]; then - echo "Expected secrets from 1Password to be unset" - exit 1 - fi + run: ./tests/assert-env-unset.sh - name: Load secret again uses: ./ env: OP_CONNECT_HOST: http://localhost:8080 OP_CONNECT_TOKEN: ${{ secrets.OP_CONNECT_TOKEN }} SECRET: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/password + MULTILINE_SECRET: op://v5pz6venw4roosmkzdq2nhpv6u/ghtz3jvcc6dqmzc53d3r3eskge/notesPlain - name: Print environment variables with masked secrets run: printenv - - name: Assert test secret value - env: - EXPECTED_SECRET: RGVhciBzZWN1cml0eSByZXNlYXJjaGVyLCB0aGlzIGlzIGp1c3QgYSBkdW1teSBzZWNyZXQuIFBsZWFzZSBkb24ndCByZXBvcnQgaXQu - run: | - if [ "$SECRET" != "$EXPECTED_SECRET" ]; then - echo -e "Expected test SECRET to be set to:\n$EXPECTED_SECRET\nBut got:\n$SECRET" - exit 1 - fi + - name: Assert test secret values again + run: ./tests/assert-env-set.sh diff --git a/tests/assert-env-set.sh b/tests/assert-env-set.sh new file mode 100755 index 0000000..9723f5c --- /dev/null +++ b/tests/assert-env-set.sh @@ -0,0 +1,25 @@ +#!/bin/bash +assert_env_equals() { + if [ "$(printenv $1)" != "$2" ]; then + echo -e "Expected $1 to be set to:\n$2\nBut got:\n$(printenv $1)" + exit 1 + fi +} + +assert_env_equals "SECRET" "RGVhciBzZWN1cml0eSByZXNlYXJjaGVyLCB0aGlzIGlzIGp1c3QgYSBkdW1teSBzZWNyZXQuIFBsZWFzZSBkb24ndCByZXBvcnQgaXQu" + +assert_env_equals "MULTILINE_SECRET" "$(cat << EOF +-----BEGIN PRIVATE KEY----- +RGVhciBzZWN1cml0eSByZXNlYXJjaGVyLApXaGls +ZSB3ZSBkZWVwbHkgYXBwcmVjaWF0ZSB5b3VyIHZp +Z2lsYW5jZSBhbmQgZWZmb3J0cyB0byBtYWtlIHRo +ZSB3b3JsZCBtb3JlIHNlY3VyZSwgSSdtIGFmcmFp +ZCBJIG11c3QgdGVsbCB5b3UgdGhhdCB0aGlzIHZh +bHVlIGlzIG5vdCBhIGFjdHVhbCBwcml2YXRlIGtl +eS4gCkl0J3MgYSBqdXN0IGEgZHVtbXkgc2VjcmV0 +IHRoYXQgd2UgdXNlIHRvIHRlc3QgdmFyaW91cyAx +UGFzc3dvcmQgc2VjcmV0cyBpbnRlZ3JhdGlvbnMu +IApTbyBwbGVhc2UgZG9uJ3QgcmVwb3J0IGl0IQo= +-----END PRIVATE KEY----- +EOF +)" diff --git a/tests/assert-env-unset.sh b/tests/assert-env-unset.sh new file mode 100755 index 0000000..3237c8a --- /dev/null +++ b/tests/assert-env-unset.sh @@ -0,0 +1,10 @@ +#!/bin/bash +assert_env_unset() { + if [ -n "$(printenv $1)" ]; then + echo "Expected secret $1 to be unset" + exit 1 + fi +} + +assert_env_unset "SECRET" +assert_env_unset "MULTILINE_SECRET" From 5b8ac70e926ad33b53ca52350b0a48b7d3202364 Mon Sep 17 00:00:00 2001 From: Floris van der Grinten Date: Wed, 19 May 2021 15:06:32 +0200 Subject: [PATCH 04/14] Test secret in section --- .github/workflows/test.yml | 2 ++ tests/assert-env-set.sh | 2 ++ tests/assert-env-unset.sh | 1 + 3 files changed, 5 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cf010c1..9447f59 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,6 +18,7 @@ jobs: OP_CONNECT_HOST: http://localhost:8080 OP_CONNECT_TOKEN: ${{ secrets.OP_CONNECT_TOKEN }} SECRET: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/password + SECRET_IN_SECTION: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/test-section/password MULTILINE_SECRET: op://v5pz6venw4roosmkzdq2nhpv6u/ghtz3jvcc6dqmzc53d3r3eskge/notesPlain - name: Print environment variables with masked secrets run: printenv @@ -37,6 +38,7 @@ jobs: OP_CONNECT_HOST: http://localhost:8080 OP_CONNECT_TOKEN: ${{ secrets.OP_CONNECT_TOKEN }} SECRET: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/password + SECRET_IN_SECTION: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/test-section/password MULTILINE_SECRET: op://v5pz6venw4roosmkzdq2nhpv6u/ghtz3jvcc6dqmzc53d3r3eskge/notesPlain - name: Print environment variables with masked secrets run: printenv diff --git a/tests/assert-env-set.sh b/tests/assert-env-set.sh index 9723f5c..ece0f8d 100755 --- a/tests/assert-env-set.sh +++ b/tests/assert-env-set.sh @@ -8,6 +8,8 @@ assert_env_equals() { assert_env_equals "SECRET" "RGVhciBzZWN1cml0eSByZXNlYXJjaGVyLCB0aGlzIGlzIGp1c3QgYSBkdW1teSBzZWNyZXQuIFBsZWFzZSBkb24ndCByZXBvcnQgaXQu" +assert_env_equals "SECRET_IN_SECTION" "RGVhciBzZWN1cml0eSByZXNlYXJjaGVyLCB0aGlzIGlzIGp1c3QgYSBkdW1teSBzZWNyZXQuIFBsZWFzZSBkb24ndCByZXBvcnQgaXQu" + assert_env_equals "MULTILINE_SECRET" "$(cat << EOF -----BEGIN PRIVATE KEY----- RGVhciBzZWN1cml0eSByZXNlYXJjaGVyLApXaGls diff --git a/tests/assert-env-unset.sh b/tests/assert-env-unset.sh index 3237c8a..ce630f7 100755 --- a/tests/assert-env-unset.sh +++ b/tests/assert-env-unset.sh @@ -7,4 +7,5 @@ assert_env_unset() { } assert_env_unset "SECRET" +assert_env_unset "SECRET_IN_SECTION" assert_env_unset "MULTILINE_SECRET" From 7bbc7abc2f202f25edf56b98c96366b4b9dbf85d Mon Sep 17 00:00:00 2001 From: Floris van der Grinten Date: Wed, 19 May 2021 15:52:00 +0200 Subject: [PATCH 05/14] Add subaction to persist Connect details --- .github/workflows/test.yml | 9 +++++---- configure/action.yml | 14 ++++++++++++++ configure/entrypoint.sh | 15 +++++++++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 configure/action.yml create mode 100755 configure/entrypoint.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9447f59..66e762b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,11 +12,14 @@ jobs: run: | echo "$OP_CONNECT_CREDENTIALS" > 1password-credentials.json docker-compose -f tests/fixtures/docker-compose.yml up -d && sleep 10 + - name: Configure 1Password Connect + uses: ./configure + with: + connect-host: http://localhost:8080 + connect-token: ${{ secrets.OP_CONNECT_TOKEN }} - name: Load secrets uses: ./ env: - OP_CONNECT_HOST: http://localhost:8080 - OP_CONNECT_TOKEN: ${{ secrets.OP_CONNECT_TOKEN }} SECRET: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/password SECRET_IN_SECTION: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/test-section/password MULTILINE_SECRET: op://v5pz6venw4roosmkzdq2nhpv6u/ghtz3jvcc6dqmzc53d3r3eskge/notesPlain @@ -35,8 +38,6 @@ jobs: - name: Load secret again uses: ./ env: - OP_CONNECT_HOST: http://localhost:8080 - OP_CONNECT_TOKEN: ${{ secrets.OP_CONNECT_TOKEN }} SECRET: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/password SECRET_IN_SECTION: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/test-section/password MULTILINE_SECRET: op://v5pz6venw4roosmkzdq2nhpv6u/ghtz3jvcc6dqmzc53d3r3eskge/notesPlain diff --git a/configure/action.yml b/configure/action.yml new file mode 100644 index 0000000..e9a98e7 --- /dev/null +++ b/configure/action.yml @@ -0,0 +1,14 @@ +name: configure-action +inputs: + connect-host: + description: Your 1Password Connect instance URL + connect-token: + description: Token to authenticate to your 1Password Connect instance +runs: + using: composite + steps: + - run: | + export INPUT_CONNECT_HOST=${{ inputs.connect-host }} + export INPUT_CONNECT_TOKEN=${{ inputs.connect-token }} + ${{ github.action_path }}/entrypoint.sh + shell: bash diff --git a/configure/entrypoint.sh b/configure/entrypoint.sh new file mode 100755 index 0000000..97d0853 --- /dev/null +++ b/configure/entrypoint.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -e + +# Capture Connect configuration in $GITHUB_ENV, giving (optional) inputs +# precendence over OP_CONNECT_* environment variables. + +OP_CONNECT_HOST=${INPUT_CONNECT_HOST:-$OP_CONNECT_HOST} +if [ -n "$OP_CONNECT_HOST" ]; then + echo "OP_CONNECT_HOST=$OP_CONNECT_HOST" >> $GITHUB_ENV +fi + +OP_CONNECT_TOKEN=${INPUT_CONNECT_TOKEN:-$OP_CONNECT_TOKEN} +if [ -n "$OP_CONNECT_TOKEN" ]; then + echo "OP_CONNECT_TOKEN=$OP_CONNECT_TOKEN" >> $GITHUB_ENV +fi From 4d815b6e11201a7747e5b378117709d432e9545d Mon Sep 17 00:00:00 2001 From: Floris van der Grinten Date: Wed, 19 May 2021 21:32:04 +0200 Subject: [PATCH 06/14] Add comment in test workflow to demonstrate real-world workflow --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 66e762b..4844c77 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,12 +13,12 @@ jobs: echo "$OP_CONNECT_CREDENTIALS" > 1password-credentials.json docker-compose -f tests/fixtures/docker-compose.yml up -d && sleep 10 - name: Configure 1Password Connect - uses: ./configure + uses: ./configure # 1password/env-export-action/configure@ with: connect-host: http://localhost:8080 connect-token: ${{ secrets.OP_CONNECT_TOKEN }} - name: Load secrets - uses: ./ + uses: ./ # 1password/env-export-action@ env: SECRET: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/password SECRET_IN_SECTION: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/test-section/password @@ -28,7 +28,7 @@ jobs: - name: Assert test secret values run: ./tests/assert-env-set.sh - name: Remove secrets - uses: ./ + uses: ./ # 1password/env-export-action@ with: unset-previous: true - name: Print environment variables with secrets removed @@ -36,7 +36,7 @@ jobs: - name: Assert removed secrets run: ./tests/assert-env-unset.sh - name: Load secret again - uses: ./ + uses: ./ # 1password/env-export-action@ env: SECRET: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/password SECRET_IN_SECTION: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/test-section/password From 8e88ca96a523dc0a35b6eb0adcac36177fef229a Mon Sep 17 00:00:00 2001 From: Floris van der Grinten Date: Thu, 20 May 2021 11:38:59 +0200 Subject: [PATCH 07/14] Ensure Connect has been configured --- entrypoint.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/entrypoint.sh b/entrypoint.sh index dbff759..51089b0 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -3,6 +3,11 @@ set -e managed_by_statement="Managed by 1Password" +if [ -z "$OP_CONNECT_TOKEN" ] || [ -z "$OP_CONNECT_HOST" ]; then + echo "\$OP_CONNECT_TOKEN and \$OP_CONNECT_HOST must be set" + exit 1 +fi + # Unset all secrets managed by 1Password if `unset-previous` is set. if [ "$INPUT_UNSET_PREVIOUS" == "true" ]; then echo "Unsetting previous values..." From 2d364e111f280f78fd6a5b9b32b313a14ba475a7 Mon Sep 17 00:00:00 2001 From: Floris van der Grinten Date: Thu, 20 May 2021 17:12:12 +0200 Subject: [PATCH 08/14] Add ShellCheck workflow --- .github/workflows/lint.yml | 10 ++++++++++ configure/entrypoint.sh | 5 +++-- entrypoint.sh | 1 + tests/assert-env-set.sh | 3 +++ tests/assert-env-unset.sh | 3 +++ 5 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..cc3c745 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,10 @@ +on: pull_request +name: Lint + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: ShellCheck + uses: ludeeus/action-shellcheck@1.1.0 diff --git a/configure/entrypoint.sh b/configure/entrypoint.sh index 97d0853..a0a4494 100755 --- a/configure/entrypoint.sh +++ b/configure/entrypoint.sh @@ -1,15 +1,16 @@ #!/bin/bash +# shellcheck disable=SC2086 set -e # Capture Connect configuration in $GITHUB_ENV, giving (optional) inputs # precendence over OP_CONNECT_* environment variables. -OP_CONNECT_HOST=${INPUT_CONNECT_HOST:-$OP_CONNECT_HOST} +OP_CONNECT_HOST="${INPUT_CONNECT_HOST:-$OP_CONNECT_HOST}" if [ -n "$OP_CONNECT_HOST" ]; then echo "OP_CONNECT_HOST=$OP_CONNECT_HOST" >> $GITHUB_ENV fi -OP_CONNECT_TOKEN=${INPUT_CONNECT_TOKEN:-$OP_CONNECT_TOKEN} +OP_CONNECT_TOKEN="${INPUT_CONNECT_TOKEN:-$OP_CONNECT_TOKEN}" if [ -n "$OP_CONNECT_TOKEN" ]; then echo "OP_CONNECT_TOKEN=$OP_CONNECT_TOKEN" >> $GITHUB_ENV fi diff --git a/entrypoint.sh b/entrypoint.sh index 51089b0..66b4d81 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,4 +1,5 @@ #!/bin/bash +# shellcheck disable=SC2046,SC2001,SC2086 set -e managed_by_statement="Managed by 1Password" diff --git a/tests/assert-env-set.sh b/tests/assert-env-set.sh index ece0f8d..1871707 100755 --- a/tests/assert-env-set.sh +++ b/tests/assert-env-set.sh @@ -1,4 +1,7 @@ #!/bin/bash +# shellcheck disable=SC2086 +set -e + assert_env_equals() { if [ "$(printenv $1)" != "$2" ]; then echo -e "Expected $1 to be set to:\n$2\nBut got:\n$(printenv $1)" diff --git a/tests/assert-env-unset.sh b/tests/assert-env-unset.sh index ce630f7..e4c6448 100755 --- a/tests/assert-env-unset.sh +++ b/tests/assert-env-unset.sh @@ -1,4 +1,7 @@ #!/bin/bash +# shellcheck disable=SC2086 +set -e + assert_env_unset() { if [ -n "$(printenv $1)" ]; then echo "Expected secret $1 to be unset" From 6ee4db8bcdae9817d44893425e2f670046f461df Mon Sep 17 00:00:00 2001 From: Floris van der Grinten Date: Thu, 20 May 2021 20:49:22 +0200 Subject: [PATCH 09/14] Rename to load-secrets-action --- .github/workflows/test.yml | 8 ++++---- action.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4844c77..a5f7e71 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,12 +13,12 @@ jobs: echo "$OP_CONNECT_CREDENTIALS" > 1password-credentials.json docker-compose -f tests/fixtures/docker-compose.yml up -d && sleep 10 - name: Configure 1Password Connect - uses: ./configure # 1password/env-export-action/configure@ + uses: ./configure # 1password/load-secrets-action/configure@ with: connect-host: http://localhost:8080 connect-token: ${{ secrets.OP_CONNECT_TOKEN }} - name: Load secrets - uses: ./ # 1password/env-export-action@ + uses: ./ # 1password/load-secrets-action@ env: SECRET: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/password SECRET_IN_SECTION: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/test-section/password @@ -28,7 +28,7 @@ jobs: - name: Assert test secret values run: ./tests/assert-env-set.sh - name: Remove secrets - uses: ./ # 1password/env-export-action@ + uses: ./ # 1password/load-secrets-action@ with: unset-previous: true - name: Print environment variables with secrets removed @@ -36,7 +36,7 @@ jobs: - name: Assert removed secrets run: ./tests/assert-env-unset.sh - name: Load secret again - uses: ./ # 1password/env-export-action@ + uses: ./ # 1password/load-secrets-action@ env: SECRET: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/password SECRET_IN_SECTION: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/test-section/password diff --git a/action.yml b/action.yml index 247ca13..6b59a46 100644 --- a/action.yml +++ b/action.yml @@ -1,4 +1,4 @@ -name: env-export-action +name: Load secrets from 1Password inputs: unset-previous: description: Whether to unset environment variables populated by 1Password in earlier job steps From bafb7c3e166db180aabd34dc655a21d23271c6e2 Mon Sep 17 00:00:00 2001 From: Floris van der Grinten Date: Thu, 20 May 2021 20:57:20 +0200 Subject: [PATCH 10/14] Fill out action metadata fields --- action.yml | 5 +++++ configure/action.yml | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 6b59a46..0177386 100644 --- a/action.yml +++ b/action.yml @@ -1,4 +1,9 @@ name: Load secrets from 1Password +description: Make secrets from 1Password Connect available as environment variables in the next steps. +author: 1Password +branding: + icon: lock + color: blue inputs: unset-previous: description: Whether to unset environment variables populated by 1Password in earlier job steps diff --git a/configure/action.yml b/configure/action.yml index e9a98e7..97a44b4 100644 --- a/configure/action.yml +++ b/configure/action.yml @@ -1,4 +1,6 @@ -name: configure-action +name: Configure 1Password Connect +description: Persist 1Password Connect host and token for use in next steps. +author: 1Password inputs: connect-host: description: Your 1Password Connect instance URL From 860c5ff00e429b35c5d7adc1c93aee197b44f9f1 Mon Sep 17 00:00:00 2001 From: Floris van der Grinten Date: Wed, 26 May 2021 13:13:03 +0200 Subject: [PATCH 11/14] Remove use of undocumented API in unset mechanism --- .github/workflows/test.yml | 3 +++ entrypoint.sh | 27 +++++++++++++++++---------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a5f7e71..27f8794 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,6 +22,9 @@ jobs: env: SECRET: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/password SECRET_IN_SECTION: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/test-section/password + - name: Load multiline secret + uses: ./ # 1password/load-secrets-action@ + env: MULTILINE_SECRET: op://v5pz6venw4roosmkzdq2nhpv6u/ghtz3jvcc6dqmzc53d3r3eskge/notesPlain - name: Print environment variables with masked secrets run: printenv diff --git a/entrypoint.sh b/entrypoint.sh index 66b4d81..ca4c538 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,22 +2,20 @@ # shellcheck disable=SC2046,SC2001,SC2086 set -e -managed_by_statement="Managed by 1Password" - if [ -z "$OP_CONNECT_TOKEN" ] || [ -z "$OP_CONNECT_HOST" ]; then echo "\$OP_CONNECT_TOKEN and \$OP_CONNECT_HOST must be set" exit 1 fi +managed_variables_var="OP_MANAGED_VARIABLES" +IFS=',' read -r -a managed_variables <<< "$(printenv $managed_variables_var)" + # Unset all secrets managed by 1Password if `unset-previous` is set. if [ "$INPUT_UNSET_PREVIOUS" == "true" ]; then echo "Unsetting previous values..." - # Iterate over 'Managed by 1Password' comments in environment. - printenv | grep "$managed_by_statement" | while read -r comment; do - # Extract env var name and heredoc identifier from comment. - env_var=$(echo "$comment" | sed -e "s/.*$managed_by_statement: \(.*\)=.*/\1/") - + # Find environment variables that are managed by 1Password. + for env_var in "${managed_variables[@]}"; do echo "Unsetting $env_var" unset $env_var @@ -25,11 +23,14 @@ if [ "$INPUT_UNSET_PREVIOUS" == "true" ]; then # Keep the masks, just in case. done + + managed_variables=() fi # Iterate over environment varables to find 1Password references, load the secret values, # and make them available as environment variables in the next steps. -printenv | grep "=op://" | grep -v "^#" | while read -r possible_ref; do +IFS=$'\n' +for possible_ref in $(printenv | grep "=op://" | grep -v "^#"); do env_var=$(echo "$possible_ref" | cut -d '=' -f1) ref=$(printenv $env_var) @@ -104,11 +105,17 @@ printenv | grep "=op://" | grep -v "^#" | while read -r possible_ref; do random_heredoc_identifier=$(openssl rand -hex 16) { - # Add 'Managed by 1Password' comment, so in a later step it the secret can be unset again. - echo "# $managed_by_statement: $env_var=$ref" # Populate env var, using heredoc syntax with generated identifier echo "$env_var<<${random_heredoc_identifier}" echo "$secret_value" echo "${random_heredoc_identifier}" } >> $GITHUB_ENV + + managed_variables+=("$env_var") done +unset IFS + +# Add extra env var that lists which secrets are managed by 1Password so that in a later step +# these can be unset again. +managed_variables_str=$(IFS=','; echo "${managed_variables[*]}") +echo "$managed_variables_var=$managed_variables_str" >> $GITHUB_ENV From c3679bd18a607313210cacc342f22ed8ed39e6b5 Mon Sep 17 00:00:00 2001 From: Floris van der Grinten Date: Thu, 27 May 2021 14:16:43 +0200 Subject: [PATCH 12/14] Only mask concealed or note fields --- .github/workflows/test.yml | 1 + entrypoint.sh | 33 ++++++++++++++++++++------------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 27f8794..374245e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,6 +22,7 @@ jobs: env: SECRET: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/password SECRET_IN_SECTION: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/test-section/password + UNMASKED_VALUE: op://v5pz6venw4roosmkzdq2nhpv6u/hrgkzhrlvscomepxlgafb2m3ca/test-section/username - name: Load multiline secret uses: ./ # 1password/load-secrets-action@ env: diff --git a/entrypoint.sh b/entrypoint.sh index ca4c538..5dcdeaf 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -84,20 +84,27 @@ for possible_ref in $(printenv | grep "=op://" | grep -v "^#"); do fi echo "Looking for field: $field" - secret_value=$(echo "$item_json" | jq -r "first(.fields[] | select($jq_field_selector) | .value)") + secret_field_json=$(echo "$item_json" | jq -r "first(.fields[] | select($jq_field_selector))") - # Register a mask for the secret to prevent accidental log exposure. - # To support multiline secrets, escape percent signs and add a mask per line. - escaped_mask_value=$(echo "$secret_value" | sed -e 's/%/%25/g') - IFS=$'\n' - for line in $escaped_mask_value; do - if [ "${#line}" -lt 3 ]; then - # To avoid false positives and unreadable logs, omit mask for lines that are too short. - continue - fi - echo "::add-mask::$line" - done - unset IFS + field_type=$(echo "$secret_field_json" | jq -r '.type') + field_purpose=$(echo "$secret_field_json" | jq -r '.purpose') + secret_value=$(echo "$secret_field_json" | jq -r '.value') + + # If the field is marked as concealed or is a note, register a mask + # for the secret to prevent accidental log exposure. + if [ "$field_type" == "CONCEALED" ] || [ "$field_purpose" == "NOTES" ]; then + # To support multiline secrets, escape percent signs and add a mask per line. + escaped_mask_value=$(echo "$secret_value" | sed -e 's/%/%25/g') + IFS=$'\n' + for line in $escaped_mask_value; do + if [ "${#line}" -lt 3 ]; then + # To avoid false positives and unreadable logs, omit mask for lines that are too short. + continue + fi + echo "::add-mask::$line" + done + unset IFS + fi # To support multiline secrets, we'll use the heredoc syntax to populate the environment variables. # As the heredoc identifier, we'll use a randomly generated 64-character string, From 9adb346f21944024eb46b699fc957e9696d7e6d7 Mon Sep 17 00:00:00 2001 From: Floris van der Grinten Date: Thu, 27 May 2021 14:12:46 +0200 Subject: [PATCH 13/14] Exit when secret is not found --- entrypoint.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/entrypoint.sh b/entrypoint.sh index 5dcdeaf..c581bbf 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -90,6 +90,11 @@ for possible_ref in $(printenv | grep "=op://" | grep -v "^#"); do field_purpose=$(echo "$secret_field_json" | jq -r '.purpose') secret_value=$(echo "$secret_field_json" | jq -r '.value') + if [ -z "$secret_value" ]; then + echo "Could not find or access secret $ref" + exit 1 + fi + # If the field is marked as concealed or is a note, register a mask # for the secret to prevent accidental log exposure. if [ "$field_type" == "CONCEALED" ] || [ "$field_purpose" == "NOTES" ]; then From c5c3979b32984e1bb2d559ba4eebd60c5f7df9ea Mon Sep 17 00:00:00 2001 From: Floris van der Grinten Date: Thu, 27 May 2021 14:07:20 +0200 Subject: [PATCH 14/14] Fix jq secret selector --- entrypoint.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index c581bbf..dde2fbe 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -73,18 +73,19 @@ for possible_ref in $(printenv | grep "=op://" | grep -v "^#"); do echo "Loading item $item from vault $vault..." item_json=$(curl -sSf -H "Content-Type: application/json" -H "Authorization: Bearer $OP_CONNECT_TOKEN" "$OP_CONNECT_HOST/v1/vaults/$vault/items/$item") jq_field_selector=".id == \"$field\" or .label == \"$field\"" + jq_section_selector=".section == null" # If the reference contains a section, edit the jq selector to take that into account. if [ -n "$section" ]; then echo "Looking for section: $section" section_id=$(echo "$item_json" | jq -r ".sections[] | select(.id == \"$section\" or .label == \"$section\") | .id") - jq_field_selector=".section.id == \"$section_id\" and ($jq_field_selector)" - else - jq_field_selector=".section == null" + jq_section_selector=".section.id == \"$section_id\"" fi + jq_secret_selector="$jq_section_selector and ($jq_field_selector)" + echo "Looking for field: $field" - secret_field_json=$(echo "$item_json" | jq -r "first(.fields[] | select($jq_field_selector))") + secret_field_json=$(echo "$item_json" | jq -r "first(.fields[] | select($jq_secret_selector))") field_type=$(echo "$secret_field_json" | jq -r '.type') field_purpose=$(echo "$secret_field_json" | jq -r '.purpose')