krd/vault-redux
1
0
Fork 0
Code

ci: cache go modules (#11935)

Browse Source
This commit is contained in:
Sam Salisbury 2021-06-25 17:17:05 +01:00 committed by GitHub
parent 93b3e28681
commit 558a953c88
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 209 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

126
.circleci/config.yml generated
View File

@ -258,6 +258,7 @@ jobs:
- checkout
- restore_cache:
key: yarn-lock-v6-{{ checksum "ui/yarn.lock" }}
name: Restore yarn cache
- run:
command: |
cd ui
@ -266,6 +267,7 @@ jobs:
name: Install UI dependencies
- save_cache:
key: yarn-lock-v6-{{ checksum "ui/yarn.lock" }}
name: Save yarn cache
paths:
- ui/node_modules
build-common-layers:
@ -585,6 +587,7 @@ jobs:
- checkout
- restore_cache:
key: yarn-lock-v6-{{ checksum "ui/yarn.lock" }}
name: Restore yarn cache
- attach_workspace:
at: .
- run:
@ -701,6 +704,7 @@ jobs:
- checkout
- restore_cache:
key: yarn-lock-v6-{{ checksum "ui/yarn.lock" }}
name: Restore yarn cache
- attach_workspace:
at: .
- run:
@ -726,12 +730,18 @@ jobs:
mkdir $GOPATH 2>/dev/null || { sudo mkdir $GOPATH && sudo chmod 777 $GOPATH; }
echo "export GOPATH='$GOPATH'" >> "$BASH_ENV"
echo "export PATH='$PATH:$GOPATH/bin:/usr/local/go/bin'" >> "$BASH_ENV"
echo "export GOPROXY=off" >> "$BASH_ENV"
echo "export GOPRIVATE=github.com/hashicorp/*" >> "$BASH_ENV"
echo "$ go version"
go version
name: Setup Go
working_directory: ~/
- checkout
- restore_cache:
keys:
- v1.2-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}}
name: Restore exact go modules cache
- attach_workspace:
at: .
- run:
@ -874,6 +884,10 @@ jobs:
- restore_cache:
keys:
- go-test-cache-date-v1-{{ checksum "/tmp/go-cache-key" }}
- restore_cache:
keys:
- v1.2-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}}
name: Restore exact go modules cache
- run:
command: |
set -x
@ -956,8 +970,14 @@ jobs:
docker cp . testcontainer:/go/src/github.com/hashicorp/vault/
docker cp $DOCKER_CERT_PATH/ testcontainer:$DOCKER_CERT_PATH
# Copy the downloaded modules inside the container.
docker exec testcontainer sh -c 'mkdir -p /go/pkg'
docker cp "$(go env GOPATH)/pkg/mod" testcontainer:/go/pkg/mod
docker exec -w /go/src/github.com/hashicorp/vault/ \
-e CIRCLECI -e GOCACHE=/tmp/gocache -e VAULT_CI_GO_TEST_RACE \
-e CIRCLECI -e VAULT_CI_GO_TEST_RACE \
-e GOCACHE=/tmp/gocache \
-e GOPROXY="off" \
-e VAULT_LICENSE_CI \
testcontainer \
gotestsum --format=short-verbose \
@ -1103,6 +1123,10 @@ jobs:
- restore_cache:
keys:
- go-test-cache-date-v1-{{ checksum "/tmp/go-cache-key" }}
- restore_cache:
keys:
- v1.2-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}}
name: Restore exact go modules cache
- run:
command: |
set -x
@ -1184,8 +1208,14 @@ jobs:
docker cp . testcontainer:/go/src/github.com/hashicorp/vault/
docker cp $DOCKER_CERT_PATH/ testcontainer:$DOCKER_CERT_PATH
# Copy the downloaded modules inside the container.
docker exec testcontainer sh -c 'mkdir -p /go/pkg'
docker cp "$(go env GOPATH)/pkg/mod" testcontainer:/go/pkg/mod
docker exec -w /go/src/github.com/hashicorp/vault/ \
-e CIRCLECI -e GOCACHE=/tmp/gocache -e VAULT_CI_GO_TEST_RACE \
-e CIRCLECI -e VAULT_CI_GO_TEST_RACE \
-e GOCACHE=/tmp/gocache \
-e GOPROXY="off" \
-e VAULT_LICENSE_CI \
testcontainer \
gotestsum --format=short-verbose \
@ -1582,6 +1612,10 @@ jobs:
- restore_cache:
keys:
- go-test-cache-date-v1-{{ checksum "/tmp/go-cache-key" }}
- restore_cache:
keys:
- v1.2-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}}
name: Restore exact go modules cache
- run:
command: |
set -x
@ -1663,8 +1697,14 @@ jobs:
docker cp . testcontainer:/go/src/github.com/hashicorp/vault/
docker cp $DOCKER_CERT_PATH/ testcontainer:$DOCKER_CERT_PATH
# Copy the downloaded modules inside the container.
docker exec testcontainer sh -c 'mkdir -p /go/pkg'
docker cp "$(go env GOPATH)/pkg/mod" testcontainer:/go/pkg/mod
docker exec -w /go/src/github.com/hashicorp/vault/ \
-e CIRCLECI -e GOCACHE=/tmp/gocache -e VAULT_CI_GO_TEST_RACE \
-e CIRCLECI -e VAULT_CI_GO_TEST_RACE \
-e GOCACHE=/tmp/gocache \
-e GOPROXY="off" \
-e VAULT_LICENSE_CI \
testcontainer \
gotestsum --format=short-verbose \
@ -2091,19 +2131,36 @@ jobs:
paths:
- .buildcache/packages/store
pre-flight-checks:
docker:
- image: circleci/buildpack-deps
environment:
- CCI_VERSION: 0.1.5691
shell: /usr/bin/env bash -euo pipefail
machine: true
shell: /usr/bin/env bash -euo pipefail -c
working_directory: /go/src/github.com/hashicorp/vault
steps:
- run:
command: |
[ -n "$GO_VERSION" ] || { echo "You must set GO_VERSION"; exit 1; }
# Install Go
curl -sSLO "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz"
sudo rm -rf /usr/local/go
sudo tar -C /usr/local -xzf "go${GO_VERSION}.linux-amd64.tar.gz"
rm -f "go${GO_VERSION}.linux-amd64.tar.gz"
GOPATH="/go"
mkdir $GOPATH 2>/dev/null || { sudo mkdir $GOPATH && sudo chmod 777 $GOPATH; }
echo "export GOPATH='$GOPATH'" >> "$BASH_ENV"
echo "export PATH='$PATH:$GOPATH/bin:/usr/local/go/bin'" >> "$BASH_ENV"
echo "export GOPROXY=https://proxy.golang.org,direct" >> "$BASH_ENV"
echo "export GOPRIVATE=github.com/hashicorp/*" >> "$BASH_ENV"
echo "$ go version"
go version
name: Setup Go
working_directory: ~/
- checkout
- run:
command: |
export CCI_PATH=/tmp/circleci-cli/$CCI_VERSION
export CCI_PATH=/tmp/circleci-cli/$CIRCLECI_CLI_VERSION
mkdir -p $CCI_PATH
NAME=circleci-cli_${CCI_VERSION}_${ARCH}
URL=$BASE/v${CCI_VERSION}/${NAME}.tar.gz
NAME=circleci-cli_${CIRCLECI_CLI_VERSION}_${ARCH}
URL=$BASE/v${CIRCLECI_CLI_VERSION}/${NAME}.tar.gz
curl -sSL $URL \
| tar --overwrite --strip-components=1 -xz -C $CCI_PATH "${NAME}/circleci"
# Add circleci to the path for subsequent steps.
@ -2119,6 +2176,41 @@ jobs:
name: Install CircleCI CLI
- run:
command: make ci-verify
- add_ssh_keys:
fingerprints:
- 0e:03:77:f4:e2:c3:56:c2:53:6a:03:e1:31:91:2f:06
- run:
command: |
git config --global url."git@github.com:".insteadOf https://github.com/
- restore_cache:
keys:
- v1.2-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}}
- v1.2-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}
- v1.2-{{checksum "go.sum"}}
name: Restore closest matching go modules cache
- run:
command: |
go mod download -json
( cd sdk && go mod download -json; )
( cd api && go mod download -json; )
name: go mod download
- run:
command: |
git --no-pager diff --exit-code || {
echo "ERROR: Files modified by go mod download, see above."
exit 1
}
name: Verify downloading modules did not modify any files
- save_cache:
key: v1.2-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}}
name: Save go modules cache
paths:
- /go/pkg/mod
environment:
- CIRCLECI_CLI_VERSION: 0.1.5546
- GO_TAGS: ''
- GO_VERSION: 1.16.5
- GOTESTSUM_VERSION: 0.5.2
test-go-race-remote-docker:
docker:
- image: docker.mirror.hashicorp.services/circleci/golang:1.16.5-buster
@ -2148,6 +2240,10 @@ jobs:
- restore_cache:
keys:
- go-test-cache-date-v1-{{ checksum "/tmp/go-cache-key" }}
- restore_cache:
keys:
- v1.2-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}}
name: Restore exact go modules cache
- run:
command: |
set -x
@ -2230,8 +2326,14 @@ jobs:
docker cp . testcontainer:/go/src/github.com/hashicorp/vault/
docker cp $DOCKER_CERT_PATH/ testcontainer:$DOCKER_CERT_PATH
# Copy the downloaded modules inside the container.
docker exec testcontainer sh -c 'mkdir -p /go/pkg'
docker cp "$(go env GOPATH)/pkg/mod" testcontainer:/go/pkg/mod
docker exec -w /go/src/github.com/hashicorp/vault/ \
-e CIRCLECI -e GOCACHE=/tmp/gocache -e VAULT_CI_GO_TEST_RACE \
-e CIRCLECI -e VAULT_CI_GO_TEST_RACE \
-e GOCACHE=/tmp/gocache \
-e GOPROXY="off" \
-e VAULT_LICENSE_CI \
testcontainer \
gotestsum --format=short-verbose \

52
.circleci/config/commands/@caches.yml Normal file
View File

@ -0,0 +1,52 @@
restore_yarn_cache:
steps:
- restore_cache:
name: Restore yarn cache
key: &YARN_LOCK_CACHE_KEY yarn-lock-v6-{{ checksum "ui/yarn.lock" }}
save_yarn_cache:
steps:
- save_cache:
name: Save yarn cache
key: *YARN_LOCK_CACHE_KEY
paths:
- ui/node_modules
# allows restoring go mod caches by incomplete prefix. This is useful when re-generating
# cache, but not when running builds and tests that require an exact match.
restore_go_mod_cache_permissive:
steps:
- restore_cache:
name: Restore closest matching go modules cache
keys:
- &gocachekey v1.2-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}}
- v1.2-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}
- v1.2-{{checksum "go.sum"}}
restore_go_mod_cache:
steps:
- restore_cache:
name: Restore exact go modules cache
keys:
- *gocachekey
save_go_mod_cache:
steps:
- save_cache:
name: Save go modules cache
key: *gocachekey
paths:
- /go/pkg/mod
refresh_go_mod_cache:
steps:
- restore_go_mod_cache_permissive
- run:
name: go mod download
command: |
go mod download -json
( cd sdk && go mod download -json; )
( cd api && go mod download -json; )
- run:
name: Verify downloading modules did not modify any files
command: |
git --no-pager diff --exit-code || {
echo "ERROR: Files modified by go mod download, see above."
exit 1
}
- save_go_mod_cache

10
.circleci/config/commands/@yarn-cache.yml
View File

@ -1,10 +0,0 @@
restore_yarn_cache:
steps:
- restore_cache:
key: &YARN_LOCK_CACHE_KEY yarn-lock-v6-{{ checksum "ui/yarn.lock" }}
save_yarn_cache:
steps:
- save_cache:
key: *YARN_LOCK_CACHE_KEY
paths:
- ui/node_modules

6
.circleci/config/commands/configure-git.yml Normal file
View File

@ -0,0 +1,6 @@
steps:
- add_ssh_keys:
fingerprints:
- "0e:03:77:f4:e2:c3:56:c2:53:6a:03:e1:31:91:2f:06"
- run: |
git config --global url."git@github.com:".insteadOf https://github.com/

9
.circleci/config/commands/go_test.yml
View File

@ -30,6 +30,7 @@ steps:
- restore_cache:
keys:
- go-test-cache-date-v1-{{ checksum "/tmp/go-cache-key" }}
- restore_go_mod_cache
- run:
name: Run Go tests
no_output_timeout: 60m
@ -116,8 +117,14 @@ steps:
docker cp . testcontainer:/go/src/github.com/hashicorp/vault/
docker cp $DOCKER_CERT_PATH/ testcontainer:$DOCKER_CERT_PATH
# Copy the downloaded modules inside the container.
docker exec testcontainer sh -c 'mkdir -p /go/pkg'
docker cp "$(go env GOPATH)/pkg/mod" testcontainer:/go/pkg/mod
docker exec -w /go/src/github.com/hashicorp/vault/ \
-e CIRCLECI -e GOCACHE=/tmp/gocache -e VAULT_CI_GO_TEST_RACE \
-e CIRCLECI -e VAULT_CI_GO_TEST_RACE \
-e GOCACHE=/tmp/gocache \
-e GOPROXY="off" \
-e VAULT_LICENSE_CI \
testcontainer \
gotestsum --format=short-verbose \

12
.circleci/config/commands/setup-go.yml
View File

@ -1,6 +1,16 @@
---
description: >
Ensure the right version of Go is installed and set GOPATH to $HOME/go.
parameters:
GOPROXY:
description: >
Set GOPROXY. By default this is set to "off" meaning you have to have all modules pre-downloaded.
type: string
default: "off"
GOPRIVATE:
description: Set GOPRIVATE, defaults to github.com/hashicorp/*
type: string
default: github.com/hashicorp/*
steps:
- run:
working_directory: ~/
@ -16,6 +26,8 @@ steps:
mkdir $GOPATH 2>/dev/null || { sudo mkdir $GOPATH && sudo chmod 777 $GOPATH; }
echo "export GOPATH='$GOPATH'" >> "$BASH_ENV"
echo "export PATH='$PATH:$GOPATH/bin:/usr/local/go/bin'" >> "$BASH_ENV"
echo "export GOPROXY=<<parameters.GOPROXY>>" >> "$BASH_ENV"
echo "export GOPRIVATE=<<parameters.GOPRIVATE>>" >> "$BASH_ENV"
echo "$ go version"
go version

1
.circleci/config/jobs/build-go-dev.yml
View File

@ -2,6 +2,7 @@ executor: go-machine
steps:
- setup-go
- checkout
- restore_go_mod_cache
- attach_workspace:
at: .
- run:

19
.circleci/config/jobs/pre-flight-checks.yml
View File

@ -1,10 +1,9 @@
description: Ensures nothing obvious is broken for faster failures.
docker:
- image: circleci/buildpack-deps
shell: /usr/bin/env bash -euo pipefail
environment:
CCI_VERSION: 0.1.5691
description: Ensure nothing obvious is broken, and pre-cache Go modules.
executor: go-machine
steps:
# Setup Go enabling the proxy for downloading modules.
- setup-go:
GOPROXY: https://proxy.golang.org,direct
- checkout
- run:
name: Install CircleCI CLI
@ -12,10 +11,10 @@ steps:
ARCH: linux_amd64
BASE: https://github.com/CircleCI-Public/circleci-cli/releases/download
command: |
export CCI_PATH=/tmp/circleci-cli/$CCI_VERSION
export CCI_PATH=/tmp/circleci-cli/$CIRCLECI_CLI_VERSION
mkdir -p $CCI_PATH
NAME=circleci-cli_${CCI_VERSION}_${ARCH}
URL=$BASE/v${CCI_VERSION}/${NAME}.tar.gz
NAME=circleci-cli_${CIRCLECI_CLI_VERSION}_${ARCH}
URL=$BASE/v${CIRCLECI_CLI_VERSION}/${NAME}.tar.gz
curl -sSL $URL \
| tar --overwrite --strip-components=1 -xz -C $CCI_PATH "${NAME}/circleci"
# Add circleci to the path for subsequent steps.
@ -26,3 +25,5 @@ steps:
which circleci
circleci version
- run: make ci-verify
- configure-git
- refresh_go_mod_cache

1
.circleci/config/jobs/test-go-nightly.yml
View File

@ -2,6 +2,7 @@ executor: go-machine
steps:
- setup-go
- checkout
- restore_go_mod_cache
- go_test:
log_dir: "/tmp/testlogs"
save_cache: true

5
api/go.sum
View File

@ -228,6 +228,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@ -237,6 +238,7 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -246,6 +248,7 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@ -292,6 +295,8 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=