Add an enos scenario to test vault docker images using k8s/kind/helm (#17515)

* Added a scenario to test docker artifacts using the vault helm chart and a kind cluster
* Addedt enos-k8s github workflow

.github/workflows/build.yml

@@ -254,6 +254,26 @@ jobs:
             docker.io/hashicorp/${{env.repo}}:${{env.version}}
             public.ecr.aws/hashicorp/${{env.repo}}:${{env.version}}
 
+  enos-test-docker:
+    name: Enos Docker
+    # Only run the Enos workflow against branches that are created from the
+    # hashicorp/vault repository. This has the effect of limiting execution of
+    # Enos scenarios to branches that originate from authors that have write
+    # access to hashicorp/vault repository. This is required as Github Actions
+    # will not populate the required secrets for branches created by outside
+    # contributors in order to protect the secrets integrity.
+    if: "! github.event.pull_request.head.repo.fork"
+    needs:
+      - product-metadata
+      - build-docker
+    uses: ./.github/workflows/enos-run-k8s.yml
+    with:
+      artifact-build-date: "${{needs.product-metadata.outputs.build-date}}"
+      artifact-name: "${{github.event.repository.name}}_default_linux_amd64_${{needs.product-metadata.outputs.product-version}}_${{needs.product-metadata.outputs.product-revision}}.docker.tar"
+      artifact-revision: "${{needs.product-metadata.outputs.product-revision}}"
+      artifact-version: "${{needs.product-metadata.outputs.product-version}}"
+    secrets: inherit
+
   build-ubi:
     name: Red Hat UBI ${{ matrix.arch }} build
     needs:

.github/workflows/enos-run-k8s.yml

@@ -0,0 +1,109 @@
+---
+name: enos-k8s
+
+on:
+  workflow_call:
+    inputs:
+      artifact-build-date:
+        required: false
+        type: string
+      artifact-name:
+        required: true
+        type: string
+      artifact-revision:
+        required: true
+        type: string
+      artifact-version:
+        required: true
+        type: string
+
+env:
+  ARTIFACT_BUILD_DATE: ${{ inputs.artifact-build-date }}
+  ARTIFACT_NAME: ${{ inputs.artifact-name }}
+  ARTIFACT_REVISION: ${{ inputs.artifact-revision }}
+  ARTIFACT_VERSION: ${{ inputs.artifact-version }}
+
+jobs:
+  enos:
+    name: Integration
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up Terraform
+        uses: hashicorp/setup-terraform@v2
+        with:
+          # the Terraform wrapper will break Terraform execution in Enos because
+          # it changes the output to text when we expect it to be JSON.
+          terraform_wrapper: false
+      - name: Set up Enos
+        uses: hashicorp/action-setup-enos@v1
+        with:
+          github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
+      - name: Download Docker Image
+        id: download
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ inputs.artifact-name }}
+          path: ./enos/support/downloads
+      - name: Prepare for scenario execution
+        env:
+          IS_ENT: ${{ startsWith(env.ARTIFACT_NAME, 'vault-enterprise' ) }}
+        run: |
+          mkdir -p ./enos/support/terraform-plugin-cache
+          if ${IS_ENT} == true; then
+            echo "${{ secrets.VAULT_LICENSE }}" > ./enos/support/vault.hclic || true
+            echo "edition=ent" >> $GITHUB_ENV
+            echo "edition set to ent"
+          else
+            echo "edition=oss" >> $GITHUB_ENV
+            echo "edition set to oss"
+          fi
+      - name: Run Enos scenario
+        id: run
+        # Continue once and retry to handle occasional blips when creating
+        # infrastructure.
+        continue-on-error: true
+        env:
+          ENOS_VAR_tfc_api_token: ${{ secrets.TF_API_TOKEN }}
+          ENOS_VAR_terraform_plugin_cache_dir: ../support/terraform-plugin-cache
+          ENOS_VAR_vault_build_date: ${{ env.ARTIFACT_BUILD_DATE }}
+          ENOS_VAR_vault_product_version: ${{ env.ARTIFACT_VERSION }}
+          ENOS_VAR_vault_product_revision: ${{ env.ARTIFACT_REVISION }}
+          ENOS_VAR_vault_docker_image_archive: ${{steps.download.outputs.download-path}}/${{ env.ARTIFACT_NAME }}
+          ENOS_VAR_vault_image_repository: hashicorp/vault
+        run: |
+          enos scenario run --timeout 10m0s --chdir ./enos/k8s edition:${{ env.edition }}
+      - name: Retry Enos scenario
+        id: run_retry
+        if: steps.run.outcome == 'failure'
+        env:
+          ENOS_VAR_tfc_api_token: ${{ secrets.TF_API_TOKEN }}
+          ENOS_VAR_terraform_plugin_cache_dir: ../support/terraform-plugin-cache
+          ENOS_VAR_vault_build_date: ${{ env.ARTIFACT_BUILD_DATE }}
+          ENOS_VAR_vault_product_version: ${{ env.ARTIFACT_VERSION }}
+          ENOS_VAR_vault_product_revision: ${{ env.ARTIFACT_REVISION }}
+          ENOS_VAR_vault_docker_image_archive: ${{steps.download.outputs.download-path}}/${{ env.ARTIFACT_NAME }}
+          ENOS_VAR_vault_image_repository: hashicorp/vault
+        run: |
+          enos scenario run --timeout 10m0s --chdir ./enos/k8s edition:${{ env.edition }}
+      - name: Destroy Enos scenario
+        if: ${{ always() }}
+        env:
+          ENOS_VAR_tfc_api_token: ${{ secrets.TF_API_TOKEN }}
+          ENOS_VAR_terraform_plugin_cache_dir: ./support/terraform-plugin-cache
+          ENOS_VAR_vault_build_date: ${{ env.ARTIFACT_BUILD_DATE }}
+          ENOS_VAR_vault_product_version: ${{ env.ARTIFACT_VERSION }}
+          ENOS_VAR_vault_product_revision: ${{ env.ARTIFACT_REVISION }}
+          ENOS_VAR_vault_docker_image_archive: ${{steps.download.outputs.download-path}}
+          ENOS_VAR_vault_image_repository: hashicorp/vault
+        run: |
+          enos scenario destroy --timeout 10m0s --chdir ./enos/k8s edition:${{ env.edition }}
+      - name: Cleanup Enos runtime directories
+        if: ${{ always() }}
+        run: |
+          rm -rf /tmp/enos*
+          rm -rf ./enos/support
+          rm -rf ./enos/k8s/.enos

.gitignore

@@ -56,7 +56,7 @@ Vagrantfile
 !.release/linux/package/etc/vault.d/vault.hcl
 !command/agent/config/test-fixtures/*.hcl
 !command/server/test-fixtures/**/*.hcl
-!enos/*.hcl
+!enos/**/*.hcl
 
 # Enos
 enos/.enos

enos/k8s/enos-modules-k8s.hcl

@@ -0,0 +1,49 @@
+module "create_kind_cluster" {
+  source = "../modules/local_kind_cluster"
+}
+
+module "load_docker_image" {
+  source = "../modules/load_docker_image"
+}
+
+module "k8s_deploy_vault" {
+  source = "../modules/k8s_deploy_vault"
+
+  vault_instance_count = var.vault_instance_count
+}
+
+module "k8s_verify_build_date" {
+  source = "../modules/k8s_vault_verify_build_date"
+
+  vault_instance_count = var.vault_instance_count
+}
+
+module "k8s_verify_replication" {
+  source = "../modules/k8s_vault_verify_replication"
+
+  vault_instance_count = var.vault_instance_count
+}
+
+module "k8s_verify_ui" {
+  source = "../modules/k8s_vault_verify_ui"
+
+  vault_instance_count = var.vault_instance_count
+}
+
+module "k8s_verify_version" {
+  source = "../modules/k8s_vault_verify_version"
+
+  vault_instance_count   = var.vault_instance_count
+  vault_product_version  = var.vault_product_version
+  vault_product_revision = var.vault_product_revision
+}
+
+module "k8s_verify_write_data" {
+  source = "../modules/k8s_vault_verify_write_data"
+
+  vault_instance_count   = var.vault_instance_count
+}
+
+module "read_license" {
+  source = "../modules/read_license"
+}

enos/k8s/enos-providers-k8s.hcl

@@ -0,0 +1,7 @@
+provider "enos" "default" {}
+
+provider "helm" "default" {
+  kubernetes {
+    config_path = abspath(joinpath(path.root, "kubeconfig"))
+  }
+}

enos/k8s/enos-scenario-k8s.hcl

@@ -0,0 +1,139 @@
+scenario "k8s" {
+  matrix {
+    edition = ["oss", "ent"]
+  }
+
+  terraform_cli = terraform_cli.default
+  terraform     = terraform.k8s
+
+  providers = [
+    provider.enos.default,
+    provider.helm.default,
+  ]
+
+  locals {
+    image_path = abspath(var.vault_docker_image_archive)
+
+    image_repo = var.vault_image_repository != null ? var.vault_image_repository : matrix.edition == "oss" ? "hashicorp/vault" : "hashicorp/vault-enterprise"
+    image_tag = replace(var.vault_product_version, "+ent", "-ent")
+
+    // The additional '-0' is required in the constraint since without it, the semver function will
+    // only compare the non-pre-release parts (Major.Minor.Patch) of the version and the constraint,
+    // which can lead to unexpected results.
+    version_includes_build_date = semverconstraint(var.vault_product_version, ">=1.11.0-0")
+  }
+
+  step "read_license" {
+    skip_step = matrix.edition == "oss"
+    module    = module.read_license
+
+    variables {
+      file_name = abspath(joinpath(path.root, "../support/vault.hclic"))
+    }
+  }
+
+  step "create_kind_cluster" {
+    module = module.create_kind_cluster
+
+    variables {
+      kubeconfig_path = abspath(joinpath(path.root, "kubeconfig"))
+    }
+  }
+
+  step "load_docker_image" {
+    module = module.load_docker_image
+
+    variables {
+      cluster_name = step.create_kind_cluster.cluster_name
+      image        = local.image_repo
+      tag          = local.image_tag
+      archive      = var.vault_docker_image_archive
+    }
+
+    depends_on = [step.create_kind_cluster]
+  }
+
+  step "deploy_vault" {
+    module = module.k8s_deploy_vault
+
+    variables {
+      image_tag         = step.load_docker_image.tag
+      context_name      = step.create_kind_cluster.context_name
+      image_repository  = step.load_docker_image.repository
+      kubeconfig_base64 = step.create_kind_cluster.kubeconfig_base64
+      vault_edition     = matrix.edition
+      ent_license       = matrix.edition != "oss" ? step.read_license.license : null
+    }
+
+    depends_on = [step.load_docker_image, step.create_kind_cluster]
+  }
+
+  step "verify_build_date" {
+    skip_step = !local.version_includes_build_date
+    module = module.k8s_verify_build_date
+
+    variables {
+      vault_pods        = step.deploy_vault.vault_pods
+      vault_root_token  = step.deploy_vault.vault_root_token
+      kubeconfig_base64 = step.create_kind_cluster.kubeconfig_base64
+      context_name      = step.create_kind_cluster.context_name
+    }
+
+    depends_on = [step.deploy_vault]
+  }
+
+  step "verify_replication" {
+    module = module.k8s_verify_replication
+
+    variables {
+      vault_pods        = step.deploy_vault.vault_pods
+      vault_edition     = matrix.edition
+      kubeconfig_base64 = step.create_kind_cluster.kubeconfig_base64
+      context_name      = step.create_kind_cluster.context_name
+    }
+
+    depends_on = [step.deploy_vault]
+  }
+
+  step "verify_ui" {
+    module = module.k8s_verify_ui
+    skip_step = matrix.edition == "oss"
+
+    variables {
+      vault_pods        = step.deploy_vault.vault_pods
+      kubeconfig_base64 = step.create_kind_cluster.kubeconfig_base64
+      context_name      = step.create_kind_cluster.context_name
+    }
+
+    depends_on = [step.deploy_vault]
+  }
+
+  step "verify_version" {
+    module = module.k8s_verify_version
+
+    variables {
+      vault_pods        = step.deploy_vault.vault_pods
+      vault_root_token  = step.deploy_vault.vault_root_token
+      vault_edition     = matrix.edition
+      kubeconfig_base64 = step.create_kind_cluster.kubeconfig_base64
+      context_name      = step.create_kind_cluster.context_name
+      check_build_date  = local.version_includes_build_date
+      vault_build_date  = var.vault_build_date
+    }
+
+    depends_on = [step.deploy_vault]
+  }
+
+  step "verify_write_data" {
+    module = module.k8s_verify_write_data
+
+    variables {
+      vault_pods        = step.deploy_vault.vault_pods
+      vault_root_token  = step.deploy_vault.vault_root_token
+      kubeconfig_base64 = step.create_kind_cluster.kubeconfig_base64
+      context_name      = step.create_kind_cluster.context_name
+    }
+
+    depends_on = [step.deploy_vault]
+  }
+}

enos/k8s/enos-terraform-k8s.hcl

@@ -0,0 +1,20 @@
+terraform "k8s" {
+  required_version = ">= 1.2.0"
+  required_providers {
+    enos = {
+      source = "app.terraform.io/hashicorp-qti/enos"
+    }
+
+    helm = {
+      source  = "hashicorp/helm"
+    }
+  }
+}
+
+terraform_cli "default" {
+  plugin_cache_dir = var.terraform_plugin_cache_dir != null ? abspath(var.terraform_plugin_cache_dir) : null
+
+  credentials "app.terraform.io" {
+    token = var.tfc_api_token
+  }
+}

enos/k8s/enos-variables-k8s.hcl

@@ -0,0 +1,46 @@
+variable "vault_image_repository" {
+  description = "The repository for the docker image to load, i.e. hashicorp/vault"
+  type        = string
+  default     = null
+}
+
+variable "vault_product_version" {
+  description = "The vault product version to test"
+  type        = string
+  default     = null
+}
+
+variable "vault_product_revision" {
+  type        = string
+  description = "The vault product revision to test"
+  default     = null
+}
+
+variable "vault_docker_image_archive" {
+  description = "The path to the location of the docker image archive to test"
+  type        = string
+  default     = null
+}
+
+variable "vault_instance_count" {
+  description = "How many instances to create for the Vault cluster"
+  type        = number
+  default     = 3
+}
+
+variable "terraform_plugin_cache_dir" {
+  description = "The directory to cache Terraform modules and providers"
+  type        = string
+  default     = null
+}
+
+variable "tfc_api_token" {
+  description = "The Terraform Cloud QTI Organization API token."
+  type        = string
+}
+
+variable "vault_build_date" {
+  description = "The build date for the vault docker image"
+  type        = string
+  default     = ""
+}

enos/modules/build_local/main.tf

@@ -1,3 +1,11 @@
+terraform {
+  required_providers {
+    enos = {
+      source = "app.terraform.io/hashicorp-qti/enos"
+    }
+  }
+}
+
 variable "bundle_path" {
   type    = string
   default = "/tmp/vault.zip"

enos/modules/k8s_deploy_vault/main.tf

@@ -0,0 +1,161 @@
+terraform {
+  required_version = ">= 1.0"
+
+  required_providers {
+    enos = {
+      source = "app.terraform.io/hashicorp-qti/enos"
+    }
+
+    helm = {
+      source  = "hashicorp/helm"
+      version = "2.6.0"
+    }
+  }
+}
+
+locals {
+  helm_chart_settings = {
+    "server.ha.enabled"             = "true"
+    "server.ha.replicas"            = var.vault_instance_count
+    "server.ha.raft.enabled"        = "true"
+    "server.affinity"               = ""
+    "server.image.repository"       = var.image_repository
+    "server.image.tag"              = var.image_tag
+    "server.image.pullPolicy"       = "Never" # Forces local image use
+    "server.resources.requests.cpu" = "50m"
+    "server.limits.memory"          = "200m"
+    "server.limits.cpu"             = "200m"
+    "server.ha.raft.config"         = file("${abspath(path.module)}/raft-config.hcl")
+    "server.dataStorage.size"       = "100m"
+  }
+  all_helm_chart_settings = var.ent_license == null ? local.helm_chart_settings : merge(local.helm_chart_settings, {
+    "server.extraEnvironmentVars.VAULT_LICENSE" = var.ent_license
+  })
+
+  vault_address = "http://127.0.0.1:8200"
+
+  instance_indexes = [for idx in range(var.vault_instance_count) : tostring(idx)]
+
+  leader_idx    = local.instance_indexes[0]
+  followers_idx = toset(slice(local.instance_indexes, 1, var.vault_instance_count))
+}
+
+resource "helm_release" "vault" {
+  name = "vault"
+
+  repository = "https://helm.releases.hashicorp.com"
+  chart      = "vault"
+
+  dynamic "set" {
+    for_each = local.all_helm_chart_settings
+
+    content {
+      name  = set.key
+      value = set.value
+    }
+  }
+}
+
+data "enos_kubernetes_pods" "vault_pods" {
+  kubeconfig_base64 = var.kubeconfig_base64
+  context_name      = var.context_name
+  namespace         = helm_release.vault.namespace
+  label_selectors = [
+    "app.kubernetes.io/name=vault",
+    "component=server"
+  ]
+
+  depends_on = [helm_release.vault]
+}
+
+resource "enos_vault_init" "leader" {
+  bin_path   = "/bin/vault"
+  vault_addr = local.vault_address
+
+  key_shares    = 5
+  key_threshold = 3
+
+  transport = {
+    kubernetes = {
+      kubeconfig_base64 = var.kubeconfig_base64
+      context_name      = var.context_name
+      pod               = data.enos_kubernetes_pods.vault_pods.pods[local.leader_idx].name
+      namespace         = data.enos_kubernetes_pods.vault_pods.pods[local.leader_idx].namespace
+    }
+  }
+}
+
+resource "enos_vault_unseal" "leader" {
+  bin_path    = "/bin/vault"
+  vault_addr  = local.vault_address
+  seal_type   = "shamir"
+  unseal_keys = enos_vault_init.leader.unseal_keys_b64
+
+  transport = {
+    kubernetes = {
+      kubeconfig_base64 = var.kubeconfig_base64
+      context_name      = var.context_name
+      pod               = data.enos_kubernetes_pods.vault_pods.pods[local.leader_idx].name
+      namespace         = data.enos_kubernetes_pods.vault_pods.pods[local.leader_idx].namespace
+    }
+  }
+
+  depends_on = [enos_vault_init.leader]
+}
+
+// We need to manually join the followers since the join request must only happen after the leader
+// has been initialized. We could use retry join, but in that case we'd need to restart the follower
+// pods once the leader is setup. The default helm deployment configuration for an HA cluster as
+// documented here: https://learn.hashicorp.com/tutorials/vault/kubernetes-raft-deployment-guide#configure-vault-helm-chart
+// uses a liveness probe that automatically restarts nodes that are not healthy. This works well for
+// clusters that are configured with auto-unseal as eventually the nodes would join and unseal.
+resource "enos_remote_exec" "raft_join" {
+  for_each = local.followers_idx
+
+  inline = [
+    // asserts that vault is ready
+    "for i in 1 2 3 4 5; do vault status > /dev/null 2>&1 && break || sleep 5; done",
+    // joins the follower to the leader
+    "vault operator raft join http://vault-0.vault-internal:8200"
+  ]
+
+  transport = {
+    kubernetes = {
+      kubeconfig_base64 = var.kubeconfig_base64
+      context_name      = var.context_name
+      pod               = data.enos_kubernetes_pods.vault_pods.pods[each.key].name
+      namespace         = data.enos_kubernetes_pods.vault_pods.pods[each.key].namespace
+    }
+  }
+
+  depends_on = [enos_vault_unseal.leader]
+}
+
+
+resource "enos_vault_unseal" "followers" {
+  for_each = local.followers_idx
+
+  bin_path    = "/bin/vault"
+  vault_addr  = local.vault_address
+  seal_type   = "shamir"
+  unseal_keys = enos_vault_init.leader.unseal_keys_b64
+
+  transport = {
+    kubernetes = {
+      kubeconfig_base64 = var.kubeconfig_base64
+      context_name      = var.context_name
+      pod               = data.enos_kubernetes_pods.vault_pods.pods[each.key].name
+      namespace         = data.enos_kubernetes_pods.vault_pods.pods[each.key].namespace
+    }
+  }
+
+  depends_on = [enos_remote_exec.raft_join]
+}
+
+output "vault_root_token" {
+  value = enos_vault_init.leader.root_token
+}
+
+output "vault_pods" {
+  value = data.enos_kubernetes_pods.vault_pods.pods
+}

enos/modules/k8s_deploy_vault/raft-config.hcl

@@ -0,0 +1,20 @@
+ui = true
+listener "tcp" {
+  address = "[::]:8200"
+  cluster_address = "[::]:8201"
+  tls_disable = true
+}
+
+storage "raft" {
+  path = "/vault/data"
+  autopilot {
+    cleanup_dead_servers = "true"
+    last_contact_threshold = "200ms"
+    last_contact_failure_threshold = "10m"
+    max_trailing_logs = 250000
+    min_quorum = 5
+    server_stabilization_time = "10s"
+  }
+}
+
+service_registration "kubernetes" {}

enos/modules/k8s_deploy_vault/variables.tf

@@ -0,0 +1,34 @@
+variable "context_name" {
+  type        = string
+  description = "The name of the k8s context for Vault"
+}
+
+variable "ent_license" {
+  type        = string
+  description = "The value of a valid Vault Enterprise license"
+}
+
+variable "image_repository" {
+  type        = string
+  description = "The name of the Vault repository, ie hashicorp/vault or hashicorp/vault-enterprise for the image to deploy"
+}
+
+variable "image_tag" {
+  type        = string
+  description = "The tag of the vault image to deploy"
+}
+
+variable "kubeconfig_base64" {
+  type        = string
+  description = "The base64 encoded version of the Kubernetes configuration file"
+}
+
+variable "vault_edition" {
+  type        = string
+  description = "The Vault product edition"
+}
+
+variable "vault_instance_count" {
+  type        = number
+  description = "How many vault instances are in the cluster"
+}

enos/modules/k8s_vault_verify_build_date/main.tf

@@ -0,0 +1,58 @@
+
+terraform {
+  required_providers {
+    enos = {
+      source = "app.terraform.io/hashicorp-qti/enos"
+    }
+  }
+}
+
+locals {
+  vault_instances = toset([for idx in range(var.vault_instance_count) : tostring(idx)])
+}
+
+# Get the date from the vault status command      - status_date
+# Format the original status output with ISO-8601 - formatted_date
+# Format the original status output with awk      - awk_date
+# Compare the formatted outputs                   - date_comparison
+resource "enos_remote_exec" "status_date" {
+  for_each = local.vault_instances
+
+  transport = {
+    kubernetes = {
+      kubeconfig_base64 = var.kubeconfig_base64
+      context_name      = var.context_name
+      pod               = var.vault_pods[each.key].name
+      namespace         = var.vault_pods[each.key].namespace
+    }
+  }
+
+  inline = ["${var.vault_bin_path} status -format=json | grep build_date | cut -d \\\" -f 4"]
+}
+
+resource "enos_remote_exec" "formatted_date" {
+  for_each = local.vault_instances
+
+  transport = {
+    kubernetes = {
+      kubeconfig_base64 = var.kubeconfig_base64
+      context_name      = var.context_name
+      pod               = var.vault_pods[each.key].name
+      namespace         = var.vault_pods[each.key].namespace
+    }
+  }
+
+  inline = ["date -d \"${enos_remote_exec.status_date[each.key].stdout}\" -D '%Y-%m-%dT%H:%M:%SZ' -I"]
+}
+
+resource "enos_local_exec" "awk_date" {
+  for_each = local.vault_instances
+
+  inline = ["echo ${enos_remote_exec.status_date[each.key].stdout} | awk -F\"T\" '{printf $1}'"]
+}
+
+resource "enos_local_exec" "date_comparison" {
+  for_each = local.vault_instances
+
+  inline = ["[[ ${enos_local_exec.awk_date[each.key].stdout} == ${enos_remote_exec.formatted_date[each.key].stdout} ]] && echo \"Verification for build date format ${enos_remote_exec.status_date[each.key].stdout} succeeded\" || \"invalid build_date, must be formatted as RFC 3339: ${enos_remote_exec.status_date[each.key].stdout}\""]
+}

enos/modules/k8s_vault_verify_build_date/variables.tf

@@ -0,0 +1,33 @@
+variable "vault_instance_count" {
+  type        = number
+  description = "How many vault instances are in the cluster"
+}
+
+variable "vault_pods" {
+  type = list(object({
+    name      = string
+    namespace = string
+  }))
+  description = "The vault instances for the cluster to verify"
+}
+
+variable "vault_bin_path" {
+  type        = string
+  description = "The path to the vault binary"
+  default     = "/bin/vault"
+}
+
+variable "vault_root_token" {
+  type        = string
+  description = "The vault root token"
+}
+
+variable "kubeconfig_base64" {
+  type        = string
+  description = "The base64 encoded version of the Kubernetes configuration file"
+}
+
+variable "context_name" {
+  type        = string
+  description = "The name of the k8s context for Vault"
+}

enos/modules/k8s_vault_verify_replication/main.tf

@@ -0,0 +1,39 @@
+
+terraform {
+  required_providers {
+    enos = {
+      source = "app.terraform.io/hashicorp-qti/enos"
+    }
+  }
+}
+
+locals {
+  instances = toset([for idx in range(var.vault_instance_count) : tostring(idx)])
+}
+
+resource "enos_remote_exec" "replication_status" {
+  for_each = local.instances
+
+  inline = ["vault read -format=json sys/replication/status"]
+
+  transport = {
+    kubernetes = {
+      kubeconfig_base64 = var.kubeconfig_base64
+      context_name      = var.context_name
+      pod               = var.vault_pods[each.key].name
+      namespace         = var.vault_pods[each.key].namespace
+    }
+  }
+}
+
+resource "enos_local_exec" "verify_replication_status" {
+
+  for_each = enos_remote_exec.replication_status
+
+  environment = {
+    STATUS        = each.value.stdout
+    VAULT_EDITION = var.vault_edition
+  }
+
+  content = abspath("${path.module}/scripts/smoke-verify-replication.sh")
+}

enos/modules/k8s_vault_verify_replication/scripts/smoke-verify-replication.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash 
+
+# The Vault replication smoke test, documented in
+# https://docs.google.com/document/d/16sjIk3hzFDPyY5A9ncxTZV_9gnpYSF1_Vx6UA1iiwgI/edit#heading=h.kgrxf0f1et25
+
+set -e
+
+fail() {
+	echo "$1" 1>&2
+	exit 1
+}
+
+# Replication STATUS endpoint should have data.mode disabled for OSS release
+if [ "$VAULT_EDITION" == "oss" ]; then
+  if [ "$(echo "${STATUS}" | jq -r '.data.mode')" != "disabled" ]; then
+    fail "replication data mode is not disabled for OSS release!"
+  fi
+else
+  if [ "$(echo "${STATUS}" | jq -r '.data.dr')" == "" ]; then
+    fail "DR replication should be available for an ENT release!"
+  fi
+  if [ "$(echo "${STATUS}" | jq -r '.data.performance')" == "" ]; then
+    fail "Performance replication should be available for an ENT release!"
+  fi
+fi

enos/modules/k8s_vault_verify_replication/variables.tf

@@ -0,0 +1,27 @@
+variable "vault_instance_count" {
+  type        = number
+  description = "How many vault instances are in the cluster"
+}
+
+variable "vault_edition" {
+  type        = string
+  description = "The vault product edition"
+}
+
+variable "vault_pods" {
+  type = list(object({
+    name      = string
+    namespace = string
+  }))
+  description = "The vault instances for the cluster to verify"
+}
+
+variable "kubeconfig_base64" {
+  type        = string
+  description = "The base64 encoded version of the Kubernetes configuration file"
+}
+
+variable "context_name" {
+  type        = string
+  description = "The name of the k8s context for Vault"
+}

enos/modules/k8s_vault_verify_ui/main.tf

@@ -0,0 +1,42 @@
+
+terraform {
+  required_providers {
+    enos = {
+      version = ">= 0.1.17"
+      source  = "app.terraform.io/hashicorp-qti/enos"
+    }
+  }
+}
+
+locals {
+  instances = toset([for idx in range(var.vault_instance_count) : tostring(idx)])
+}
+
+resource "enos_remote_exec" "curl_ui" {
+  for_each = local.instances
+
+  inline = [
+    "curl -s -o /dev/null -w '%%{redirect_url}' http://localhost:8200/",
+    "curl -s -o /dev/null -Iw '%%{http_code}\n' http://localhost:8200/ui/"
+  ]
+
+  transport = {
+    kubernetes = {
+      kubeconfig_base64 = var.kubeconfig_base64
+      context_name      = var.context_name
+      pod               = var.vault_pods[each.key].name
+      namespace         = var.vault_pods[each.key].namespace
+    }
+  }
+}
+
+resource "enos_local_exec" "verify_ui" {
+  for_each = enos_remote_exec.curl_ui
+
+  environment = {
+    REDIRECT_URL  = split("\n", each.value.stdout)[0]
+    UI_URL_RESULT = split("\n", each.value.stdout)[1]
+  }
+
+  scripts = [abspath("${path.module}/scripts/smoke-verify-ui.sh")]
+}

enos/modules/k8s_vault_verify_ui/scripts/smoke-verify-ui.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -e
+
+fail() {
+	echo "$1" 1>&2
+	exit 1
+}
+
+if [ "${REDIRECT_URL}" != "http://localhost:8200/ui/" ]; then
+    fail "Port 8200 not redirecting to UI"
+fi
+if [ "${UI_URL_RESULT}" != "200" ]; then
+    fail "Vault UI is not available"
+fi

enos/modules/k8s_vault_verify_ui/variables.tf

@@ -0,0 +1,22 @@
+variable "vault_instance_count" {
+  type        = number
+  description = "How many vault instances are in the cluster"
+}
+
+variable "vault_pods" {
+  type = list(object({
+    name      = string
+    namespace = string
+  }))
+  description = "The vault instances for the cluster to verify"
+}
+
+variable "kubeconfig_base64" {
+  type        = string
+  description = "The base64 encoded version of the Kubernetes configuration file"
+}
+
+variable "context_name" {
+  type        = string
+  description = "The name of the k8s context for Vault"
+}

enos/modules/k8s_vault_verify_version/main.tf

@@ -0,0 +1,48 @@
+
+terraform {
+  required_providers {
+    enos = {
+      source = "app.terraform.io/hashicorp-qti/enos"
+    }
+  }
+}
+
+locals {
+  instances        = toset([for idx in range(var.vault_instance_count) : tostring(idx)])
+  expected_version = var.vault_edition == "oss" ? var.vault_product_version : "${var.vault_product_version}-ent"
+}
+
+resource "enos_remote_exec" "release_info" {
+  for_each = local.instances
+
+  environment = {
+    VAULT_BIN_PATH = var.vault_bin_path
+  }
+
+  scripts = [abspath("${path.module}/scripts/get-status.sh")]
+
+  transport = {
+    kubernetes = {
+      kubeconfig_base64 = var.kubeconfig_base64
+      context_name      = var.context_name
+      pod               = var.vault_pods[each.key].name
+      namespace         = var.vault_pods[each.key].namespace
+    }
+  }
+}
+
+resource "enos_local_exec" "smoke-verify-version" {
+  for_each = enos_remote_exec.release_info
+
+  environment = {
+    VAULT_STATUS     = jsonencode(jsondecode(each.value.stdout).status)
+    ACTUAL_VERSION   = jsondecode(each.value.stdout).version
+    EXPECTED_VERSION = var.vault_product_version,
+    VAULT_EDITION    = var.vault_edition,
+    VAULT_REVISION   = var.vault_product_revision,
+    CHECK_BUILD_DATE = var.check_build_date
+    BUILD_DATE       = var.vault_build_date
+  }
+
+  scripts = [abspath("${path.module}/scripts/smoke-verify-version.sh")]
+}

enos/modules/k8s_vault_verify_version/scripts/get-status.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env sh
+
+set -e
+
+status=$(${VAULT_BIN_PATH} status -format=json)
+version=$(${VAULT_BIN_PATH} version)
+
+echo "{\"status\": ${status}, \"version\": \"${version}\"}"

enos/modules/k8s_vault_verify_version/scripts/smoke-verify-version.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+# The Vault smoke test to verify the Vault version installed
+
+set -e
+
+fail() {
+	echo "$1" 1>&2
+	exit 1
+}
+
+if [[ "${CHECK_BUILD_DATE}" == "false" ]]; then
+  expected_build_date=""
+else
+  build_date="${BUILD_DATE}"
+  if [[ "${build_date}" == "" ]]; then
+    build_date=$(echo "${VAULT_STATUS}" | jq -Mr .build_date)
+  fi
+  expected_build_date=", built $build_date"
+fi
+
+vault_expected_version="Vault v${EXPECTED_VERSION} (${VAULT_REVISION})"
+
+case "${VAULT_EDITION}" in
+  oss) version_expected="${vault_expected_version}${expected_build_date}";;
+	ent) version_expected="${vault_expected_version}${expected_build_date}";;
+	ent.hsm) version_expected="${vault_expected_version}${expected_build_date} (cgo)";;
+	ent.fips1402) version_expected="${vault_expected_version}${expected_build_date} (cgo)" ;;
+	ent.hsm.fips1402) version_expected="${vault_expected_version}${expected_build_date} (cgo)" ;;
+  *) fail "(${VAULT_EDITION}) does not match any known Vault editions"
+esac
+
+version_expected_nosha=$(echo "$version_expected" | awk '!($3="")' | sed 's/  / /' | sed -e 's/[[:space:]]*$//')
+
+if [[ "${ACTUAL_VERSION}" == "$version_expected_nosha" ]] || [[ "${ACTUAL_VERSION}" == "$version_expected" ]]; then
+	echo "Version verification succeeded!"
+else
+  echo "CHECK_BUILD_DATE: ${CHECK_BUILD_DATE}"
+  echo "BUILD_DATE: ${BUILD_DATE}"
+  echo "build_date: ${build_date}"
+	fail "expected Version=$version_expected or $version_expected_nosha, got: ${ACTUAL_VERSION}"
+fi

enos/modules/k8s_vault_verify_version/variables.tf

@@ -0,0 +1,59 @@
+variable "vault_instance_count" {
+  type        = number
+  description = "How many vault instances are in the cluster"
+}
+
+variable "vault_pods" {
+  type = list(object({
+    name      = string
+    namespace = string
+  }))
+  description = "The vault instances for the cluster to verify"
+}
+
+variable "vault_bin_path" {
+  type        = string
+  description = "The path to the vault binary"
+  default     = "/bin/vault"
+}
+
+variable "vault_product_version" {
+  type        = string
+  description = "The vault product version"
+}
+
+variable "vault_product_revision" {
+  type        = string
+  description = "The vault product revision"
+}
+
+variable "vault_edition" {
+  type        = string
+  description = "The vault product edition"
+}
+
+variable "vault_root_token" {
+  type        = string
+  description = "The vault root token"
+}
+
+variable "kubeconfig_base64" {
+  type        = string
+  description = "The base64 encoded version of the Kubernetes configuration file"
+}
+
+variable "context_name" {
+  type        = string
+  description = "The name of the k8s context for Vault"
+}
+
+variable "check_build_date" {
+  type        = bool
+  description = "Whether or not to verify that the version includes the build date"
+}
+
+variable "vault_build_date" {
+  type        = string
+  description = "The build date of the vault docker image to check"
+  default     = ""
+}

enos/modules/k8s_vault_verify_write_data/main.tf

@@ -0,0 +1,50 @@
+
+terraform {
+  required_providers {
+    enos = {
+      source = "app.terraform.io/hashicorp-qti/enos"
+    }
+  }
+}
+
+locals {
+  instances = toset([for idx in range(var.vault_instance_count) : tostring(idx)])
+}
+
+resource "enos_remote_exec" "smoke-enable-secrets-kv" {
+  environment = {
+    VAULT_TOKEN = var.vault_root_token
+  }
+
+  inline = ["${var.vault_bin_path} secrets enable -path=\"secret\" kv"]
+
+  transport = {
+    kubernetes = {
+      kubeconfig_base64 = var.kubeconfig_base64
+      context_name      = var.context_name
+      pod               = var.vault_pods[0].name
+      namespace         = var.vault_pods[0].namespace
+    }
+  }
+}
+
+# Verify that we can enable the k/v secrets engine and write data to it.
+resource "enos_remote_exec" "smoke-write-test-data" {
+  depends_on = [enos_remote_exec.smoke-enable-secrets-kv]
+  for_each   = local.instances
+
+  environment = {
+    VAULT_TOKEN = var.vault_root_token
+  }
+
+  inline = ["${var.vault_bin_path} kv put secret/test smoke${each.key}=fire"]
+
+  transport = {
+    kubernetes = {
+      kubeconfig_base64 = var.kubeconfig_base64
+      context_name      = var.context_name
+      pod               = var.vault_pods[each.key].name
+      namespace         = var.vault_pods[each.key].namespace
+    }
+  }
+}

enos/modules/k8s_vault_verify_write_data/variables.tf

@@ -0,0 +1,33 @@
+variable "vault_instance_count" {
+  type        = number
+  description = "How many vault instances are in the cluster"
+}
+
+variable "vault_pods" {
+  type = list(object({
+    name      = string
+    namespace = string
+  }))
+  description = "The vault instances for the cluster to verify"
+}
+
+variable "vault_bin_path" {
+  type        = string
+  description = "The path to the vault binary"
+  default     = "/bin/vault"
+}
+
+variable "vault_root_token" {
+  type        = string
+  description = "The vault root token"
+}
+
+variable "kubeconfig_base64" {
+  type        = string
+  description = "The base64 encoded version of the Kubernetes configuration file"
+}
+
+variable "context_name" {
+  type        = string
+  description = "The name of the k8s context for Vault"
+}

enos/modules/load_docker_image/main.tf

@@ -0,0 +1,50 @@
+terraform {
+  required_providers {
+    enos = {
+      source = "app.terraform.io/hashicorp-qti/enos"
+    }
+  }
+}
+
+variable "cluster_name" {
+  type        = string
+  description = "The name of the cluster to load the image into"
+}
+
+variable "image" {
+  type        = string
+  description = "The image name for the image to load, i.e. hashicorp/vault"
+}
+
+variable "tag" {
+  type        = string
+  description = "The tag for the image to load, i.e. 1.12.0-dev"
+}
+
+variable "archive" {
+  type        = string
+  description = "The path to the image archive to load"
+  default     = null
+}
+
+resource "enos_local_kind_load_image" "vault" {
+  cluster_name = var.cluster_name
+  image        = var.image
+  tag          = var.tag
+  archive      = var.archive
+}
+
+output "tag" {
+  value       = var.tag
+  description = "The tag of the docker image to load without the tag, i.e. 1.10.0"
+}
+
+output "image" {
+  value       = var.image
+  description = "The tag of the docker image to load without the tag, i.e. vault"
+}
+
+output "repository" {
+  value       = enos_local_kind_load_image.vault.loaded_images.repository
+  description = "The name of the image's repository, i.e. hashicorp/vault"
+}

enos/modules/local_kind_cluster/main.tf

@@ -0,0 +1,50 @@
+terraform {
+  required_providers {
+    enos = {
+      source = "app.terraform.io/hashicorp-qti/enos"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = ">= 3.4.3"
+    }
+  }
+}
+
+resource "random_pet" "cluster_name" {}
+
+resource "enos_local_kind_cluster" "this" {
+  name            = random_pet.cluster_name.id
+  kubeconfig_path = var.kubeconfig_path
+}
+
+variable "kubeconfig_path" {
+  type = string
+}
+
+output "cluster_name" {
+  value = random_pet.cluster_name.id
+}
+
+output "kubeconfig_base64" {
+  value = enos_local_kind_cluster.this.kubeconfig_base64
+}
+
+output "context_name" {
+  value = enos_local_kind_cluster.this.context_name
+}
+
+output "host" {
+  value = enos_local_kind_cluster.this.endpoint
+}
+
+output "client_certificate" {
+  value = enos_local_kind_cluster.this.client_certificate
+}
+
+output "client_key" {
+  value = enos_local_kind_cluster.this.client_key
+}
+
+output "cluster_ca_certificate" {
+  value = enos_local_kind_cluster.this.cluster_ca_certificate
+}