remove foundationdb
This commit is contained in:
parent
bea345a84c
commit
e0e8caaa58
4
Makefile
4
Makefile
|
|
@ -18,10 +18,6 @@ GO_VERSION_MIN=$$(cat $(CURDIR)/.go-version)
|
|||
PROTOC_VERSION_MIN=3.21.12
|
||||
GO_CMD?=go
|
||||
CGO_ENABLED?=0
|
||||
ifneq ($(FDB_ENABLED), )
|
||||
CGO_ENABLED=1
|
||||
BUILD_TAGS+=foundationdb
|
||||
endif
|
||||
|
||||
default: dev
|
||||
|
||||
|
|
|
|||
|
|
@ -42,7 +42,6 @@ import (
|
|||
logicalDb "github.com/hashicorp/vault/builtin/logical/database"
|
||||
|
||||
physConsul "github.com/hashicorp/vault/physical/consul"
|
||||
physFoundationDB "github.com/hashicorp/vault/physical/foundationdb"
|
||||
physOCI "github.com/hashicorp/vault/physical/oci"
|
||||
physRaft "github.com/hashicorp/vault/physical/raft"
|
||||
physFile "github.com/hashicorp/vault/sdk/physical/file"
|
||||
|
|
@ -168,7 +167,6 @@ var (
|
|||
"consul": physConsul.NewConsulBackend,
|
||||
"file_transactional": physFile.NewTransactionalFileBackend,
|
||||
"file": physFile.NewFileBackend,
|
||||
"foundationdb": physFoundationDB.NewFDBBackend,
|
||||
"inmem_ha": physInmem.NewInmemHA,
|
||||
"inmem_transactional_ha": physInmem.NewTransactionalInmemHA,
|
||||
"inmem_transactional": physInmem.NewTransactionalInmem,
|
||||
|
|
|
|||
2
go.mod
2
go.mod
|
|
@ -26,7 +26,6 @@ replace github.com/hashicorp/vault/sdk => ./sdk
|
|||
|
||||
require (
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230626094100-7e9e0395ebec
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20190411004307-cd5c9d91fad2
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/armon/go-radix v1.0.0
|
||||
github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef
|
||||
|
|
@ -173,7 +172,6 @@ require (
|
|||
golang.org/x/tools v0.18.0
|
||||
google.golang.org/grpc v1.61.1
|
||||
google.golang.org/protobuf v1.34.1
|
||||
gopkg.in/ory-am/dockertest.v3 v3.3.4
|
||||
k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5
|
||||
layeh.com/radius v0.0.0-20231213012653-1006025d24f8
|
||||
nhooyr.io/websocket v1.8.7
|
||||
|
|
|
|||
4
go.sum
4
go.sum
|
|
@ -966,8 +966,6 @@ github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/P
|
|||
github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
|
||||
github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw=
|
||||
github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20190411004307-cd5c9d91fad2 h1:VoHKYIXEQU5LWoambPBOvYxyLqZYHuj+rj5DVnMUc3k=
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20190411004307-cd5c9d91fad2/go.mod h1:OMVSB21p9+xQUIqlGizHPZfjK+SHws1ht+ZytVDoz9U=
|
||||
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
|
||||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
||||
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
|
||||
|
|
@ -3997,8 +3995,6 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
|||
gopkg.in/jcmturner/goidentity.v3 v3.0.0 h1:1duIyWiTaYvVx3YX2CYtpJbUFd7/UuPYCfgXtQ3VTbI=
|
||||
gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
|
||||
gopkg.in/ory-am/dockertest.v3 v3.3.4 h1:oen8RiwxVNxtQ1pRoV4e4jqh6UjNsOuIZ1NXns6jdcw=
|
||||
gopkg.in/ory-am/dockertest.v3 v3.3.4/go.mod h1:s9mmoLkaGeAh97qygnNj4xWkiN7e1SKekYC6CovU+ek=
|
||||
gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI=
|
||||
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
|
||||
gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
|
||||
|
|
|
|||
|
|
@ -1,47 +0,0 @@
|
|||
# FoundationDB storage backend
|
||||
|
||||
Extra steps are required to produce a Vault build containing the FoundationDB
|
||||
backend; attempts to use the backend on a build produced without following
|
||||
this procedure will fail with a descriptive error message at runtime.
|
||||
|
||||
## Installing the Go bindings
|
||||
|
||||
### Picking a version
|
||||
|
||||
The version of the Go bindings and the FoundationDB client library used to
|
||||
build them must match.
|
||||
|
||||
This version will determine the minimum API version that can be used, hence
|
||||
it should be no higher than the version of FoundationDB used in your cluster,
|
||||
and must also satisfy the requirements of the backend code.
|
||||
|
||||
The minimum required API version for the FoundationDB backend is 520.
|
||||
|
||||
### Installation
|
||||
|
||||
Make sure you have Mono installed (core is enough), then install the
|
||||
Go bindings using the `fdb-go-install.sh` script:
|
||||
|
||||
```
|
||||
$ physical/foundationdb/fdb-go-install.sh install --fdbver x.y.z
|
||||
```
|
||||
|
||||
By default, if `--fdbver x.y.z` is not specified, version 5.2.4 will be used.
|
||||
|
||||
## Building Vault
|
||||
|
||||
To build Vault the FoundationDB backend, add FDB_ENABLED=1 when invoking
|
||||
`make`, e.g.
|
||||
|
||||
```
|
||||
$ make dev FDB_ENABLED=1
|
||||
```
|
||||
|
||||
## Running tests
|
||||
|
||||
Similarly, add FDB_ENABLED=1 to your `make` invocation when running tests,
|
||||
e.g.
|
||||
|
||||
```
|
||||
$ make test TEST=./physical/foundationdb FDB_ENABLED=1
|
||||
```
|
||||
|
|
@ -1,333 +0,0 @@
|
|||
#!/bin/bash -eu
|
||||
# Copyright (c) HashiCorp, Inc.
|
||||
# SPDX-License-Identifier: BUSL-1.1
|
||||
|
||||
#
|
||||
# fdb-go-install.sh
|
||||
#
|
||||
# Installs the FoundationDB Go bindings for a client. This will download
|
||||
# the repository from the remote repo either into the go directory
|
||||
# with the appropriate semantic version. It will then build a few
|
||||
# generated files that need to be present for the go build to work.
|
||||
# At the end, it has some advice for flags to modify within your
|
||||
# go environment so that other packages may successfully use this
|
||||
# library.
|
||||
#
|
||||
|
||||
DESTDIR="${DESTDIR:-}"
|
||||
FDBVER="${FDBVER:-5.2.4}"
|
||||
REMOTE="${REMOTE:-github.com}"
|
||||
FDBREPO="${FDBREPO:-apple/foundationdb}"
|
||||
|
||||
status=0
|
||||
|
||||
platform=$(uname)
|
||||
if [[ "${platform}" == "Darwin" ]] ; then
|
||||
FDBLIBDIR="${FDBLIBDIR:-/usr/local/lib}"
|
||||
libfdbc="libfdb_c.dylib"
|
||||
elif [[ "${platform}" == "Linux" ]] ; then
|
||||
libfdbc="libfdb_c.so"
|
||||
custom_libdir="${FDBLIBDIR:-}"
|
||||
FDBLIBDIR=""
|
||||
|
||||
if [[ -z "${custom_libdir}" ]]; then
|
||||
search_libdirs=( '/usr/lib' '/usr/lib64' )
|
||||
else
|
||||
search_libdirs=( "${custom_libdir}" )
|
||||
fi
|
||||
|
||||
for libdir in "${search_libdirs[@]}" ; do
|
||||
if [[ -e "${libdir}/${libfdbc}" ]]; then
|
||||
FDBLIBDIR="${libdir}"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -z "${FDBLIBDIR}" ]]; then
|
||||
echo "The FoundationDB C library could not be found in any of:"
|
||||
for libdir in "${search_libdirs[@]}" ; do
|
||||
echo " ${libdir}"
|
||||
done
|
||||
echo "Your installation may be incomplete, or you need to set a custom FDBLIBDIR."
|
||||
let status="${status} + 1"
|
||||
fi
|
||||
|
||||
else
|
||||
echo "Unsupported platform ${platform}".
|
||||
echo "At the moment, only macOS and Linux are supported by this script."
|
||||
let status="${status} + 1"
|
||||
fi
|
||||
|
||||
filedir=$(cd `dirname "${BASH_SOURCE[0]}"` && pwd)
|
||||
destdir=""
|
||||
|
||||
function printUsage() {
|
||||
echo "Usage: fdb-go-install.sh <cmd>"
|
||||
echo
|
||||
echo "cmd: One of the commands to run. The options are:"
|
||||
echo " install Download the FDB go bindings and install them"
|
||||
echo " localinstall Install a into the go path a local copy of the repo"
|
||||
echo " download Download but do not prepare the FoundationDB bindings"
|
||||
echo " help Print this help message and then quit"
|
||||
echo
|
||||
echo "Command Line Options:"
|
||||
echo " --fdbver <version> FoundationDB semantic version (default is ${FDBVER})"
|
||||
echo " -d/--dest-dir <dest> Local location for the repo (default is to place in go path)"
|
||||
echo
|
||||
echo "Environment Variable Options:"
|
||||
echo " REMOTE Remote repository to download from (currently ${REMOTE})"
|
||||
echo " FDBREPO Repository of FoundationDB library to download (currently ${FDBREPO})"
|
||||
echo " FDBLIBDIR Directory within which should be the FoundationDB c library (currently ${FDBLIBDIR})"
|
||||
}
|
||||
|
||||
function parseArgs() {
|
||||
local status=0
|
||||
|
||||
if [[ "${#}" -lt 0 ]] ; then
|
||||
printUsage
|
||||
let status="${status} + 1"
|
||||
else
|
||||
operation="${1}"
|
||||
shift
|
||||
if [[ "${operation}" != "install" ]] && [[ "${operation}" != "localinstall" ]] && [[ "${operation}" != "download" ]] && [[ "${operation}" != "help" ]] ; then
|
||||
echo "Unknown command: ${operation}"
|
||||
printUsage
|
||||
let status="${status} + 1"
|
||||
fi
|
||||
fi
|
||||
|
||||
while [[ "${#}" -gt 0 ]] && [[ "${status}" -eq 0 ]] ; do
|
||||
local key="${1}"
|
||||
case "${key}" in
|
||||
--fdbver)
|
||||
if [[ "${#}" -lt 2 ]] ; then
|
||||
echo "No version specified with --fdbver flag"
|
||||
printUsage
|
||||
let status="${status} + 1"
|
||||
else
|
||||
FDBVER="${2}"
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
|
||||
-d|--dest-dir)
|
||||
if [[ "${#}" -lt 2 ]] ; then
|
||||
echo "No destination specified with ${key} flag"
|
||||
printUsage
|
||||
let status="${status} + 1"
|
||||
else
|
||||
destdir="${2}"
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "Unrecognized argument ${key}"
|
||||
printUsage
|
||||
let status="${status} + 1"
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
return "${status}"
|
||||
}
|
||||
|
||||
function checkBin() {
|
||||
if [[ "${#}" -lt 1 ]] ; then
|
||||
echo "Usage: checkBin <binary>"
|
||||
return 1
|
||||
else
|
||||
if [[ -n $(which "${1}") ]] ; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ "${status}" -gt 0 ]] ; then
|
||||
# We have already failed.
|
||||
:
|
||||
elif [[ "${#}" -lt 1 ]] ; then
|
||||
printUsage
|
||||
else
|
||||
required_bins=( 'go' 'git' 'make' 'mono' )
|
||||
|
||||
missing_bins=()
|
||||
for bin in "${required_bins[@]}" ; do
|
||||
if ! checkBin "${bin}" ; then
|
||||
missing_bins+=("${bin}")
|
||||
let status="${status} + 1"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "${status}" -gt 0 ]] ; then
|
||||
echo "Missing binaries: ${missing_bins[*]}"
|
||||
elif ! parseArgs ${@} ; then
|
||||
let status="${status} + 1"
|
||||
elif [[ "${operation}" == "help" ]] ; then
|
||||
printUsage
|
||||
else
|
||||
# Add go-specific environment variables.
|
||||
eval $(go env)
|
||||
|
||||
golibdir=$(dirname "${GOPATH}/src/${REMOTE}/${FDBREPO}")
|
||||
if [[ -z "${destdir}" ]] ; then
|
||||
if [[ "${operation}" == "localinstall" ]] ; then
|
||||
# Assume its the local directory.
|
||||
destdir=$(cd "${filedir}/../../.." && pwd)
|
||||
else
|
||||
destdir="${golibdir}"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ ! -d "${destdir}" ]] ; then
|
||||
cmd=( 'mkdir' '-p' "${destdir}" )
|
||||
echo "${cmd[*]}"
|
||||
if ! "${cmd[@]}" ; then
|
||||
let status="${status} + 1"
|
||||
echo "Could not create destination directory ${destdir}."
|
||||
fi
|
||||
fi
|
||||
|
||||
# Step 1: Make sure repository is present.
|
||||
|
||||
if [[ "${status}" -eq 0 ]] ; then
|
||||
destdir=$( cd "${destdir}" && pwd ) # Get absolute path of destination dir.
|
||||
fdbdir="${destdir}/foundationdb"
|
||||
|
||||
if [[ ! -d "${destdir}" ]] ; then
|
||||
cmd=("mkdir" "-p" "${destdir}")
|
||||
echo "${cmd[*]}"
|
||||
if ! "${cmd[@]}" ; then
|
||||
echo "Could not create destination directory ${destdir}."
|
||||
let status="${status} + 1"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "${operation}" == "localinstall" ]] ; then
|
||||
# No download occurs in this case.
|
||||
:
|
||||
else
|
||||
if [[ -d "${fdbdir}" ]] ; then
|
||||
echo "Directory ${fdbdir} already exists ; checking out appropriate tag"
|
||||
cmd1=( 'git' '-C' "${fdbdir}" 'fetch' 'origin' )
|
||||
cmd2=( 'git' '-C' "${fdbdir}" 'checkout' "${FDBVER}" )
|
||||
|
||||
if ! echo "${cmd1[*]}" || ! "${cmd1[@]}" ; then
|
||||
let status="${status} + 1"
|
||||
echo "Could not pull latest changes from origin"
|
||||
elif ! echo "${cmd2[*]}" || ! "${cmd2[@]}" ; then
|
||||
let status="${status} + 1"
|
||||
echo "Could not checkout tag ${FDBVER}."
|
||||
fi
|
||||
else
|
||||
echo "Downloading foundation repository into ${destdir}:"
|
||||
cmd=( 'git' '-C' "${destdir}" 'clone' '--branch' "${FDBVER}" "https://${REMOTE}/${FDBREPO}.git" )
|
||||
|
||||
echo "${cmd[*]}"
|
||||
if ! "${cmd[@]}" ; then
|
||||
let status="${status} + 1"
|
||||
echo "Could not download repository."
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Step 2: Build generated things.
|
||||
|
||||
if [[ "${operation}" == "download" ]] ; then
|
||||
# The generated files are not created under a strict download.
|
||||
:
|
||||
elif [[ "${status}" -eq 0 ]] ; then
|
||||
echo "Building generated files."
|
||||
# FoundationDB starting with 6.0 can figure that out on its own
|
||||
if [ -e '/usr/bin/mcs' ]; then
|
||||
MCS_BIN=/usr/bin/mcs
|
||||
else
|
||||
MCS_BIN=/usr/bin/dmcs
|
||||
fi
|
||||
cmd=( 'make' '-C' "${fdbdir}" 'bindings/c/foundationdb/fdb_c_options.g.h' "MCS=$MCS_BIN" )
|
||||
|
||||
echo "${cmd[*]}"
|
||||
if ! "${cmd[@]}" ; then
|
||||
let status="${status} + 1"
|
||||
echo "Could not generate required c header"
|
||||
else
|
||||
infile="${fdbdir}/fdbclient/vexillographer/fdb.options"
|
||||
outfile="${fdbdir}/bindings/go/src/fdb/generated.go"
|
||||
cmd=( 'go' 'run' "${fdbdir}/bindings/go/src/_util/translate_fdb_options.go" )
|
||||
echo "${cmd[*]} < ${infile} > ${outfile}"
|
||||
if ! "${cmd[@]}" < "${infile}" > "${outfile}" ; then
|
||||
let status="${status} + 1"
|
||||
echo "Could not generate generated go file."
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Step 3: Add to go path.
|
||||
|
||||
if [[ "${operation}" == "download" ]] ; then
|
||||
# The files are not moved under a strict download.
|
||||
:
|
||||
elif [[ "${status}" -eq 0 ]] ; then
|
||||
linkpath="${GOPATH}/src/${REMOTE}/${FDBREPO}"
|
||||
if [[ "${linkpath}" == "${fdbdir}" ]] ; then
|
||||
# Downloaded directly into go path. Skip making the link.
|
||||
:
|
||||
elif [[ -e "${linkpath}" ]] ; then
|
||||
echo "Warning: link path (${linkpath}) already exists. Leaving in place."
|
||||
else
|
||||
dirpath=$(dirname "${linkpath}")
|
||||
if [[ ! -d "${dirpath}" ]] ; then
|
||||
cmd=( 'mkdir' '-p' "${dirpath}" )
|
||||
echo "${cmd[*]}"
|
||||
if ! "${cmd[@]}" ; then
|
||||
let status="${status} + 1"
|
||||
echo "Could not create directory for link."
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "${status}" -eq 0 ]] ; then
|
||||
cmd=( 'ln' '-s' "${fdbdir}" "${linkpath}" )
|
||||
echo "${cmd[*]}"
|
||||
if ! "${cmd[@]}" ; then
|
||||
let status="${status} + 1"
|
||||
echo "Could not create link within go path."
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Step 4: Build the binaries.
|
||||
|
||||
if [[ "${operation}" == "download" ]] ; then
|
||||
# Do not install if only downloading
|
||||
:
|
||||
elif [[ "${status}" -eq 0 ]] ; then
|
||||
cgo_cppflags="-I${linkpath}/bindings/c"
|
||||
cgo_cflags="-g -O2"
|
||||
cgo_ldflags="-L${FDBLIBDIR}"
|
||||
fdb_go_path="${REMOTE}/${FDBREPO}/bindings/go/src"
|
||||
|
||||
if ! CGO_CPPFLAGS="${cgo_cppflags}" CGO_CFLAGS="${cgo_cflags}" CGO_LDFLAGS="${cgo_ldflags}" go install "${fdb_go_path}/fdb" "${fdb_go_path}/fdb/tuple" "${fdb_go_path}/fdb/subspace" "${fdb_go_path}/fdb/directory" ; then
|
||||
let status="${status} + 1"
|
||||
echo "Could not build FoundationDB go libraries."
|
||||
fi
|
||||
fi
|
||||
|
||||
# Step 5: Explain CGO flags.
|
||||
|
||||
if [[ "${status}" -eq 0 && ("${operation}" == "localinstall" || "${operation}" == "install" ) ]] ; then
|
||||
echo
|
||||
echo "The FoundationDB go bindings were successfully installed."
|
||||
echo "To build packages which use the go bindings, you will need to"
|
||||
echo "set the following environment variables:"
|
||||
echo " CGO_CPPFLAGS=\"${cgo_cppflags}\""
|
||||
echo " CGO_CFLAGS=\"${cgo_cflags}\""
|
||||
echo " CGO_LDFLAGS=\"${cgo_ldflags}\""
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
exit "${status}"
|
||||
|
|
@ -1,886 +0,0 @@
|
|||
// Copyright (c) HashiCorp, Inc.
|
||||
// SPDX-License-Identifier: BUSL-1.1
|
||||
|
||||
//go:build foundationdb
|
||||
|
||||
package foundationdb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
log "github.com/hashicorp/go-hclog"
|
||||
uuid "github.com/hashicorp/go-uuid"
|
||||
|
||||
"github.com/apple/foundationdb/bindings/go/src/fdb"
|
||||
"github.com/apple/foundationdb/bindings/go/src/fdb/directory"
|
||||
"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
|
||||
"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
|
||||
|
||||
metrics "github.com/armon/go-metrics"
|
||||
"github.com/hashicorp/vault/sdk/physical"
|
||||
)
|
||||
|
||||
const (
|
||||
// The minimum acceptable API version
|
||||
minAPIVersion = 520
|
||||
|
||||
// The namespace under our top directory containing keys only for list operations
|
||||
metaKeysNamespace = "_meta-keys"
|
||||
|
||||
// The namespace under our top directory containing the actual data
|
||||
dataNamespace = "_data"
|
||||
|
||||
// The namespace under our top directory containing locks
|
||||
lockNamespace = "_lock"
|
||||
|
||||
// Path hierarchy markers
|
||||
// - an entry in a directory (included in list)
|
||||
dirEntryMarker = "/\x01"
|
||||
// - a path component (excluded from list)
|
||||
dirPathMarker = "/\x02"
|
||||
)
|
||||
|
||||
var (
|
||||
// 64bit 1 and -1 for FDB atomic Add()
|
||||
atomicArgOne = []byte{0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
|
||||
atomicArgMinusOne = []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
|
||||
)
|
||||
|
||||
// Verify FDBBackend satisfies the correct interfaces
|
||||
var (
|
||||
_ physical.Backend = (*FDBBackend)(nil)
|
||||
_ physical.Transactional = (*FDBBackend)(nil)
|
||||
_ physical.HABackend = (*FDBBackend)(nil)
|
||||
_ physical.Lock = (*FDBBackendLock)(nil)
|
||||
)
|
||||
|
||||
// FDBBackend is a physical backend that stores data at a specific
|
||||
// prefix within FoundationDB.
|
||||
type FDBBackend struct {
|
||||
logger log.Logger
|
||||
haEnabled bool
|
||||
db fdb.Database
|
||||
metaKeysSpace subspace.Subspace
|
||||
dataSpace subspace.Subspace
|
||||
lockSpace subspace.Subspace
|
||||
instanceUUID string
|
||||
}
|
||||
|
||||
func concat(a []byte, b ...byte) []byte {
|
||||
r := make([]byte, len(a)+len(b))
|
||||
|
||||
copy(r, a)
|
||||
copy(r[len(a):], b)
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
func decoratePrefix(prefix string) ([]byte, error) {
|
||||
pathElements := strings.Split(prefix, "/")
|
||||
decoratedPrefix := strings.Join(pathElements[:len(pathElements)-1], dirPathMarker)
|
||||
|
||||
return []byte(decoratedPrefix + dirEntryMarker), nil
|
||||
}
|
||||
|
||||
// Turn a path string into a decorated byte array to be used as (part of) a key
|
||||
// foo /\x01foo
|
||||
// foo/ /\x01foo/
|
||||
// foo/bar /\x02foo/\x01bar
|
||||
// foo/bar/ /\x02foo/\x01bar/
|
||||
// foo/bar/baz /\x02foo/\x02bar/\x01baz
|
||||
// foo/bar/baz/ /\x02foo/\x02bar/\x01baz/
|
||||
// foo/bar/baz/quux /\x02foo/\x02bar/\x02baz/\x01quux
|
||||
// This allows for range queries to retrieve the "directory" listing. The
|
||||
// decoratePrefix() function builds the path leading up to the leaf.
|
||||
func decoratePath(path string) ([]byte, error) {
|
||||
if path == "" {
|
||||
return nil, fmt.Errorf("Invalid empty path")
|
||||
}
|
||||
|
||||
path = "/" + path
|
||||
|
||||
isDir := strings.HasSuffix(path, "/")
|
||||
path = strings.TrimRight(path, "/")
|
||||
|
||||
lastSlash := strings.LastIndexByte(path, '/')
|
||||
decoratedPrefix, err := decoratePrefix(path[:lastSlash+1])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
leaf := path[lastSlash+1:]
|
||||
if isDir {
|
||||
leaf += "/"
|
||||
}
|
||||
|
||||
return concat(decoratedPrefix, []byte(leaf)...), nil
|
||||
}
|
||||
|
||||
// Turn a decorated byte array back into a path string
|
||||
func undecoratePath(decoratedPath []byte) string {
|
||||
ret := strings.ReplaceAll(string(decoratedPath), dirPathMarker, "/")
|
||||
ret = strings.ReplaceAll(ret, dirEntryMarker, "/")
|
||||
|
||||
return strings.TrimLeft(ret, "/")
|
||||
}
|
||||
|
||||
// NewFDBBackend constructs a FoundationDB backend storing keys in the
|
||||
// top-level directory designated by path
|
||||
func NewFDBBackend(conf map[string]string, logger log.Logger) (physical.Backend, error) {
|
||||
// Get the top-level directory name
|
||||
path, ok := conf["path"]
|
||||
if !ok {
|
||||
path = "vault"
|
||||
}
|
||||
logger.Debug("config path set", "path", path)
|
||||
|
||||
dirPath := strings.Split(strings.Trim(path, "/"), "/")
|
||||
|
||||
// TLS support
|
||||
tlsCertFile, hasCertFile := conf["tls_cert_file"]
|
||||
tlsKeyFile, hasKeyFile := conf["tls_key_file"]
|
||||
tlsCAFile, hasCAFile := conf["tls_ca_file"]
|
||||
|
||||
tlsEnabled := hasCertFile && hasKeyFile && hasCAFile
|
||||
|
||||
if (hasCertFile || hasKeyFile || hasCAFile) && !tlsEnabled {
|
||||
return nil, fmt.Errorf("FoundationDB TLS requires all 3 of tls_cert_file, tls_key_file, and tls_ca_file")
|
||||
}
|
||||
|
||||
tlsVerifyPeers, ok := conf["tls_verify_peers"]
|
||||
if !ok && tlsEnabled {
|
||||
return nil, fmt.Errorf("Required option tls_verify_peers not set in configuration")
|
||||
}
|
||||
|
||||
// FoundationDB API version
|
||||
fdbApiVersionStr, ok := conf["api_version"]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("FoundationDB API version not specified")
|
||||
}
|
||||
|
||||
fdbApiVersionInt, err := strconv.Atoi(fdbApiVersionStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse fdb_api_version parameter: %w", err)
|
||||
}
|
||||
|
||||
// Check requested FDB API version against minimum required API version
|
||||
if fdbApiVersionInt < minAPIVersion {
|
||||
return nil, fmt.Errorf("Configured FoundationDB API version lower than minimum required version: %d < %d", fdbApiVersionInt, minAPIVersion)
|
||||
}
|
||||
|
||||
logger.Debug("FoundationDB API version set", "fdb_api_version", fdbApiVersionInt)
|
||||
|
||||
// FoundationDB cluster file
|
||||
fdbClusterFile, ok := conf["cluster_file"]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("FoundationDB cluster file not specified")
|
||||
}
|
||||
|
||||
haEnabled := false
|
||||
haEnabledStr, ok := conf["ha_enabled"]
|
||||
if ok {
|
||||
haEnabled, err = strconv.ParseBool(haEnabledStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse ha_enabled parameter: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
instanceUUID, err := uuid.GenerateUUID()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not generate instance UUID: %w", err)
|
||||
}
|
||||
logger.Debug("Instance UUID", "uuid", instanceUUID)
|
||||
|
||||
if err := fdb.APIVersion(fdbApiVersionInt); err != nil {
|
||||
return nil, fmt.Errorf("failed to set FDB API version: %w", err)
|
||||
}
|
||||
|
||||
if tlsEnabled {
|
||||
opts := fdb.Options()
|
||||
|
||||
tlsPassword, ok := conf["tls_password"]
|
||||
if ok {
|
||||
err := opts.SetTLSPassword(tlsPassword)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to set TLS password: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
err := opts.SetTLSCaPath(tlsCAFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to set TLS CA bundle path: %w", err)
|
||||
}
|
||||
|
||||
err = opts.SetTLSCertPath(tlsCertFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to set TLS certificate path: %w", err)
|
||||
}
|
||||
|
||||
err = opts.SetTLSKeyPath(tlsKeyFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to set TLS key path: %w", err)
|
||||
}
|
||||
|
||||
err = opts.SetTLSVerifyPeers([]byte(tlsVerifyPeers))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to set TLS peer verification criteria: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
db, err := fdb.Open(fdbClusterFile, []byte("DB"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open database with cluster file %q: %w", fdbClusterFile, err)
|
||||
}
|
||||
|
||||
topDir, err := directory.CreateOrOpen(db, dirPath, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create/open top-level directory %q: %w", path, err)
|
||||
}
|
||||
|
||||
// Setup the backend
|
||||
f := &FDBBackend{
|
||||
logger: logger,
|
||||
haEnabled: haEnabled,
|
||||
db: db,
|
||||
metaKeysSpace: topDir.Sub(metaKeysNamespace),
|
||||
dataSpace: topDir.Sub(dataNamespace),
|
||||
lockSpace: topDir.Sub(lockNamespace),
|
||||
instanceUUID: instanceUUID,
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Increase refcount on directories in the path, from the bottom -> up
|
||||
func (f *FDBBackend) incDirsRefcount(tr fdb.Transaction, path string) error {
|
||||
pathElements := strings.Split(strings.TrimRight(path, "/"), "/")
|
||||
|
||||
for i := len(pathElements) - 1; i != 0; i-- {
|
||||
dPath, err := decoratePath(strings.Join(pathElements[:i], "/") + "/")
|
||||
if err != nil {
|
||||
return fmt.Errorf("error incrementing directories refcount: %w", err)
|
||||
}
|
||||
|
||||
// Atomic +1
|
||||
tr.Add(fdb.Key(concat(f.metaKeysSpace.Bytes(), dPath...)), atomicArgOne)
|
||||
tr.Add(fdb.Key(concat(f.dataSpace.Bytes(), dPath...)), atomicArgOne)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type DirsDecTodo struct {
|
||||
fkey fdb.Key
|
||||
future fdb.FutureByteSlice
|
||||
}
|
||||
|
||||
// Decrease refcount on directories in the path, from the bottom -> up, and remove empty ones
|
||||
func (f *FDBBackend) decDirsRefcount(tr fdb.Transaction, path string) error {
|
||||
pathElements := strings.Split(strings.TrimRight(path, "/"), "/")
|
||||
|
||||
dirsTodo := make([]DirsDecTodo, 0, len(pathElements)*2)
|
||||
|
||||
for i := len(pathElements) - 1; i != 0; i-- {
|
||||
dPath, err := decoratePath(strings.Join(pathElements[:i], "/") + "/")
|
||||
if err != nil {
|
||||
return fmt.Errorf("error decrementing directories refcount: %w", err)
|
||||
}
|
||||
|
||||
metaFKey := fdb.Key(concat(f.metaKeysSpace.Bytes(), dPath...))
|
||||
dirsTodo = append(dirsTodo, DirsDecTodo{
|
||||
fkey: metaFKey,
|
||||
future: tr.Get(metaFKey),
|
||||
})
|
||||
|
||||
dataFKey := fdb.Key(concat(f.dataSpace.Bytes(), dPath...))
|
||||
dirsTodo = append(dirsTodo, DirsDecTodo{
|
||||
fkey: dataFKey,
|
||||
future: tr.Get(dataFKey),
|
||||
})
|
||||
}
|
||||
|
||||
for _, todo := range dirsTodo {
|
||||
value, err := todo.future.Get()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error getting directory refcount while decrementing: %w", err)
|
||||
}
|
||||
|
||||
// The directory entry does not exist; this is not expected
|
||||
if value == nil {
|
||||
return fmt.Errorf("non-existent directory while decrementing directory refcount")
|
||||
}
|
||||
|
||||
var count int64
|
||||
err = binary.Read(bytes.NewReader(value), binary.LittleEndian, &count)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reading directory refcount while decrementing: %w", err)
|
||||
}
|
||||
|
||||
if count > 1 {
|
||||
// Atomic -1
|
||||
tr.Add(todo.fkey, atomicArgMinusOne)
|
||||
} else {
|
||||
// Directory is empty, remove it
|
||||
tr.Clear(todo.fkey)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *FDBBackend) internalPut(tr fdb.Transaction, decoratedPath []byte, path string, value []byte) error {
|
||||
// Check that the meta key exists before blindly increasing the refcounts
|
||||
// in the directory hierarchy; this protects against commit_unknown_result
|
||||
// and other similar cases where a previous transaction may have gone
|
||||
// through without us knowing for sure.
|
||||
|
||||
metaFKey := fdb.Key(concat(f.metaKeysSpace.Bytes(), decoratedPath...))
|
||||
metaFuture := tr.Get(metaFKey)
|
||||
|
||||
dataFKey := fdb.Key(concat(f.dataSpace.Bytes(), decoratedPath...))
|
||||
tr.Set(dataFKey, value)
|
||||
|
||||
value, err := metaFuture.Get()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Put error while getting meta key: %w", err)
|
||||
}
|
||||
|
||||
if value == nil {
|
||||
tr.Set(metaFKey, []byte{})
|
||||
return f.incDirsRefcount(tr, path)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *FDBBackend) internalClear(tr fdb.Transaction, decoratedPath []byte, path string) error {
|
||||
// Same as above - check existence of the meta key before taking any
|
||||
// action, to protect against a possible previous commit_unknown_result
|
||||
// error.
|
||||
|
||||
metaFKey := fdb.Key(concat(f.metaKeysSpace.Bytes(), decoratedPath...))
|
||||
|
||||
value, err := tr.Get(metaFKey).Get()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Delete error while getting meta key: %w", err)
|
||||
}
|
||||
|
||||
if value != nil {
|
||||
dataFKey := fdb.Key(concat(f.dataSpace.Bytes(), decoratedPath...))
|
||||
tr.Clear(dataFKey)
|
||||
tr.Clear(metaFKey)
|
||||
return f.decDirsRefcount(tr, path)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type TxnTodo struct {
|
||||
decoratedPath []byte
|
||||
op *physical.TxnEntry
|
||||
}
|
||||
|
||||
// Used to run multiple entries via a transaction
|
||||
func (f *FDBBackend) Transaction(ctx context.Context, txns []*physical.TxnEntry) error {
|
||||
if len(txns) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
todo := make([]*TxnTodo, len(txns))
|
||||
|
||||
for i, op := range txns {
|
||||
if op.Operation != physical.DeleteOperation && op.Operation != physical.PutOperation {
|
||||
return fmt.Errorf("%q is not a supported transaction operation", op.Operation)
|
||||
}
|
||||
|
||||
decoratedPath, err := decoratePath(op.Entry.Key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not build decorated path for transaction item %s: %w", op.Entry.Key, err)
|
||||
}
|
||||
|
||||
todo[i] = &TxnTodo{
|
||||
decoratedPath: decoratedPath,
|
||||
op: op,
|
||||
}
|
||||
}
|
||||
|
||||
_, err := f.db.Transact(func(tr fdb.Transaction) (interface{}, error) {
|
||||
for _, txnTodo := range todo {
|
||||
var err error
|
||||
switch txnTodo.op.Operation {
|
||||
case physical.DeleteOperation:
|
||||
err = f.internalClear(tr, txnTodo.decoratedPath, txnTodo.op.Entry.Key)
|
||||
case physical.PutOperation:
|
||||
err = f.internalPut(tr, txnTodo.decoratedPath, txnTodo.op.Entry.Key, txnTodo.op.Entry.Value)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("operation %s failed for transaction item %s: %w", txnTodo.op.Operation, txnTodo.op.Entry.Key, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("transaction failed: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Put is used to insert or update an entry
|
||||
func (f *FDBBackend) Put(ctx context.Context, entry *physical.Entry) error {
|
||||
defer metrics.MeasureSince([]string{"foundationdb", "put"}, time.Now())
|
||||
|
||||
decoratedPath, err := decoratePath(entry.Key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not build decorated path to put item %s: %w", entry.Key, err)
|
||||
}
|
||||
|
||||
_, err = f.db.Transact(func(tr fdb.Transaction) (interface{}, error) {
|
||||
err := f.internalPut(tr, decoratedPath, entry.Key, entry.Value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("put failed for item %s: %w", entry.Key, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get is used to fetch an entry
|
||||
// Return nil for non-existent keys
|
||||
func (f *FDBBackend) Get(ctx context.Context, key string) (*physical.Entry, error) {
|
||||
defer metrics.MeasureSince([]string{"foundationdb", "get"}, time.Now())
|
||||
|
||||
decoratedPath, err := decoratePath(key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not build decorated path to get item %s: %w", key, err)
|
||||
}
|
||||
|
||||
fkey := fdb.Key(concat(f.dataSpace.Bytes(), decoratedPath...))
|
||||
|
||||
value, err := f.db.ReadTransact(func(rtr fdb.ReadTransaction) (interface{}, error) {
|
||||
value, err := rtr.Get(fkey).Get()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return value, nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get failed for item %s: %w", key, err)
|
||||
}
|
||||
if value.([]byte) == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return &physical.Entry{
|
||||
Key: key,
|
||||
Value: value.([]byte),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Delete is used to permanently delete an entry
|
||||
func (f *FDBBackend) Delete(ctx context.Context, key string) error {
|
||||
defer metrics.MeasureSince([]string{"foundationdb", "delete"}, time.Now())
|
||||
|
||||
decoratedPath, err := decoratePath(key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not build decorated path to delete item %s: %w", key, err)
|
||||
}
|
||||
|
||||
_, err = f.db.Transact(func(tr fdb.Transaction) (interface{}, error) {
|
||||
err := f.internalClear(tr, decoratedPath, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete failed for item %s: %w", key, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// List is used to list all the keys under a given
|
||||
// prefix, up to the next prefix.
|
||||
// Return empty string slice for non-existent directories
|
||||
func (f *FDBBackend) List(ctx context.Context, prefix string) ([]string, error) {
|
||||
defer metrics.MeasureSince([]string{"foundationdb", "list"}, time.Now())
|
||||
|
||||
prefix = strings.TrimRight("/"+prefix, "/") + "/"
|
||||
|
||||
decoratedPrefix, err := decoratePrefix(prefix)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not build decorated path to list prefix %s: %w", prefix, err)
|
||||
}
|
||||
|
||||
// The beginning of the range is /\x02foo/\x02bar/\x01 (the decorated prefix) to list foo/bar/
|
||||
rangeBegin := fdb.Key(concat(f.metaKeysSpace.Bytes(), decoratedPrefix...))
|
||||
rangeEnd := fdb.Key(concat(rangeBegin, 0xff))
|
||||
pathRange := fdb.KeyRange{rangeBegin, rangeEnd}
|
||||
keyPrefixLen := len(rangeBegin)
|
||||
|
||||
content, err := f.db.ReadTransact(func(rtr fdb.ReadTransaction) (interface{}, error) {
|
||||
dirList := make([]string, 0, 0)
|
||||
|
||||
ri := rtr.GetRange(pathRange, fdb.RangeOptions{Mode: fdb.StreamingModeWantAll}).Iterator()
|
||||
|
||||
for ri.Advance() {
|
||||
kv := ri.MustGet()
|
||||
|
||||
// Strip length of the rangeBegin key off the FDB key, yielding
|
||||
// the part of the key we're interested in, which does not need
|
||||
// to be undecorated, by construction.
|
||||
dirList = append(dirList, string(kv.Key[keyPrefixLen:]))
|
||||
}
|
||||
|
||||
return dirList, nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not list prefix %s: %w", prefix, err)
|
||||
}
|
||||
|
||||
return content.([]string), nil
|
||||
}
|
||||
|
||||
type FDBBackendLock struct {
|
||||
f *FDBBackend
|
||||
key string
|
||||
value string
|
||||
fkey fdb.Key
|
||||
lock sync.Mutex
|
||||
}
|
||||
|
||||
// LockWith is used for mutual exclusion based on the given key.
|
||||
func (f *FDBBackend) LockWith(key, value string) (physical.Lock, error) {
|
||||
return &FDBBackendLock{
|
||||
f: f,
|
||||
key: key,
|
||||
value: value,
|
||||
fkey: f.lockSpace.Pack(tuple.Tuple{key}),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (f *FDBBackend) HAEnabled() bool {
|
||||
return f.haEnabled
|
||||
}
|
||||
|
||||
const (
|
||||
// Position of elements in the lock content tuple
|
||||
lockContentValueIdx = 0
|
||||
lockContentOwnerIdx = 1
|
||||
lockContentExpiresIdx = 2
|
||||
|
||||
// Number of elements in the lock content tuple
|
||||
lockTupleContentElts = 3
|
||||
|
||||
lockTTL = 15 * time.Second
|
||||
lockRenewInterval = 5 * time.Second
|
||||
lockAcquireRetryInterval = 5 * time.Second
|
||||
)
|
||||
|
||||
type FDBBackendLockContent struct {
|
||||
value string
|
||||
ownerUUID string
|
||||
expires time.Time
|
||||
}
|
||||
|
||||
func packLock(content *FDBBackendLockContent) []byte {
|
||||
t := tuple.Tuple{content.value, content.ownerUUID, content.expires.UnixNano()}
|
||||
|
||||
return t.Pack()
|
||||
}
|
||||
|
||||
func unpackLock(tupleContent []byte) (*FDBBackendLockContent, error) {
|
||||
t, err := tuple.Unpack(tupleContent)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(t) != lockTupleContentElts {
|
||||
return nil, fmt.Errorf("unexpected lock content, len %d != %d", len(t), lockTupleContentElts)
|
||||
}
|
||||
|
||||
return &FDBBackendLockContent{
|
||||
value: t[lockContentValueIdx].(string),
|
||||
ownerUUID: t[lockContentOwnerIdx].(string),
|
||||
expires: time.Unix(0, t[lockContentExpiresIdx].(int64)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (fl *FDBBackendLock) getLockContent(tr fdb.Transaction) (*FDBBackendLockContent, error) {
|
||||
tupleContent, err := tr.Get(fl.fkey).Get()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Lock doesn't exist
|
||||
if tupleContent == nil {
|
||||
return nil, fmt.Errorf("non-existent lock %s", fl.key)
|
||||
}
|
||||
|
||||
content, err := unpackLock(tupleContent)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to unpack lock %s: %w", fl.key, err)
|
||||
}
|
||||
|
||||
return content, nil
|
||||
}
|
||||
|
||||
func (fl *FDBBackendLock) setLockContent(tr fdb.Transaction, content *FDBBackendLockContent) {
|
||||
tr.Set(fl.fkey, packLock(content))
|
||||
}
|
||||
|
||||
func (fl *FDBBackendLock) isOwned(content *FDBBackendLockContent) bool {
|
||||
return content.ownerUUID == fl.f.instanceUUID
|
||||
}
|
||||
|
||||
func (fl *FDBBackendLock) isExpired(content *FDBBackendLockContent) bool {
|
||||
return time.Now().After(content.expires)
|
||||
}
|
||||
|
||||
func (fl *FDBBackendLock) acquireTryLock(acquired chan struct{}, errors chan error) (bool, error) {
|
||||
wonTheRace, err := fl.f.db.Transact(func(tr fdb.Transaction) (interface{}, error) {
|
||||
tupleContent, err := tr.Get(fl.fkey).Get()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not read lock: %w", err)
|
||||
}
|
||||
|
||||
// Lock exists
|
||||
if tupleContent != nil {
|
||||
content, err := unpackLock(tupleContent)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to unpack lock %s: %w", fl.key, err)
|
||||
}
|
||||
|
||||
if fl.isOwned(content) {
|
||||
return nil, fmt.Errorf("lock %s already held", fl.key)
|
||||
}
|
||||
|
||||
// The lock already exists, is not owned by us, and is not expired
|
||||
if !fl.isExpired(content) {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Lock doesn't exist, or exists but is expired, we can go ahead
|
||||
content := &FDBBackendLockContent{
|
||||
value: fl.value,
|
||||
ownerUUID: fl.f.instanceUUID,
|
||||
expires: time.Now().Add(lockTTL),
|
||||
}
|
||||
|
||||
fl.setLockContent(tr, content)
|
||||
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
errors <- err
|
||||
return false, err
|
||||
}
|
||||
|
||||
if wonTheRace.(bool) {
|
||||
close(acquired)
|
||||
}
|
||||
|
||||
return wonTheRace.(bool), nil
|
||||
}
|
||||
|
||||
func (fl *FDBBackendLock) acquireLock(abandon chan struct{}, acquired chan struct{}, errors chan error) {
|
||||
ticker := time.NewTicker(lockAcquireRetryInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
lockAcquired, err := fl.acquireTryLock(acquired, errors)
|
||||
if lockAcquired || err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-abandon:
|
||||
return
|
||||
case <-ticker.C:
|
||||
lockAcquired, err := fl.acquireTryLock(acquired, errors)
|
||||
if lockAcquired || err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (fl *FDBBackendLock) maintainLock(lost <-chan struct{}) {
|
||||
ticker := time.NewTicker(lockRenewInterval)
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
_, err := fl.f.db.Transact(func(tr fdb.Transaction) (interface{}, error) {
|
||||
content, err := fl.getLockContent(tr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// We don't own the lock
|
||||
if !fl.isOwned(content) {
|
||||
return nil, fmt.Errorf("lost lock %s", fl.key)
|
||||
}
|
||||
|
||||
// The lock is expired
|
||||
if fl.isExpired(content) {
|
||||
return nil, fmt.Errorf("lock %s expired", fl.key)
|
||||
}
|
||||
|
||||
content.expires = time.Now().Add(lockTTL)
|
||||
|
||||
fl.setLockContent(tr, content)
|
||||
|
||||
return nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
fl.f.logger.Error("lock maintain", "error", err)
|
||||
}
|
||||
|
||||
// Failure to renew the lock will cause another node to take over
|
||||
// and the watch to fire. DB errors will also be caught by the watch.
|
||||
case <-lost:
|
||||
ticker.Stop()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (fl *FDBBackendLock) watchLock(lost chan struct{}) {
|
||||
for {
|
||||
watch, err := fl.f.db.Transact(func(tr fdb.Transaction) (interface{}, error) {
|
||||
content, err := fl.getLockContent(tr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// We don't own the lock
|
||||
if !fl.isOwned(content) {
|
||||
return nil, fmt.Errorf("lost lock %s", fl.key)
|
||||
}
|
||||
|
||||
// The lock is expired
|
||||
if fl.isExpired(content) {
|
||||
return nil, fmt.Errorf("lock %s expired", fl.key)
|
||||
}
|
||||
|
||||
// Set FDB watch on the lock
|
||||
future := tr.Watch(fl.fkey)
|
||||
|
||||
return future, nil
|
||||
})
|
||||
if err != nil {
|
||||
fl.f.logger.Error("lock watch", "error", err)
|
||||
break
|
||||
}
|
||||
|
||||
// Wait for the watch to fire, and go again
|
||||
watch.(fdb.FutureNil).Get()
|
||||
}
|
||||
|
||||
close(lost)
|
||||
}
|
||||
|
||||
func (fl *FDBBackendLock) Lock(stopCh <-chan struct{}) (<-chan struct{}, error) {
|
||||
fl.lock.Lock()
|
||||
defer fl.lock.Unlock()
|
||||
|
||||
var (
|
||||
// Inform the lock owner that we lost the lock
|
||||
lost = make(chan struct{})
|
||||
|
||||
// Tell our watch and renewal routines the lock has been abandoned
|
||||
abandon = make(chan struct{})
|
||||
|
||||
// Feedback from lock acquisition routine
|
||||
acquired = make(chan struct{})
|
||||
errors = make(chan error)
|
||||
)
|
||||
|
||||
// try to acquire the lock asynchronously
|
||||
go fl.acquireLock(abandon, acquired, errors)
|
||||
|
||||
select {
|
||||
case <-acquired:
|
||||
// Maintain the lock after initial acquisition
|
||||
go fl.maintainLock(lost)
|
||||
// Watch the lock for changes
|
||||
go fl.watchLock(lost)
|
||||
case err := <-errors:
|
||||
// Initial acquisition failed
|
||||
close(abandon)
|
||||
return nil, err
|
||||
case <-stopCh:
|
||||
// Prospective lock owner cancelling lock acquisition
|
||||
close(abandon)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return lost, nil
|
||||
}
|
||||
|
||||
func (fl *FDBBackendLock) Unlock() error {
|
||||
fl.lock.Lock()
|
||||
defer fl.lock.Unlock()
|
||||
|
||||
_, err := fl.f.db.Transact(func(tr fdb.Transaction) (interface{}, error) {
|
||||
content, err := fl.getLockContent(tr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not get lock content: %w", err)
|
||||
}
|
||||
|
||||
// We don't own the lock
|
||||
if !fl.isOwned(content) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
tr.Clear(fl.fkey)
|
||||
|
||||
return nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("unlock failed: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fl *FDBBackendLock) Value() (bool, string, error) {
|
||||
tupleContent, err := fl.f.db.ReadTransact(func(rtr fdb.ReadTransaction) (interface{}, error) {
|
||||
tupleContent, err := rtr.Get(fl.fkey).Get()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not read lock: %w", err)
|
||||
}
|
||||
|
||||
return tupleContent, nil
|
||||
})
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("get lock value failed for lock %s: %w", fl.key, err)
|
||||
}
|
||||
if tupleContent.([]byte) == nil {
|
||||
return false, "", nil
|
||||
}
|
||||
|
||||
content, err := unpackLock(tupleContent.([]byte))
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("get lock value failed to unpack lock %s: %w", fl.key, err)
|
||||
}
|
||||
|
||||
return true, content.value, nil
|
||||
}
|
||||
|
|
@ -1,199 +0,0 @@
|
|||
// Copyright (c) HashiCorp, Inc.
|
||||
// SPDX-License-Identifier: BUSL-1.1
|
||||
|
||||
//go:build foundationdb
|
||||
|
||||
package foundationdb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
log "github.com/hashicorp/go-hclog"
|
||||
uuid "github.com/hashicorp/go-uuid"
|
||||
|
||||
"github.com/apple/foundationdb/bindings/go/src/fdb"
|
||||
"github.com/apple/foundationdb/bindings/go/src/fdb/directory"
|
||||
|
||||
"github.com/hashicorp/vault/sdk/helper/logging"
|
||||
"github.com/hashicorp/vault/sdk/physical"
|
||||
|
||||
dockertest "gopkg.in/ory-am/dockertest.v3"
|
||||
)
|
||||
|
||||
func connectToFoundationDB(clusterFile string) (*fdb.Database, error) {
|
||||
if err := fdb.APIVersion(520); err != nil {
|
||||
return nil, fmt.Errorf("failed to set FDB API version: %w", err)
|
||||
}
|
||||
|
||||
db, err := fdb.Open(clusterFile, []byte("DB"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open database: %w", err)
|
||||
}
|
||||
|
||||
return &db, nil
|
||||
}
|
||||
|
||||
func cleanupTopDir(clusterFile, topDir string) error {
|
||||
db, err := connectToFoundationDB(clusterFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not connect to FDB for cleanup: %w", err)
|
||||
}
|
||||
|
||||
if _, err := directory.Root().Remove(db, []string{topDir}); err != nil {
|
||||
return fmt.Errorf("could not remove directory: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestFoundationDBPathDecoration(t *testing.T) {
|
||||
cases := map[string][]byte{
|
||||
"foo": []byte("/\x01foo"),
|
||||
"foo/": []byte("/\x01foo/"),
|
||||
"foo/bar": []byte("/\x02foo/\x01bar"),
|
||||
"foo/bar/": []byte("/\x02foo/\x01bar/"),
|
||||
"foo/bar/baz": []byte("/\x02foo/\x02bar/\x01baz"),
|
||||
"foo/bar/baz/": []byte("/\x02foo/\x02bar/\x01baz/"),
|
||||
"foo/bar/baz/quux": []byte("/\x02foo/\x02bar/\x02baz/\x01quux"),
|
||||
}
|
||||
|
||||
for path, expected := range cases {
|
||||
decorated, err := decoratePath(path)
|
||||
if err != nil {
|
||||
t.Fatalf("path %s error: %s", path, err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(expected, decorated) {
|
||||
t.Fatalf("path %s expected %v got %v", path, expected, decorated)
|
||||
}
|
||||
|
||||
undecorated := undecoratePath(decorated)
|
||||
if undecorated != path {
|
||||
t.Fatalf("expected %s got %s", path, undecorated)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFoundationDBBackend(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skipf("skipping in short mode")
|
||||
}
|
||||
|
||||
testUUID, err := uuid.GenerateUUID()
|
||||
if err != nil {
|
||||
t.Fatalf("foundationdb: could not generate UUID to top-level directory: %s", err)
|
||||
}
|
||||
|
||||
topDir := fmt.Sprintf("vault-test-%s", testUUID)
|
||||
|
||||
var clusterFile string
|
||||
clusterFile = os.Getenv("FOUNDATIONDB_CLUSTER_FILE")
|
||||
if clusterFile == "" {
|
||||
var cleanup func()
|
||||
cleanup, clusterFile = prepareFoundationDBTestDirectory(t, topDir)
|
||||
defer cleanup()
|
||||
}
|
||||
|
||||
// Remove the test data once done
|
||||
defer func() {
|
||||
if err := cleanupTopDir(clusterFile, topDir); err != nil {
|
||||
t.Fatalf("foundationdb: could not cleanup test data at end of test: %s", err)
|
||||
}
|
||||
}()
|
||||
|
||||
// Remove any leftover test data before starting
|
||||
if err := cleanupTopDir(clusterFile, topDir); err != nil {
|
||||
t.Fatalf("foundationdb: could not cleanup test data before starting test: %s", err)
|
||||
}
|
||||
|
||||
// Run vault tests
|
||||
logger := logging.NewVaultLogger(log.Debug)
|
||||
config := map[string]string{
|
||||
"path": topDir,
|
||||
"api_version": "520",
|
||||
"cluster_file": clusterFile,
|
||||
}
|
||||
|
||||
b, err := NewFDBBackend(config, logger)
|
||||
if err != nil {
|
||||
t.Fatalf("foundationdb: failed to create new backend: %s", err)
|
||||
}
|
||||
|
||||
b2, err := NewFDBBackend(config, logger)
|
||||
if err != nil {
|
||||
t.Fatalf("foundationdb: failed to create new backend: %s", err)
|
||||
}
|
||||
|
||||
physical.ExerciseBackend(t, b)
|
||||
physical.ExerciseBackend_ListPrefix(t, b)
|
||||
physical.ExerciseTransactionalBackend(t, b)
|
||||
physical.ExerciseHABackend(t, b.(physical.HABackend), b2.(physical.HABackend))
|
||||
}
|
||||
|
||||
func prepareFoundationDBTestDirectory(t *testing.T, topDir string) (func(), string) {
|
||||
pool, err := dockertest.NewPool("")
|
||||
if err != nil {
|
||||
t.Fatalf("foundationdb: failed to connect to docker: %s", err)
|
||||
}
|
||||
|
||||
resource, err := pool.Run("foundationdb", "5.1.7", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("foundationdb: could not start container: %s", err)
|
||||
}
|
||||
|
||||
tmpFile, err := ioutil.TempFile("", topDir)
|
||||
if err != nil {
|
||||
t.Fatalf("foundationdb: could not create temporary file for cluster file: %s", err)
|
||||
}
|
||||
|
||||
clusterFile := tmpFile.Name()
|
||||
|
||||
cleanup := func() {
|
||||
var err error
|
||||
for i := 0; i < 10; i++ {
|
||||
err = pool.Purge(resource)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
os.Remove(clusterFile)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to cleanup local container: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
setup := func() error {
|
||||
connectString := fmt.Sprintf("foundationdb:foundationdb@127.0.0.1:%s", resource.GetPort("4500/tcp"))
|
||||
|
||||
if err := tmpFile.Truncate(0); err != nil {
|
||||
return fmt.Errorf("could not truncate cluster file: %w", err)
|
||||
}
|
||||
|
||||
_, err := tmpFile.WriteAt([]byte(connectString), 0)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not write cluster file: %w", err)
|
||||
}
|
||||
|
||||
if _, err := connectToFoundationDB(clusterFile); err != nil {
|
||||
return fmt.Errorf("could not connect to FoundationDB after starting container: %s", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
if pool.Retry(setup); err != nil {
|
||||
cleanup()
|
||||
|
||||
t.Fatalf("foundationdb: could not setup container: %s", err)
|
||||
}
|
||||
|
||||
tmpFile.Close()
|
||||
|
||||
return cleanup, clusterFile
|
||||
}
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
// Copyright (c) HashiCorp, Inc.
|
||||
// SPDX-License-Identifier: BUSL-1.1
|
||||
|
||||
//go:build !foundationdb
|
||||
|
||||
package foundationdb
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
log "github.com/hashicorp/go-hclog"
|
||||
|
||||
"github.com/hashicorp/vault/sdk/physical"
|
||||
)
|
||||
|
||||
func NewFDBBackend(conf map[string]string, logger log.Logger) (physical.Backend, error) {
|
||||
return nil, fmt.Errorf("FoundationDB backend not available in this Vault build")
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user