remove manta (joyent)
This commit is contained in:
parent
faed4aa7af
commit
847f8ad8e7
@ -48,7 +48,6 @@ import (
|
|||||||
physCouchDB "github.com/hashicorp/vault/physical/couchdb"
|
physCouchDB "github.com/hashicorp/vault/physical/couchdb"
|
||||||
physEtcd "github.com/hashicorp/vault/physical/etcd"
|
physEtcd "github.com/hashicorp/vault/physical/etcd"
|
||||||
physFoundationDB "github.com/hashicorp/vault/physical/foundationdb"
|
physFoundationDB "github.com/hashicorp/vault/physical/foundationdb"
|
||||||
physManta "github.com/hashicorp/vault/physical/manta"
|
|
||||||
physMySQL "github.com/hashicorp/vault/physical/mysql"
|
physMySQL "github.com/hashicorp/vault/physical/mysql"
|
||||||
physOCI "github.com/hashicorp/vault/physical/oci"
|
physOCI "github.com/hashicorp/vault/physical/oci"
|
||||||
physPostgreSQL "github.com/hashicorp/vault/physical/postgresql"
|
physPostgreSQL "github.com/hashicorp/vault/physical/postgresql"
|
||||||
@ -189,7 +188,6 @@ var (
|
|||||||
"inmem_transactional_ha": physInmem.NewTransactionalInmemHA,
|
"inmem_transactional_ha": physInmem.NewTransactionalInmemHA,
|
||||||
"inmem_transactional": physInmem.NewTransactionalInmem,
|
"inmem_transactional": physInmem.NewTransactionalInmem,
|
||||||
"inmem": physInmem.NewInmem,
|
"inmem": physInmem.NewInmem,
|
||||||
"manta": physManta.NewMantaBackend,
|
|
||||||
"mysql": physMySQL.NewMySQLBackend,
|
"mysql": physMySQL.NewMySQLBackend,
|
||||||
"oci": physOCI.NewBackend,
|
"oci": physOCI.NewBackend,
|
||||||
"postgresql": physPostgreSQL.NewPostgreSQLBackend,
|
"postgresql": physPostgreSQL.NewPostgreSQLBackend,
|
||||||
|
2
go.mod
2
go.mod
@ -125,7 +125,6 @@ require (
|
|||||||
github.com/jcmturner/gokrb5/v8 v8.4.4
|
github.com/jcmturner/gokrb5/v8 v8.4.4
|
||||||
github.com/jefferai/isbadcipher v0.0.0-20190226160619-51d2077c035f
|
github.com/jefferai/isbadcipher v0.0.0-20190226160619-51d2077c035f
|
||||||
github.com/jefferai/jsonx v1.0.0
|
github.com/jefferai/jsonx v1.0.0
|
||||||
github.com/joyent/triton-go v1.7.1-0.20200416154420-6801d15b779f
|
|
||||||
github.com/klauspost/compress v1.16.7
|
github.com/klauspost/compress v1.16.7
|
||||||
github.com/kr/pretty v0.3.1
|
github.com/kr/pretty v0.3.1
|
||||||
github.com/kr/text v0.2.0
|
github.com/kr/text v0.2.0
|
||||||
@ -312,6 +311,7 @@ require (
|
|||||||
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
|
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
|
||||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||||
github.com/josharian/intern v1.0.0 // indirect
|
github.com/josharian/intern v1.0.0 // indirect
|
||||||
|
github.com/joyent/triton-go v1.7.1-0.20200416154420-6801d15b779f // indirect
|
||||||
github.com/json-iterator/go v1.1.12 // indirect
|
github.com/json-iterator/go v1.1.12 // indirect
|
||||||
github.com/kelseyhightower/envconfig v1.4.0 // indirect
|
github.com/kelseyhightower/envconfig v1.4.0 // indirect
|
||||||
github.com/klauspost/pgzip v1.2.5 // indirect
|
github.com/klauspost/pgzip v1.2.5 // indirect
|
||||||
|
@ -1,266 +0,0 @@
|
|||||||
// Copyright (c) HashiCorp, Inc.
|
|
||||||
// SPDX-License-Identifier: BUSL-1.1
|
|
||||||
|
|
||||||
package manta
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path"
|
|
||||||
"sort"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
metrics "github.com/armon/go-metrics"
|
|
||||||
log "github.com/hashicorp/go-hclog"
|
|
||||||
"github.com/hashicorp/vault/sdk/physical"
|
|
||||||
triton "github.com/joyent/triton-go"
|
|
||||||
"github.com/joyent/triton-go/authentication"
|
|
||||||
"github.com/joyent/triton-go/errors"
|
|
||||||
"github.com/joyent/triton-go/storage"
|
|
||||||
)
|
|
||||||
|
|
||||||
const mantaDefaultRootStore = "/stor"
|
|
||||||
|
|
||||||
type MantaBackend struct {
|
|
||||||
logger log.Logger
|
|
||||||
permitPool *physical.PermitPool
|
|
||||||
client *storage.StorageClient
|
|
||||||
directory string
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewMantaBackend(conf map[string]string, logger log.Logger) (physical.Backend, error) {
|
|
||||||
user := os.Getenv("MANTA_USER")
|
|
||||||
if user == "" {
|
|
||||||
user = conf["user"]
|
|
||||||
}
|
|
||||||
|
|
||||||
keyId := os.Getenv("MANTA_KEY_ID")
|
|
||||||
if keyId == "" {
|
|
||||||
keyId = conf["key_id"]
|
|
||||||
}
|
|
||||||
|
|
||||||
url := os.Getenv("MANTA_URL")
|
|
||||||
if url == "" {
|
|
||||||
url = conf["url"]
|
|
||||||
} else {
|
|
||||||
url = "https://us-east.manta.joyent.com"
|
|
||||||
}
|
|
||||||
|
|
||||||
subuser := os.Getenv("MANTA_SUBUSER")
|
|
||||||
if subuser == "" {
|
|
||||||
if confUser, ok := conf["subuser"]; ok {
|
|
||||||
subuser = confUser
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
input := authentication.SSHAgentSignerInput{
|
|
||||||
KeyID: keyId,
|
|
||||||
AccountName: user,
|
|
||||||
Username: subuser,
|
|
||||||
}
|
|
||||||
signer, err := authentication.NewSSHAgentSigner(input)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("Error Creating SSH Agent Signer: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
maxParStr, ok := conf["max_parallel"]
|
|
||||||
var maxParInt int
|
|
||||||
if ok {
|
|
||||||
maxParInt, err = strconv.Atoi(maxParStr)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed parsing max_parallel parameter: %w", err)
|
|
||||||
}
|
|
||||||
if logger.IsDebug() {
|
|
||||||
logger.Debug("max_parallel set", "max_parallel", maxParInt)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
config := &triton.ClientConfig{
|
|
||||||
MantaURL: url,
|
|
||||||
AccountName: user,
|
|
||||||
Signers: []authentication.Signer{signer},
|
|
||||||
}
|
|
||||||
|
|
||||||
client, err := storage.NewClient(config)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed initialising Storage client: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &MantaBackend{
|
|
||||||
client: client,
|
|
||||||
directory: conf["directory"],
|
|
||||||
logger: logger,
|
|
||||||
permitPool: physical.NewPermitPool(maxParInt),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Put is used to insert or update an entry
|
|
||||||
func (m *MantaBackend) Put(ctx context.Context, entry *physical.Entry) error {
|
|
||||||
defer metrics.MeasureSince([]string{"manta", "put"}, time.Now())
|
|
||||||
|
|
||||||
m.permitPool.Acquire()
|
|
||||||
defer m.permitPool.Release()
|
|
||||||
|
|
||||||
r := bytes.NewReader(entry.Value)
|
|
||||||
r.Seek(0, 0)
|
|
||||||
|
|
||||||
return m.client.Objects().Put(ctx, &storage.PutObjectInput{
|
|
||||||
ObjectPath: path.Join(mantaDefaultRootStore, m.directory, entry.Key, ".vault_value"),
|
|
||||||
ObjectReader: r,
|
|
||||||
ContentLength: uint64(len(entry.Value)),
|
|
||||||
ForceInsert: true,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get is used to fetch an entry
|
|
||||||
func (m *MantaBackend) Get(ctx context.Context, key string) (*physical.Entry, error) {
|
|
||||||
defer metrics.MeasureSince([]string{"manta", "get"}, time.Now())
|
|
||||||
|
|
||||||
m.permitPool.Acquire()
|
|
||||||
defer m.permitPool.Release()
|
|
||||||
|
|
||||||
output, err := m.client.Objects().Get(ctx, &storage.GetObjectInput{
|
|
||||||
ObjectPath: path.Join(mantaDefaultRootStore, m.directory, key, ".vault_value"),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
if strings.Contains(err.Error(), "ResourceNotFound") {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer output.ObjectReader.Close()
|
|
||||||
|
|
||||||
data := make([]byte, output.ContentLength)
|
|
||||||
_, err = io.ReadFull(output.ObjectReader, data)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
ent := &physical.Entry{
|
|
||||||
Key: key,
|
|
||||||
Value: data,
|
|
||||||
}
|
|
||||||
|
|
||||||
return ent, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Delete is used to permanently delete an entry
|
|
||||||
func (m *MantaBackend) Delete(ctx context.Context, key string) error {
|
|
||||||
defer metrics.MeasureSince([]string{"manta", "delete"}, time.Now())
|
|
||||||
|
|
||||||
m.permitPool.Acquire()
|
|
||||||
defer m.permitPool.Release()
|
|
||||||
|
|
||||||
if strings.HasSuffix(key, "/") {
|
|
||||||
err := m.client.Dir().Delete(ctx, &storage.DeleteDirectoryInput{
|
|
||||||
DirectoryName: path.Join(mantaDefaultRootStore, m.directory, key),
|
|
||||||
ForceDelete: true,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
err := m.client.Objects().Delete(ctx, &storage.DeleteObjectInput{
|
|
||||||
ObjectPath: path.Join(mantaDefaultRootStore, m.directory, key, ".vault_value"),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
if errors.IsResourceNotFound(err) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return tryDeleteDirectory(ctx, m, path.Join(mantaDefaultRootStore, m.directory, key))
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func tryDeleteDirectory(ctx context.Context, m *MantaBackend, directoryPath string) error {
|
|
||||||
objs, err := m.client.Dir().List(ctx, &storage.ListDirectoryInput{
|
|
||||||
DirectoryName: directoryPath,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
if errors.IsResourceNotFound(err) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if objs != nil && len(objs.Entries) == 0 {
|
|
||||||
err := m.client.Dir().Delete(ctx, &storage.DeleteDirectoryInput{
|
|
||||||
DirectoryName: directoryPath,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return tryDeleteDirectory(ctx, m, path.Dir(directoryPath))
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// List is used to list all the keys under a given
|
|
||||||
// prefix, up to the next prefix.
|
|
||||||
func (m *MantaBackend) List(ctx context.Context, prefix string) ([]string, error) {
|
|
||||||
defer metrics.MeasureSince([]string{"manta", "list"}, time.Now())
|
|
||||||
|
|
||||||
m.permitPool.Acquire()
|
|
||||||
defer m.permitPool.Release()
|
|
||||||
|
|
||||||
objs, err := m.client.Dir().List(ctx, &storage.ListDirectoryInput{
|
|
||||||
DirectoryName: path.Join(mantaDefaultRootStore, m.directory, prefix),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
if errors.IsResourceNotFound(err) {
|
|
||||||
return []string{}, nil
|
|
||||||
}
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
keys := []string{}
|
|
||||||
for _, obj := range objs.Entries {
|
|
||||||
if obj.Type == "directory" {
|
|
||||||
objs, err := m.client.Dir().List(ctx, &storage.ListDirectoryInput{
|
|
||||||
DirectoryName: path.Join(mantaDefaultRootStore, m.directory, prefix, obj.Name),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
if !errors.IsResourceNotFound(err) {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// We need to check to see if there is something more than just the `value` file
|
|
||||||
// if the length of the children is:
|
|
||||||
// > 1 and includes the value `index` then we need to add foo and foo/
|
|
||||||
// = 1 and the value is `index` then we need to add foo
|
|
||||||
// = 1 and the value is not `index` then we need to add foo/
|
|
||||||
if len(objs.Entries) == 1 {
|
|
||||||
if objs.Entries[0].Name != ".vault_value" {
|
|
||||||
keys = append(keys, fmt.Sprintf("%s/", obj.Name))
|
|
||||||
} else {
|
|
||||||
keys = append(keys, obj.Name)
|
|
||||||
}
|
|
||||||
} else if len(objs.Entries) > 1 {
|
|
||||||
for _, childObj := range objs.Entries {
|
|
||||||
if childObj.Name == ".vault_value" {
|
|
||||||
keys = append(keys, obj.Name)
|
|
||||||
} else {
|
|
||||||
keys = append(keys, fmt.Sprintf("%s/", obj.Name))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
keys = append(keys, obj.Name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
sort.Strings(keys)
|
|
||||||
|
|
||||||
return keys, nil
|
|
||||||
}
|
|
@ -1,89 +0,0 @@
|
|||||||
// Copyright (c) HashiCorp, Inc.
|
|
||||||
// SPDX-License-Identifier: BUSL-1.1
|
|
||||||
|
|
||||||
package manta
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"math/rand"
|
|
||||||
"os"
|
|
||||||
"path"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
log "github.com/hashicorp/go-hclog"
|
|
||||||
"github.com/hashicorp/vault/sdk/helper/logging"
|
|
||||||
"github.com/hashicorp/vault/sdk/physical"
|
|
||||||
triton "github.com/joyent/triton-go"
|
|
||||||
"github.com/joyent/triton-go/authentication"
|
|
||||||
tt "github.com/joyent/triton-go/errors"
|
|
||||||
"github.com/joyent/triton-go/storage"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestMantaBackend(t *testing.T) {
|
|
||||||
user := os.Getenv("MANTA_USER")
|
|
||||||
keyId := os.Getenv("MANTA_KEY_ID")
|
|
||||||
url := "https://us-east.manta.joyent.com"
|
|
||||||
testHarnessBucket := fmt.Sprintf("test-bucket-%d", randInt())
|
|
||||||
|
|
||||||
if user == "" || keyId == "" {
|
|
||||||
t.SkipNow()
|
|
||||||
}
|
|
||||||
|
|
||||||
input := authentication.SSHAgentSignerInput{
|
|
||||||
KeyID: keyId,
|
|
||||||
AccountName: user,
|
|
||||||
Username: "",
|
|
||||||
}
|
|
||||||
signer, err := authentication.NewSSHAgentSigner(input)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Error Creating SSH Agent Signer: %s", err.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
config := &triton.ClientConfig{
|
|
||||||
MantaURL: url,
|
|
||||||
AccountName: user,
|
|
||||||
Signers: []authentication.Signer{signer},
|
|
||||||
}
|
|
||||||
|
|
||||||
client, err := storage.NewClient(config)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed initialising Storage client: %s", err.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
logger := logging.NewVaultLogger(log.Debug)
|
|
||||||
mb := &MantaBackend{
|
|
||||||
client: client,
|
|
||||||
directory: testHarnessBucket,
|
|
||||||
logger: logger.Named("storage.mantabackend"),
|
|
||||||
permitPool: physical.NewPermitPool(128),
|
|
||||||
}
|
|
||||||
|
|
||||||
err = mb.client.Dir().Put(context.Background(), &storage.PutDirectoryInput{
|
|
||||||
DirectoryName: path.Join(mantaDefaultRootStore),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal("Error creating test harness directory")
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
err = mb.client.Dir().Delete(context.Background(), &storage.DeleteDirectoryInput{
|
|
||||||
DirectoryName: path.Join(mantaDefaultRootStore, testHarnessBucket),
|
|
||||||
ForceDelete: true,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
if !tt.IsResourceNotFoundError(err) {
|
|
||||||
t.Fatal("failed to delete test harness directory")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
physical.ExerciseBackend(t, mb)
|
|
||||||
physical.ExerciseBackend_ListPrefix(t, mb)
|
|
||||||
}
|
|
||||||
|
|
||||||
func randInt() int {
|
|
||||||
rand.Seed(time.Now().UTC().UnixNano())
|
|
||||||
return rand.New(rand.NewSource(time.Now().UnixNano())).Int()
|
|
||||||
}
|
|
Loading…
Reference in New Issue
Block a user