krd/vault-redux
1
0
Fork 0
Code

remove manta (joyent)

Browse Source
This commit is contained in:
Konstantin Demin 2024-07-01 03:30:54 +03:00
parent faed4aa7af
commit 847f8ad8e7
4 changed files with 1 additions and 358 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
command/commands.go
View File

@ -48,7 +48,6 @@ import (
physCouchDB "github.com/hashicorp/vault/physical/couchdb"
physEtcd "github.com/hashicorp/vault/physical/etcd"
physFoundationDB "github.com/hashicorp/vault/physical/foundationdb"
physManta "github.com/hashicorp/vault/physical/manta"
physMySQL "github.com/hashicorp/vault/physical/mysql"
physOCI "github.com/hashicorp/vault/physical/oci"
physPostgreSQL "github.com/hashicorp/vault/physical/postgresql"
@ -189,7 +188,6 @@ var (
"inmem_transactional_ha": physInmem.NewTransactionalInmemHA,
"inmem_transactional": physInmem.NewTransactionalInmem,
"inmem": physInmem.NewInmem,
"manta": physManta.NewMantaBackend,
"mysql": physMySQL.NewMySQLBackend,
"oci": physOCI.NewBackend,
"postgresql": physPostgreSQL.NewPostgreSQLBackend,

2
go.mod
View File

@ -125,7 +125,6 @@ require (
github.com/jcmturner/gokrb5/v8 v8.4.4
github.com/jefferai/isbadcipher v0.0.0-20190226160619-51d2077c035f
github.com/jefferai/jsonx v1.0.0
github.com/joyent/triton-go v1.7.1-0.20200416154420-6801d15b779f
github.com/klauspost/compress v1.16.7
github.com/kr/pretty v0.3.1
github.com/kr/text v0.2.0
@ -312,6 +311,7 @@ require (
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/joyent/triton-go v1.7.1-0.20200416154420-6801d15b779f // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kelseyhightower/envconfig v1.4.0 // indirect
github.com/klauspost/pgzip v1.2.5 // indirect

266
physical/manta/manta.go
View File

@ -1,266 +0,0 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
package manta
import (
"bytes"
"context"
"fmt"
"io"
"os"
"path"
"sort"
"strconv"
"strings"
"time"
metrics "github.com/armon/go-metrics"
log "github.com/hashicorp/go-hclog"
"github.com/hashicorp/vault/sdk/physical"
triton "github.com/joyent/triton-go"
"github.com/joyent/triton-go/authentication"
"github.com/joyent/triton-go/errors"
"github.com/joyent/triton-go/storage"
)
const mantaDefaultRootStore = "/stor"
type MantaBackend struct {
logger log.Logger
permitPool *physical.PermitPool
client *storage.StorageClient
directory string
}
func NewMantaBackend(conf map[string]string, logger log.Logger) (physical.Backend, error) {
user := os.Getenv("MANTA_USER")
if user == "" {
user = conf["user"]
}
keyId := os.Getenv("MANTA_KEY_ID")
if keyId == "" {
keyId = conf["key_id"]
}
url := os.Getenv("MANTA_URL")
if url == "" {
url = conf["url"]
} else {
url = "https://us-east.manta.joyent.com"
}
subuser := os.Getenv("MANTA_SUBUSER")
if subuser == "" {
if confUser, ok := conf["subuser"]; ok {
subuser = confUser
}
}
input := authentication.SSHAgentSignerInput{
KeyID: keyId,
AccountName: user,
Username: subuser,
}
signer, err := authentication.NewSSHAgentSigner(input)
if err != nil {
return nil, fmt.Errorf("Error Creating SSH Agent Signer: %w", err)
}
maxParStr, ok := conf["max_parallel"]
var maxParInt int
if ok {
maxParInt, err = strconv.Atoi(maxParStr)
if err != nil {
return nil, fmt.Errorf("failed parsing max_parallel parameter: %w", err)
}
if logger.IsDebug() {
logger.Debug("max_parallel set", "max_parallel", maxParInt)
}
}
config := &triton.ClientConfig{
MantaURL: url,
AccountName: user,
Signers: []authentication.Signer{signer},
}
client, err := storage.NewClient(config)
if err != nil {
return nil, fmt.Errorf("failed initialising Storage client: %w", err)
}
return &MantaBackend{
client: client,
directory: conf["directory"],
logger: logger,
permitPool: physical.NewPermitPool(maxParInt),
}, nil
}
// Put is used to insert or update an entry
func (m *MantaBackend) Put(ctx context.Context, entry *physical.Entry) error {
defer metrics.MeasureSince([]string{"manta", "put"}, time.Now())
m.permitPool.Acquire()
defer m.permitPool.Release()
r := bytes.NewReader(entry.Value)
r.Seek(0, 0)
return m.client.Objects().Put(ctx, &storage.PutObjectInput{
ObjectPath: path.Join(mantaDefaultRootStore, m.directory, entry.Key, ".vault_value"),
ObjectReader: r,
ContentLength: uint64(len(entry.Value)),
ForceInsert: true,
})
}
// Get is used to fetch an entry
func (m *MantaBackend) Get(ctx context.Context, key string) (*physical.Entry, error) {
defer metrics.MeasureSince([]string{"manta", "get"}, time.Now())
m.permitPool.Acquire()
defer m.permitPool.Release()
output, err := m.client.Objects().Get(ctx, &storage.GetObjectInput{
ObjectPath: path.Join(mantaDefaultRootStore, m.directory, key, ".vault_value"),
})
if err != nil {
if strings.Contains(err.Error(), "ResourceNotFound") {
return nil, nil
}
return nil, err
}
defer output.ObjectReader.Close()
data := make([]byte, output.ContentLength)
_, err = io.ReadFull(output.ObjectReader, data)
if err != nil {
return nil, err
}
ent := &physical.Entry{
Key: key,
Value: data,
}
return ent, nil
}
// Delete is used to permanently delete an entry
func (m *MantaBackend) Delete(ctx context.Context, key string) error {
defer metrics.MeasureSince([]string{"manta", "delete"}, time.Now())
m.permitPool.Acquire()
defer m.permitPool.Release()
if strings.HasSuffix(key, "/") {
err := m.client.Dir().Delete(ctx, &storage.DeleteDirectoryInput{
DirectoryName: path.Join(mantaDefaultRootStore, m.directory, key),
ForceDelete: true,
})
if err != nil {
return err
}
} else {
err := m.client.Objects().Delete(ctx, &storage.DeleteObjectInput{
ObjectPath: path.Join(mantaDefaultRootStore, m.directory, key, ".vault_value"),
})
if err != nil {
if errors.IsResourceNotFound(err) {
return nil
}
return err
}
return tryDeleteDirectory(ctx, m, path.Join(mantaDefaultRootStore, m.directory, key))
}
return nil
}
func tryDeleteDirectory(ctx context.Context, m *MantaBackend, directoryPath string) error {
objs, err := m.client.Dir().List(ctx, &storage.ListDirectoryInput{
DirectoryName: directoryPath,
})
if err != nil {
if errors.IsResourceNotFound(err) {
return nil
}
return err
}
if objs != nil && len(objs.Entries) == 0 {
err := m.client.Dir().Delete(ctx, &storage.DeleteDirectoryInput{
DirectoryName: directoryPath,
})
if err != nil {
return err
}
return tryDeleteDirectory(ctx, m, path.Dir(directoryPath))
}
return nil
}
// List is used to list all the keys under a given
// prefix, up to the next prefix.
func (m *MantaBackend) List(ctx context.Context, prefix string) ([]string, error) {
defer metrics.MeasureSince([]string{"manta", "list"}, time.Now())
m.permitPool.Acquire()
defer m.permitPool.Release()
objs, err := m.client.Dir().List(ctx, &storage.ListDirectoryInput{
DirectoryName: path.Join(mantaDefaultRootStore, m.directory, prefix),
})
if err != nil {
if errors.IsResourceNotFound(err) {
return []string{}, nil
}
return nil, err
}
keys := []string{}
for _, obj := range objs.Entries {
if obj.Type == "directory" {
objs, err := m.client.Dir().List(ctx, &storage.ListDirectoryInput{
DirectoryName: path.Join(mantaDefaultRootStore, m.directory, prefix, obj.Name),
})
if err != nil {
if !errors.IsResourceNotFound(err) {
return nil, err
}
}
// We need to check to see if there is something more than just the `value` file
// if the length of the children is:
// > 1 and includes the value `index` then we need to add foo and foo/
// = 1 and the value is `index` then we need to add foo
// = 1 and the value is not `index` then we need to add foo/
if len(objs.Entries) == 1 {
if objs.Entries[0].Name != ".vault_value" {
keys = append(keys, fmt.Sprintf("%s/", obj.Name))
} else {
keys = append(keys, obj.Name)
}
} else if len(objs.Entries) > 1 {
for _, childObj := range objs.Entries {
if childObj.Name == ".vault_value" {
keys = append(keys, obj.Name)
} else {
keys = append(keys, fmt.Sprintf("%s/", obj.Name))
}
}
} else {
keys = append(keys, obj.Name)
}
}
}
sort.Strings(keys)
return keys, nil
}

89
physical/manta/manta_test.go
View File

@ -1,89 +0,0 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
package manta
import (
"context"
"fmt"
"math/rand"
"os"
"path"
"testing"
"time"
log "github.com/hashicorp/go-hclog"
"github.com/hashicorp/vault/sdk/helper/logging"
"github.com/hashicorp/vault/sdk/physical"
triton "github.com/joyent/triton-go"
"github.com/joyent/triton-go/authentication"
tt "github.com/joyent/triton-go/errors"
"github.com/joyent/triton-go/storage"
)
func TestMantaBackend(t *testing.T) {
user := os.Getenv("MANTA_USER")
keyId := os.Getenv("MANTA_KEY_ID")
url := "https://us-east.manta.joyent.com"
testHarnessBucket := fmt.Sprintf("test-bucket-%d", randInt())
if user == "" || keyId == "" {
t.SkipNow()
}
input := authentication.SSHAgentSignerInput{
KeyID: keyId,
AccountName: user,
Username: "",
}
signer, err := authentication.NewSSHAgentSigner(input)
if err != nil {
t.Fatalf("Error Creating SSH Agent Signer: %s", err.Error())
}
config := &triton.ClientConfig{
MantaURL: url,
AccountName: user,
Signers: []authentication.Signer{signer},
}
client, err := storage.NewClient(config)
if err != nil {
t.Fatalf("failed initialising Storage client: %s", err.Error())
}
logger := logging.NewVaultLogger(log.Debug)
mb := &MantaBackend{
client: client,
directory: testHarnessBucket,
logger: logger.Named("storage.mantabackend"),
permitPool: physical.NewPermitPool(128),
}
err = mb.client.Dir().Put(context.Background(), &storage.PutDirectoryInput{
DirectoryName: path.Join(mantaDefaultRootStore),
})
if err != nil {
t.Fatal("Error creating test harness directory")
}
defer func() {
err = mb.client.Dir().Delete(context.Background(), &storage.DeleteDirectoryInput{
DirectoryName: path.Join(mantaDefaultRootStore, testHarnessBucket),
ForceDelete: true,
})
if err != nil {
if !tt.IsResourceNotFoundError(err) {
t.Fatal("failed to delete test harness directory")
}
}
}()
physical.ExerciseBackend(t, mb)
physical.ExerciseBackend_ListPrefix(t, mb)
}
func randInt() int {
rand.Seed(time.Now().UTC().UnixNano())
return rand.New(rand.NewSource(time.Now().UnixNano())).Int()
}