From 847f8ad8e754f1e69673134c1213508097a4e777 Mon Sep 17 00:00:00 2001 From: Konstantin Demin Date: Mon, 1 Jul 2024 03:30:54 +0300 Subject: [PATCH] remove manta (joyent) --- command/commands.go | 2 - go.mod | 2 +- physical/manta/manta.go | 266 ----------------------------------- physical/manta/manta_test.go | 89 ------------ 4 files changed, 1 insertion(+), 358 deletions(-) delete mode 100644 physical/manta/manta.go delete mode 100644 physical/manta/manta_test.go diff --git a/command/commands.go b/command/commands.go index 9b7f25374..34417b817 100644 --- a/command/commands.go +++ b/command/commands.go @@ -48,7 +48,6 @@ import ( physCouchDB "github.com/hashicorp/vault/physical/couchdb" physEtcd "github.com/hashicorp/vault/physical/etcd" physFoundationDB "github.com/hashicorp/vault/physical/foundationdb" - physManta "github.com/hashicorp/vault/physical/manta" physMySQL "github.com/hashicorp/vault/physical/mysql" physOCI "github.com/hashicorp/vault/physical/oci" physPostgreSQL "github.com/hashicorp/vault/physical/postgresql" @@ -189,7 +188,6 @@ var ( "inmem_transactional_ha": physInmem.NewTransactionalInmemHA, "inmem_transactional": physInmem.NewTransactionalInmem, "inmem": physInmem.NewInmem, - "manta": physManta.NewMantaBackend, "mysql": physMySQL.NewMySQLBackend, "oci": physOCI.NewBackend, "postgresql": physPostgreSQL.NewPostgreSQLBackend, diff --git a/go.mod b/go.mod index 9217c92ee..e757a89e0 100644 --- a/go.mod +++ b/go.mod @@ -125,7 +125,6 @@ require ( github.com/jcmturner/gokrb5/v8 v8.4.4 github.com/jefferai/isbadcipher v0.0.0-20190226160619-51d2077c035f github.com/jefferai/jsonx v1.0.0 - github.com/joyent/triton-go v1.7.1-0.20200416154420-6801d15b779f github.com/klauspost/compress v1.16.7 github.com/kr/pretty v0.3.1 github.com/kr/text v0.2.0 @@ -312,6 +311,7 @@ require ( github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/joyent/triton-go v1.7.1-0.20200416154420-6801d15b779f // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/klauspost/pgzip v1.2.5 // indirect diff --git a/physical/manta/manta.go b/physical/manta/manta.go deleted file mode 100644 index 5ab1c4e05..000000000 --- a/physical/manta/manta.go +++ /dev/null @@ -1,266 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package manta - -import ( - "bytes" - "context" - "fmt" - "io" - "os" - "path" - "sort" - "strconv" - "strings" - "time" - - metrics "github.com/armon/go-metrics" - log "github.com/hashicorp/go-hclog" - "github.com/hashicorp/vault/sdk/physical" - triton "github.com/joyent/triton-go" - "github.com/joyent/triton-go/authentication" - "github.com/joyent/triton-go/errors" - "github.com/joyent/triton-go/storage" -) - -const mantaDefaultRootStore = "/stor" - -type MantaBackend struct { - logger log.Logger - permitPool *physical.PermitPool - client *storage.StorageClient - directory string -} - -func NewMantaBackend(conf map[string]string, logger log.Logger) (physical.Backend, error) { - user := os.Getenv("MANTA_USER") - if user == "" { - user = conf["user"] - } - - keyId := os.Getenv("MANTA_KEY_ID") - if keyId == "" { - keyId = conf["key_id"] - } - - url := os.Getenv("MANTA_URL") - if url == "" { - url = conf["url"] - } else { - url = "https://us-east.manta.joyent.com" - } - - subuser := os.Getenv("MANTA_SUBUSER") - if subuser == "" { - if confUser, ok := conf["subuser"]; ok { - subuser = confUser - } - } - - input := authentication.SSHAgentSignerInput{ - KeyID: keyId, - AccountName: user, - Username: subuser, - } - signer, err := authentication.NewSSHAgentSigner(input) - if err != nil { - return nil, fmt.Errorf("Error Creating SSH Agent Signer: %w", err) - } - - maxParStr, ok := conf["max_parallel"] - var maxParInt int - if ok { - maxParInt, err = strconv.Atoi(maxParStr) - if err != nil { - return nil, fmt.Errorf("failed parsing max_parallel parameter: %w", err) - } - if logger.IsDebug() { - logger.Debug("max_parallel set", "max_parallel", maxParInt) - } - } - - config := &triton.ClientConfig{ - MantaURL: url, - AccountName: user, - Signers: []authentication.Signer{signer}, - } - - client, err := storage.NewClient(config) - if err != nil { - return nil, fmt.Errorf("failed initialising Storage client: %w", err) - } - - return &MantaBackend{ - client: client, - directory: conf["directory"], - logger: logger, - permitPool: physical.NewPermitPool(maxParInt), - }, nil -} - -// Put is used to insert or update an entry -func (m *MantaBackend) Put(ctx context.Context, entry *physical.Entry) error { - defer metrics.MeasureSince([]string{"manta", "put"}, time.Now()) - - m.permitPool.Acquire() - defer m.permitPool.Release() - - r := bytes.NewReader(entry.Value) - r.Seek(0, 0) - - return m.client.Objects().Put(ctx, &storage.PutObjectInput{ - ObjectPath: path.Join(mantaDefaultRootStore, m.directory, entry.Key, ".vault_value"), - ObjectReader: r, - ContentLength: uint64(len(entry.Value)), - ForceInsert: true, - }) -} - -// Get is used to fetch an entry -func (m *MantaBackend) Get(ctx context.Context, key string) (*physical.Entry, error) { - defer metrics.MeasureSince([]string{"manta", "get"}, time.Now()) - - m.permitPool.Acquire() - defer m.permitPool.Release() - - output, err := m.client.Objects().Get(ctx, &storage.GetObjectInput{ - ObjectPath: path.Join(mantaDefaultRootStore, m.directory, key, ".vault_value"), - }) - if err != nil { - if strings.Contains(err.Error(), "ResourceNotFound") { - return nil, nil - } - return nil, err - } - - defer output.ObjectReader.Close() - - data := make([]byte, output.ContentLength) - _, err = io.ReadFull(output.ObjectReader, data) - if err != nil { - return nil, err - } - - ent := &physical.Entry{ - Key: key, - Value: data, - } - - return ent, nil -} - -// Delete is used to permanently delete an entry -func (m *MantaBackend) Delete(ctx context.Context, key string) error { - defer metrics.MeasureSince([]string{"manta", "delete"}, time.Now()) - - m.permitPool.Acquire() - defer m.permitPool.Release() - - if strings.HasSuffix(key, "/") { - err := m.client.Dir().Delete(ctx, &storage.DeleteDirectoryInput{ - DirectoryName: path.Join(mantaDefaultRootStore, m.directory, key), - ForceDelete: true, - }) - if err != nil { - return err - } - } else { - err := m.client.Objects().Delete(ctx, &storage.DeleteObjectInput{ - ObjectPath: path.Join(mantaDefaultRootStore, m.directory, key, ".vault_value"), - }) - if err != nil { - if errors.IsResourceNotFound(err) { - return nil - } - return err - } - - return tryDeleteDirectory(ctx, m, path.Join(mantaDefaultRootStore, m.directory, key)) - } - - return nil -} - -func tryDeleteDirectory(ctx context.Context, m *MantaBackend, directoryPath string) error { - objs, err := m.client.Dir().List(ctx, &storage.ListDirectoryInput{ - DirectoryName: directoryPath, - }) - if err != nil { - if errors.IsResourceNotFound(err) { - return nil - } - return err - } - if objs != nil && len(objs.Entries) == 0 { - err := m.client.Dir().Delete(ctx, &storage.DeleteDirectoryInput{ - DirectoryName: directoryPath, - }) - if err != nil { - return err - } - - return tryDeleteDirectory(ctx, m, path.Dir(directoryPath)) - } - return nil -} - -// List is used to list all the keys under a given -// prefix, up to the next prefix. -func (m *MantaBackend) List(ctx context.Context, prefix string) ([]string, error) { - defer metrics.MeasureSince([]string{"manta", "list"}, time.Now()) - - m.permitPool.Acquire() - defer m.permitPool.Release() - - objs, err := m.client.Dir().List(ctx, &storage.ListDirectoryInput{ - DirectoryName: path.Join(mantaDefaultRootStore, m.directory, prefix), - }) - if err != nil { - if errors.IsResourceNotFound(err) { - return []string{}, nil - } - return nil, err - } - - keys := []string{} - for _, obj := range objs.Entries { - if obj.Type == "directory" { - objs, err := m.client.Dir().List(ctx, &storage.ListDirectoryInput{ - DirectoryName: path.Join(mantaDefaultRootStore, m.directory, prefix, obj.Name), - }) - if err != nil { - if !errors.IsResourceNotFound(err) { - return nil, err - } - } - - // We need to check to see if there is something more than just the `value` file - // if the length of the children is: - // > 1 and includes the value `index` then we need to add foo and foo/ - // = 1 and the value is `index` then we need to add foo - // = 1 and the value is not `index` then we need to add foo/ - if len(objs.Entries) == 1 { - if objs.Entries[0].Name != ".vault_value" { - keys = append(keys, fmt.Sprintf("%s/", obj.Name)) - } else { - keys = append(keys, obj.Name) - } - } else if len(objs.Entries) > 1 { - for _, childObj := range objs.Entries { - if childObj.Name == ".vault_value" { - keys = append(keys, obj.Name) - } else { - keys = append(keys, fmt.Sprintf("%s/", obj.Name)) - } - } - } else { - keys = append(keys, obj.Name) - } - } - } - - sort.Strings(keys) - - return keys, nil -} diff --git a/physical/manta/manta_test.go b/physical/manta/manta_test.go deleted file mode 100644 index 11b024dd9..000000000 --- a/physical/manta/manta_test.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package manta - -import ( - "context" - "fmt" - "math/rand" - "os" - "path" - "testing" - "time" - - log "github.com/hashicorp/go-hclog" - "github.com/hashicorp/vault/sdk/helper/logging" - "github.com/hashicorp/vault/sdk/physical" - triton "github.com/joyent/triton-go" - "github.com/joyent/triton-go/authentication" - tt "github.com/joyent/triton-go/errors" - "github.com/joyent/triton-go/storage" -) - -func TestMantaBackend(t *testing.T) { - user := os.Getenv("MANTA_USER") - keyId := os.Getenv("MANTA_KEY_ID") - url := "https://us-east.manta.joyent.com" - testHarnessBucket := fmt.Sprintf("test-bucket-%d", randInt()) - - if user == "" || keyId == "" { - t.SkipNow() - } - - input := authentication.SSHAgentSignerInput{ - KeyID: keyId, - AccountName: user, - Username: "", - } - signer, err := authentication.NewSSHAgentSigner(input) - if err != nil { - t.Fatalf("Error Creating SSH Agent Signer: %s", err.Error()) - } - - config := &triton.ClientConfig{ - MantaURL: url, - AccountName: user, - Signers: []authentication.Signer{signer}, - } - - client, err := storage.NewClient(config) - if err != nil { - t.Fatalf("failed initialising Storage client: %s", err.Error()) - } - - logger := logging.NewVaultLogger(log.Debug) - mb := &MantaBackend{ - client: client, - directory: testHarnessBucket, - logger: logger.Named("storage.mantabackend"), - permitPool: physical.NewPermitPool(128), - } - - err = mb.client.Dir().Put(context.Background(), &storage.PutDirectoryInput{ - DirectoryName: path.Join(mantaDefaultRootStore), - }) - if err != nil { - t.Fatal("Error creating test harness directory") - } - - defer func() { - err = mb.client.Dir().Delete(context.Background(), &storage.DeleteDirectoryInput{ - DirectoryName: path.Join(mantaDefaultRootStore, testHarnessBucket), - ForceDelete: true, - }) - if err != nil { - if !tt.IsResourceNotFoundError(err) { - t.Fatal("failed to delete test harness directory") - } - } - }() - - physical.ExerciseBackend(t, mb) - physical.ExerciseBackend_ListPrefix(t, mb) -} - -func randInt() int { - rand.Seed(time.Now().UTC().UnixNano()) - return rand.New(rand.NewSource(time.Now().UnixNano())).Int() -}