// Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: BUSL-1.1 package vault import ( "context" "crypto/subtle" "encoding/base64" "errors" "fmt" "strings" "time" "github.com/golang/protobuf/proto" wrapping "github.com/hashicorp/go-kms-wrapping/v2" "github.com/hashicorp/go-uuid" "github.com/hashicorp/vault/helper/constants" "github.com/hashicorp/vault/helper/namespace" "github.com/hashicorp/vault/physical/raft" "github.com/hashicorp/vault/sdk/framework" "github.com/hashicorp/vault/sdk/logical" "github.com/hashicorp/vault/sdk/physical" "github.com/mitchellh/mapstructure" ) // raftStoragePaths returns paths for use when raft is the storage mechanism. func (b *SystemBackend) raftStoragePaths() []*framework.Path { return []*framework.Path{ { Pattern: "storage/raft/bootstrap/answer", Fields: map[string]*framework.FieldSchema{ "server_id": { Type: framework.TypeString, }, "answer": { Type: framework.TypeString, }, "cluster_addr": { Type: framework.TypeString, }, "non_voter": { Type: framework.TypeBool, }, }, Operations: map[logical.Operation]framework.OperationHandler{ logical.UpdateOperation: &framework.PathOperation{ Callback: b.handleRaftBootstrapAnswerWrite(), Summary: "Accepts an answer from the peer to be joined to the fact cluster.", }, }, HelpSynopsis: strings.TrimSpace(sysRaftHelp["raft-bootstrap-answer"][0]), HelpDescription: strings.TrimSpace(sysRaftHelp["raft-bootstrap-answer"][1]), }, { Pattern: "storage/raft/bootstrap/challenge", Fields: map[string]*framework.FieldSchema{ "server_id": { Type: framework.TypeString, }, }, Operations: map[logical.Operation]framework.OperationHandler{ logical.UpdateOperation: &framework.PathOperation{ Callback: b.handleRaftBootstrapChallengeWrite(), Summary: "Creates a challenge for the new peer to be joined to the raft cluster.", }, }, HelpSynopsis: strings.TrimSpace(sysRaftHelp["raft-bootstrap-challenge"][0]), HelpDescription: strings.TrimSpace(sysRaftHelp["raft-bootstrap-challenge"][1]), }, { Pattern: "storage/raft/remove-peer", Fields: map[string]*framework.FieldSchema{ "dr_operation_token": { Type: framework.TypeString, Description: "DR operation token used to authorize this request (if a DR secondary node).", }, "server_id": { Type: framework.TypeString, }, }, Operations: map[logical.Operation]framework.OperationHandler{ logical.UpdateOperation: &framework.PathOperation{ Callback: b.verifyDROperationTokenOnSecondary(b.handleRaftRemovePeerUpdate(), false), Summary: "Remove a peer from the raft cluster.", }, }, HelpSynopsis: strings.TrimSpace(sysRaftHelp["raft-remove-peer"][0]), HelpDescription: strings.TrimSpace(sysRaftHelp["raft-remove-peer"][1]), }, { Pattern: "storage/raft/configuration", Fields: map[string]*framework.FieldSchema{ "dr_operation_token": { Type: framework.TypeString, Description: "DR operation token used to authorize this request (if a DR secondary node).", }, }, Operations: map[logical.Operation]framework.OperationHandler{ logical.ReadOperation: &framework.PathOperation{ Callback: b.handleRaftConfigurationGet(), Summary: "Returns the configuration of the raft cluster.", }, // Reading configuration on a DR secondary cluster is an update // operation to allow consuming the DR operation token for // authenticating the request. logical.UpdateOperation: &framework.PathOperation{ Callback: b.verifyDROperationToken(b.handleRaftConfigurationGet(), false), Summary: "Returns the configuration of the raft cluster in a DR secondary cluster.", }, }, HelpSynopsis: strings.TrimSpace(sysRaftHelp["raft-configuration"][0]), HelpDescription: strings.TrimSpace(sysRaftHelp["raft-configuration"][1]), }, { Pattern: "storage/raft/snapshot", Operations: map[logical.Operation]framework.OperationHandler{ logical.ReadOperation: &framework.PathOperation{ Callback: b.handleStorageRaftSnapshotRead(), Summary: "Returns a snapshot of the current state of vault.", }, logical.UpdateOperation: &framework.PathOperation{ Callback: b.handleStorageRaftSnapshotWrite(false), Summary: "Installs the provided snapshot, returning the cluster to the state defined in it.", }, }, HelpSynopsis: strings.TrimSpace(sysRaftHelp["raft-snapshot"][0]), HelpDescription: strings.TrimSpace(sysRaftHelp["raft-snapshot"][1]), }, { Pattern: "storage/raft/snapshot-force", Operations: map[logical.Operation]framework.OperationHandler{ logical.UpdateOperation: &framework.PathOperation{ Callback: b.handleStorageRaftSnapshotWrite(true), Summary: "Installs the provided snapshot, returning the cluster to the state defined in it. This bypasses checks ensuring the current Autounseal or Shamir keys are consistent with the snapshot data.", }, }, HelpSynopsis: strings.TrimSpace(sysRaftHelp["raft-snapshot-force"][0]), HelpDescription: strings.TrimSpace(sysRaftHelp["raft-snapshot-force"][1]), }, { Pattern: "storage/raft/autopilot/state", Operations: map[logical.Operation]framework.OperationHandler{ logical.ReadOperation: &framework.PathOperation{ Callback: b.verifyDROperationTokenOnSecondary(b.handleStorageRaftAutopilotState(), false), Summary: "Returns the state of the raft cluster under integrated storage as seen by autopilot.", ForwardPerformanceStandby: true, }, }, HelpSynopsis: strings.TrimSpace(sysRaftHelp["raft-autopilot-state"][0]), HelpDescription: strings.TrimSpace(sysRaftHelp["raft-autopilot-state"][1]), }, { Pattern: "storage/raft/autopilot/configuration", Fields: map[string]*framework.FieldSchema{ "cleanup_dead_servers": { Type: framework.TypeBool, Description: "Controls whether to remove dead servers from the Raft peer list periodically or when a new server joins.", }, "last_contact_threshold": { Type: framework.TypeDurationSecond, Description: "Limit on the amount of time a server can go without leader contact before being considered unhealthy.", }, "dead_server_last_contact_threshold": { Type: framework.TypeDurationSecond, Description: "Limit on the amount of time a server can go without leader contact before being considered failed. This takes effect only when cleanup_dead_servers is set.", }, "max_trailing_logs": { Type: framework.TypeInt, Description: "Amount of entries in the Raft Log that a server can be behind before being considered unhealthy.", }, "min_quorum": { Type: framework.TypeInt, Description: "Minimum number of servers allowed in a cluster before autopilot can prune dead servers. This should at least be 3.", }, "server_stabilization_time": { Type: framework.TypeDurationSecond, Description: "Minimum amount of time a server must be in a stable, healthy state before it can be added to the cluster.", }, "disable_upgrade_migration": { Type: framework.TypeBool, Description: "Whether or not to perform automated version upgrades.", }, "dr_operation_token": { Type: framework.TypeString, Description: "DR operation token used to authorize this request (if a DR secondary node).", }, }, Operations: map[logical.Operation]framework.OperationHandler{ logical.ReadOperation: &framework.PathOperation{ Callback: b.verifyDROperationTokenOnSecondary(b.handleStorageRaftAutopilotConfigRead(), false), }, logical.UpdateOperation: &framework.PathOperation{ Callback: b.verifyDROperationTokenOnSecondary(b.handleStorageRaftAutopilotConfigUpdate(), false), }, }, HelpSynopsis: strings.TrimSpace(sysRaftHelp["raft-autopilot-configuration"][0]), HelpDescription: strings.TrimSpace(sysRaftHelp["raft-autopilot-configuration"][1]), }, } } func (b *SystemBackend) handleRaftConfigurationGet() framework.OperationFunc { return func(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { raftBackend := b.Core.getRaftBackend() if raftBackend == nil { return logical.ErrorResponse("raft storage is not in use"), logical.ErrInvalidRequest } config, err := raftBackend.GetConfiguration(ctx) if err != nil { return nil, err } return &logical.Response{ Data: map[string]interface{}{ "config": config, }, }, nil } } func (b *SystemBackend) handleRaftRemovePeerUpdate() framework.OperationFunc { return func(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { serverID := d.Get("server_id").(string) if len(serverID) == 0 { return logical.ErrorResponse("no server id provided"), logical.ErrInvalidRequest } raftBackend := b.Core.getRaftBackend() if raftBackend == nil { return logical.ErrorResponse("raft storage is not in use"), logical.ErrInvalidRequest } if err := raftBackend.RemovePeer(ctx, serverID); err != nil { return nil, err } b.Core.raftFollowerStates.Delete(serverID) return nil, nil } } func (b *SystemBackend) handleRaftBootstrapChallengeWrite() framework.OperationFunc { return func(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { serverID := d.Get("server_id").(string) if len(serverID) == 0 { return logical.ErrorResponse("no server id provided"), logical.ErrInvalidRequest } var answer []byte answerRaw, ok := b.Core.pendingRaftPeers.Load(serverID) if !ok { var err error answer, err = uuid.GenerateRandomBytes(16) if err != nil { return nil, err } b.Core.pendingRaftPeers.Store(serverID, answer) } else { answer = answerRaw.([]byte) } sealAccess := b.Core.seal.GetAccess() eBlob, err := sealAccess.Encrypt(ctx, answer, nil) if err != nil { return nil, err } protoBlob, err := proto.Marshal(eBlob) if err != nil { return nil, err } sealConfig, err := b.Core.seal.BarrierConfig(ctx) if err != nil { return nil, err } return &logical.Response{ Data: map[string]interface{}{ "challenge": base64.StdEncoding.EncodeToString(protoBlob), "seal_config": sealConfig, }, }, nil } } func (b *SystemBackend) handleRaftBootstrapAnswerWrite() framework.OperationFunc { return func(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { raftBackend := b.Core.getRaftBackend() if raftBackend == nil { return logical.ErrorResponse("raft storage is not in use"), logical.ErrInvalidRequest } serverID := d.Get("server_id").(string) if len(serverID) == 0 { return logical.ErrorResponse("no server_id provided"), logical.ErrInvalidRequest } answerRaw := d.Get("answer").(string) if len(answerRaw) == 0 { return logical.ErrorResponse("no answer provided"), logical.ErrInvalidRequest } clusterAddr := d.Get("cluster_addr").(string) if len(clusterAddr) == 0 { return logical.ErrorResponse("no cluster_addr provided"), logical.ErrInvalidRequest } nonVoter := d.Get("non_voter").(bool) answer, err := base64.StdEncoding.DecodeString(answerRaw) if err != nil { return logical.ErrorResponse("could not base64 decode answer"), logical.ErrInvalidRequest } expectedAnswerRaw, ok := b.Core.pendingRaftPeers.Load(serverID) if !ok { return logical.ErrorResponse("no expected answer for the server id provided"), logical.ErrInvalidRequest } b.Core.pendingRaftPeers.Delete(serverID) if subtle.ConstantTimeCompare(answer, expectedAnswerRaw.([]byte)) == 0 { return logical.ErrorResponse("invalid answer given"), logical.ErrInvalidRequest } tlsKeyringEntry, err := b.Core.barrier.Get(ctx, raftTLSStoragePath) if err != nil { return nil, err } if tlsKeyringEntry == nil { return nil, errors.New("could not find raft TLS configuration") } var keyring raft.TLSKeyring if err := tlsKeyringEntry.DecodeJSON(&keyring); err != nil { return nil, errors.New("could not decode raft TLS configuration") } var desiredSuffrage string switch nonVoter { case true: desiredSuffrage = "non-voter" default: desiredSuffrage = "voter" } added := b.Core.raftFollowerStates.Update(&raft.EchoRequestUpdate{ NodeID: serverID, DesiredSuffrage: desiredSuffrage, }) switch nonVoter { case true: err = raftBackend.AddNonVotingPeer(ctx, serverID, clusterAddr) default: err = raftBackend.AddPeer(ctx, serverID, clusterAddr) } if err != nil { if added { b.Core.raftFollowerStates.Delete(serverID) } return nil, err } peers, err := raftBackend.Peers(ctx) if err != nil { return nil, err } b.logger.Info("follower node answered the raft bootstrap challenge", "follower_server_id", serverID) return &logical.Response{ Data: map[string]interface{}{ "peers": peers, "tls_keyring": &keyring, "autoloaded_license": LicenseAutoloaded(b.Core), }, }, nil } } func (b *SystemBackend) handleStorageRaftSnapshotRead() framework.OperationFunc { return func(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { raftStorage, ok := b.Core.underlyingPhysical.(*raft.RaftBackend) if !ok { return logical.ErrorResponse("raft storage is not in use"), logical.ErrInvalidRequest } if req.ResponseWriter == nil { return nil, errors.New("no writer for request") } err := raftStorage.SnapshotHTTP(req.ResponseWriter, b.Core.seal.GetAccess()) if err != nil { return nil, err } return nil, nil } } func (b *SystemBackend) handleStorageRaftAutopilotState() framework.OperationFunc { return func(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { raftBackend := b.Core.getRaftBackend() if raftBackend == nil { return logical.ErrorResponse("raft storage is not in use"), logical.ErrInvalidRequest } state, err := raftBackend.GetAutopilotServerState(ctx) if err != nil { return nil, err } if state == nil { return nil, nil } data := make(map[string]interface{}) err = mapstructure.Decode(state, &data) if err != nil { return nil, err } return &logical.Response{ Data: data, }, nil } } func (b *SystemBackend) handleStorageRaftAutopilotConfigRead() framework.OperationFunc { return func(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { raftBackend := b.Core.getRaftBackend() if raftBackend == nil { return logical.ErrorResponse("raft storage is not in use"), logical.ErrInvalidRequest } config := raftBackend.AutopilotConfig() if config == nil { return nil, nil } return &logical.Response{ Data: map[string]interface{}{ "cleanup_dead_servers": config.CleanupDeadServers, "last_contact_threshold": config.LastContactThreshold.String(), "dead_server_last_contact_threshold": config.DeadServerLastContactThreshold.String(), "max_trailing_logs": config.MaxTrailingLogs, "min_quorum": config.MinQuorum, "server_stabilization_time": config.ServerStabilizationTime.String(), "disable_upgrade_migration": config.DisableUpgradeMigration, }, }, nil } } func (b *SystemBackend) handleStorageRaftAutopilotConfigUpdate() framework.OperationFunc { return func(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { raftBackend := b.Core.getRaftBackend() if raftBackend == nil { return logical.ErrorResponse("raft storage is not in use"), logical.ErrInvalidRequest } // Read autopilot configuration from storage config, err := b.Core.loadAutopilotConfiguration(ctx) if err != nil { b.logger.Error("failed to load autopilot config from storage when setting up cluster; continuing since autopilot falls back to default config", "error", err) } if config == nil { config = &raft.AutopilotConfig{} } persist := false cleanupDeadServers, ok := d.GetOk("cleanup_dead_servers") if ok { if cleanupDeadServers.(bool) { config.CleanupDeadServersValue = raft.CleanupDeadServersTrue } else { config.CleanupDeadServersValue = raft.CleanupDeadServersFalse } persist = true } lastContactThreshold, ok := d.GetOk("last_contact_threshold") if ok { config.LastContactThreshold = time.Duration(lastContactThreshold.(int)) * time.Second persist = true } deadServerLastContactThreshold, ok := d.GetOk("dead_server_last_contact_threshold") if ok { config.DeadServerLastContactThreshold = time.Duration(deadServerLastContactThreshold.(int)) * time.Second persist = true } maxTrailingLogs, ok := d.GetOk("max_trailing_logs") if ok { config.MaxTrailingLogs = uint64(maxTrailingLogs.(int)) persist = true } minQuorum, ok := d.GetOk("min_quorum") if ok { config.MinQuorum = uint(minQuorum.(int)) persist = true } serverStabilizationTime, ok := d.GetOk("server_stabilization_time") if ok { config.ServerStabilizationTime = time.Duration(serverStabilizationTime.(int)) * time.Second persist = true } disableUpgradeMigration, ok := d.GetOk("disable_upgrade_migration") if ok { if !constants.IsEnterprise { return logical.ErrorResponse("disable_upgrade_migration is only available in Vault Enterprise"), logical.ErrInvalidRequest } config.DisableUpgradeMigration = disableUpgradeMigration.(bool) persist = true } effectiveConf := raftBackend.AutopilotConfig() effectiveConf.Merge(config) if effectiveConf.CleanupDeadServers && effectiveConf.MinQuorum < 3 { return logical.ErrorResponse(fmt.Sprintf("min_quorum must be set when cleanup_dead_servers is set and it should at least be 3; cleanup_dead_servers: %#v, min_quorum: %#v", effectiveConf.CleanupDeadServers, effectiveConf.MinQuorum)), logical.ErrInvalidRequest } if effectiveConf.CleanupDeadServers && effectiveConf.DeadServerLastContactThreshold.Seconds() < 60 { return logical.ErrorResponse(fmt.Sprintf("dead_server_last_contact_threshold should not be set to less than 1m; received: %v", deadServerLastContactThreshold)), logical.ErrInvalidRequest } // Persist only the user supplied fields if persist { entry, err := logical.StorageEntryJSON(raftAutopilotConfigurationStoragePath, config) if err != nil { return nil, err } if err := b.Core.barrier.Put(ctx, entry); err != nil { return nil, err } } // Set the effectiveConfig raftBackend.SetAutopilotConfig(effectiveConf) return nil, nil } } func (b *SystemBackend) handleStorageRaftSnapshotWrite(force bool) framework.OperationFunc { return func(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { raftStorage, ok := b.Core.underlyingPhysical.(*raft.RaftBackend) if !ok { return logical.ErrorResponse("raft storage is not in use"), logical.ErrInvalidRequest } body, ok := logical.ContextOriginalBodyValue(ctx) if !ok { return nil, errors.New("no reader for request") } access := b.Core.seal.GetAccess() if force { access = nil } // We want to buffer the http request reader into a temp file here so we // don't have to hold the full snapshot in memory. We also want to do // the restore in two parts so we can restore the snapshot while the // stateLock is write locked. snapFile, cleanup, metadata, err := raftStorage.WriteSnapshotToTemp(body, access) switch { case err == nil: case strings.Contains(err.Error(), "failed to open the sealed hashes"): switch b.Core.seal.BarrierType() { case wrapping.WrapperTypeShamir: return logical.ErrorResponse("could not verify hash file, possibly the snapshot is using a different set of unseal keys; use the snapshot-force API to bypass this check"), logical.ErrInvalidRequest default: return logical.ErrorResponse("could not verify hash file, possibly the snapshot is using a different autoseal key; use the snapshot-force API to bypass this check"), logical.ErrInvalidRequest } case err != nil: b.Core.logger.Error("raft snapshot restore: failed to write snapshot", "error", err) return nil, err } // We want to do this in a go routine so we can upgrade the lock and // allow the client to disconnect. go func() (retErr error) { // Cleanup the temp file defer cleanup() // Grab statelock l := newLockGrabber(b.Core.stateLock.Lock, b.Core.stateLock.Unlock, b.Core.standbyStopCh.Load().(chan struct{})) go l.grab() if stopped := l.lockOrStop(); stopped { b.Core.logger.Error("not applying snapshot; shutting down") return } defer b.Core.stateLock.Unlock() // If we failed to restore the snapshot we should seal this node as // it's in an unknown state defer func() { if retErr != nil { if err := b.Core.sealInternalWithOptions(false, false, true); err != nil { b.Core.logger.Error("failed to seal node", "error", err) } } }() ctx, ctxCancel := context.WithCancel(namespace.RootContext(nil)) // We are calling the callback function synchronously here while we // have the lock. So set it to nil and restore the callback when we // finish. raftStorage.SetRestoreCallback(nil) defer raftStorage.SetRestoreCallback(b.Core.raftSnapshotRestoreCallback(true, true)) // Do a preSeal to clear vault's in-memory caches and shut down any // systems that might be holding the encryption access. b.Core.logger.Info("shutting down prior to restoring snapshot") if err := b.Core.preSeal(); err != nil { b.Core.logger.Error("raft snapshot restore failed preSeal", "error", err) return err } b.Core.logger.Info("applying snapshot") if err := raftStorage.RestoreSnapshot(ctx, metadata, snapFile); err != nil { b.Core.logger.Error("error while restoring raft snapshot", "error", err) return err } // Run invalidation logic synchronously here callback := b.Core.raftSnapshotRestoreCallback(false, false) if err := callback(ctx); err != nil { return err } { // If the snapshot was taken while another node was leader we // need to reset the leader information to this node. if err := b.Core.underlyingPhysical.Put(ctx, &physical.Entry{ Key: CoreLockPath, Value: []byte(b.Core.leaderUUID), }); err != nil { b.Core.logger.Error("cluster setup failed", "error", err) return err } // re-advertise our cluster information if err := b.Core.advertiseLeader(ctx, b.Core.leaderUUID, nil); err != nil { b.Core.logger.Error("cluster setup failed", "error", err) return err } } if err := b.Core.postUnseal(ctx, ctxCancel, standardUnsealStrategy{}); err != nil { b.Core.logger.Error("raft snapshot restore failed postUnseal", "error", err) return err } return nil }() return nil, nil } } var sysRaftHelp = map[string][2]string{ "raft-bootstrap-challenge": { "Creates a challenge for the new peer to be joined to the raft cluster.", "", }, "raft-bootstrap-answer": { "Accepts an answer from the peer to be joined to the fact cluster.", "", }, "raft-configuration": { "Returns the raft cluster configuration.", `On a DR secondary cluster, instead of a GET, this must be a POST or PUT, and furthermore a DR operation token must be provided.`, }, "raft-remove-peer": { "Removes a peer from the raft cluster.", "", }, "raft-snapshot": { "Restores and saves snapshots from the raft cluster.", "", }, "raft-snapshot-force": { "Force restore a raft cluster snapshot", "", }, "raft-autopilot-state": { "Returns the state of the raft cluster under integrated storage as seen by autopilot.", "", }, "raft-autopilot-configuration": { "Returns autopilot configuration.", "", }, }