446 lines
16 KiB
Go
446 lines
16 KiB
Go
// Copyright (c) HashiCorp, Inc.
|
|
// SPDX-License-Identifier: MPL-2.0
|
|
|
|
package api
|
|
|
|
import (
|
|
"archive/tar"
|
|
"compress/gzip"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"net/http"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/hashicorp/go-secure-stdlib/parseutil"
|
|
"github.com/mitchellh/mapstructure"
|
|
)
|
|
|
|
var ErrIncompleteSnapshot = errors.New("incomplete snapshot, unable to read SHA256SUMS.sealed file")
|
|
|
|
// RaftJoinResponse represents the response of the raft join API
|
|
type RaftJoinResponse struct {
|
|
Joined bool `json:"joined"`
|
|
}
|
|
|
|
// RaftJoinRequest represents the parameters consumed by the raft join API
|
|
type RaftJoinRequest struct {
|
|
AutoJoin string `json:"auto_join"`
|
|
AutoJoinScheme string `json:"auto_join_scheme"`
|
|
AutoJoinPort uint `json:"auto_join_port"`
|
|
LeaderAPIAddr string `json:"leader_api_addr"`
|
|
LeaderCACert string `json:"leader_ca_cert"`
|
|
LeaderClientCert string `json:"leader_client_cert"`
|
|
LeaderClientKey string `json:"leader_client_key"`
|
|
Retry bool `json:"retry"`
|
|
NonVoter bool `json:"non_voter"`
|
|
}
|
|
|
|
// AutopilotConfig is used for querying/setting the Autopilot configuration.
|
|
type AutopilotConfig struct {
|
|
CleanupDeadServers bool `json:"cleanup_dead_servers" mapstructure:"cleanup_dead_servers"`
|
|
LastContactThreshold time.Duration `json:"last_contact_threshold" mapstructure:"-"`
|
|
DeadServerLastContactThreshold time.Duration `json:"dead_server_last_contact_threshold" mapstructure:"-"`
|
|
MaxTrailingLogs uint64 `json:"max_trailing_logs" mapstructure:"max_trailing_logs"`
|
|
MinQuorum uint `json:"min_quorum" mapstructure:"min_quorum"`
|
|
ServerStabilizationTime time.Duration `json:"server_stabilization_time" mapstructure:"-"`
|
|
DisableUpgradeMigration bool `json:"disable_upgrade_migration" mapstructure:"disable_upgrade_migration"`
|
|
}
|
|
|
|
// MarshalJSON makes the autopilot config fields JSON compatible
|
|
func (ac *AutopilotConfig) MarshalJSON() ([]byte, error) {
|
|
return json.Marshal(map[string]interface{}{
|
|
"cleanup_dead_servers": ac.CleanupDeadServers,
|
|
"last_contact_threshold": ac.LastContactThreshold.String(),
|
|
"dead_server_last_contact_threshold": ac.DeadServerLastContactThreshold.String(),
|
|
"max_trailing_logs": ac.MaxTrailingLogs,
|
|
"min_quorum": ac.MinQuorum,
|
|
"server_stabilization_time": ac.ServerStabilizationTime.String(),
|
|
"disable_upgrade_migration": ac.DisableUpgradeMigration,
|
|
})
|
|
}
|
|
|
|
// UnmarshalJSON parses the autopilot config JSON blob
|
|
func (ac *AutopilotConfig) UnmarshalJSON(b []byte) error {
|
|
var data interface{}
|
|
err := json.Unmarshal(b, &data)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
conf := data.(map[string]interface{})
|
|
if err = mapstructure.WeakDecode(conf, ac); err != nil {
|
|
return err
|
|
}
|
|
if ac.LastContactThreshold, err = parseutil.ParseDurationSecond(conf["last_contact_threshold"]); err != nil {
|
|
return err
|
|
}
|
|
if ac.DeadServerLastContactThreshold, err = parseutil.ParseDurationSecond(conf["dead_server_last_contact_threshold"]); err != nil {
|
|
return err
|
|
}
|
|
if ac.ServerStabilizationTime, err = parseutil.ParseDurationSecond(conf["server_stabilization_time"]); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// AutopilotState represents the response of the raft autopilot state API
|
|
type AutopilotState struct {
|
|
Healthy bool `mapstructure:"healthy"`
|
|
FailureTolerance int `mapstructure:"failure_tolerance"`
|
|
Servers map[string]*AutopilotServer `mapstructure:"servers"`
|
|
Leader string `mapstructure:"leader"`
|
|
Voters []string `mapstructure:"voters"`
|
|
NonVoters []string `mapstructure:"non_voters"`
|
|
RedundancyZones map[string]AutopilotZone `mapstructure:"redundancy_zones,omitempty"`
|
|
Upgrade *AutopilotUpgrade `mapstructure:"upgrade_info,omitempty"`
|
|
OptimisticFailureTolerance int `mapstructure:"optimistic_failure_tolerance,omitempty"`
|
|
}
|
|
|
|
func (a *AutopilotState) String() string {
|
|
var result string
|
|
result += fmt.Sprintf("Healthy: %t. FailureTolerance: %d. Leader: %s. OptimisticFailureTolerance: %d\n", a.Healthy, a.FailureTolerance, a.Leader, a.OptimisticFailureTolerance)
|
|
for _, s := range a.Servers {
|
|
result += fmt.Sprintf("Server: %s\n", s)
|
|
}
|
|
result += fmt.Sprintf("Voters: %v\n", a.Voters)
|
|
result += fmt.Sprintf("NonVoters: %v\n", a.NonVoters)
|
|
|
|
for name, zone := range a.RedundancyZones {
|
|
result += fmt.Sprintf("RedundancyZone %s: %s\n", name, &zone)
|
|
}
|
|
|
|
result += fmt.Sprintf("Upgrade: %s", a.Upgrade)
|
|
return result
|
|
}
|
|
|
|
// AutopilotServer represents the server blocks in the response of the raft
|
|
// autopilot state API.
|
|
type AutopilotServer struct {
|
|
ID string `mapstructure:"id"`
|
|
Name string `mapstructure:"name"`
|
|
Address string `mapstructure:"address"`
|
|
NodeStatus string `mapstructure:"node_status"`
|
|
LastContact string `mapstructure:"last_contact"`
|
|
LastTerm uint64 `mapstructure:"last_term"`
|
|
LastIndex uint64 `mapstructure:"last_index"`
|
|
Healthy bool `mapstructure:"healthy"`
|
|
StableSince string `mapstructure:"stable_since"`
|
|
Status string `mapstructure:"status"`
|
|
Version string `mapstructure:"version"`
|
|
UpgradeVersion string `mapstructure:"upgrade_version,omitempty"`
|
|
RedundancyZone string `mapstructure:"redundancy_zone,omitempty"`
|
|
NodeType string `mapstructure:"node_type,omitempty"`
|
|
}
|
|
|
|
func (a *AutopilotServer) String() string {
|
|
return fmt.Sprintf("ID: %s. Name: %s. Address: %s. NodeStatus: %s. LastContact: %s. LastTerm: %d. LastIndex: %d. Healthy: %t. StableSince: %s. Status: %s. Version: %s. UpgradeVersion: %s. RedundancyZone: %s. NodeType: %s",
|
|
a.ID, a.Name, a.Address, a.NodeStatus, a.LastContact, a.LastTerm, a.LastIndex, a.Healthy, a.StableSince, a.Status, a.Version, a.UpgradeVersion, a.RedundancyZone, a.NodeType)
|
|
}
|
|
|
|
type AutopilotZone struct {
|
|
Servers []string `mapstructure:"servers,omitempty"`
|
|
Voters []string `mapstructure:"voters,omitempty"`
|
|
FailureTolerance int `mapstructure:"failure_tolerance,omitempty"`
|
|
}
|
|
|
|
func (a *AutopilotZone) String() string {
|
|
return fmt.Sprintf("Servers: %v. Voters: %v. FailureTolerance: %d", a.Servers, a.Voters, a.FailureTolerance)
|
|
}
|
|
|
|
type AutopilotUpgrade struct {
|
|
Status string `mapstructure:"status"`
|
|
TargetVersion string `mapstructure:"target_version,omitempty"`
|
|
TargetVersionVoters []string `mapstructure:"target_version_voters,omitempty"`
|
|
TargetVersionNonVoters []string `mapstructure:"target_version_non_voters,omitempty"`
|
|
TargetVersionReadReplicas []string `mapstructure:"target_version_read_replicas,omitempty"`
|
|
OtherVersionVoters []string `mapstructure:"other_version_voters,omitempty"`
|
|
OtherVersionNonVoters []string `mapstructure:"other_version_non_voters,omitempty"`
|
|
OtherVersionReadReplicas []string `mapstructure:"other_version_read_replicas,omitempty"`
|
|
RedundancyZones map[string]AutopilotZoneUpgradeVersions `mapstructure:"redundancy_zones,omitempty"`
|
|
}
|
|
|
|
func (a *AutopilotUpgrade) String() string {
|
|
result := fmt.Sprintf("Status: %s. TargetVersion: %s. TargetVersionVoters: %v. TargetVersionNonVoters: %v. TargetVersionReadReplicas: %v. OtherVersionVoters: %v. OtherVersionNonVoters: %v. OtherVersionReadReplicas: %v",
|
|
a.Status, a.TargetVersion, a.TargetVersionVoters, a.TargetVersionNonVoters, a.TargetVersionReadReplicas, a.OtherVersionVoters, a.OtherVersionNonVoters, a.OtherVersionReadReplicas)
|
|
|
|
for name, zone := range a.RedundancyZones {
|
|
result += fmt.Sprintf("Redundancy Zone %s: %s", name, zone)
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
type AutopilotZoneUpgradeVersions struct {
|
|
TargetVersionVoters []string `mapstructure:"target_version_voters,omitempty"`
|
|
TargetVersionNonVoters []string `mapstructure:"target_version_non_voters,omitempty"`
|
|
OtherVersionVoters []string `mapstructure:"other_version_voters,omitempty"`
|
|
OtherVersionNonVoters []string `mapstructure:"other_version_non_voters,omitempty"`
|
|
}
|
|
|
|
func (a *AutopilotZoneUpgradeVersions) String() string {
|
|
return fmt.Sprintf("TargetVersionVoters: %v. TargetVersionNonVoters: %v. OtherVersionVoters: %v. OtherVersionNonVoters: %v",
|
|
a.TargetVersionVoters, a.TargetVersionNonVoters, a.OtherVersionVoters, a.OtherVersionNonVoters)
|
|
}
|
|
|
|
// RaftJoin wraps RaftJoinWithContext using context.Background.
|
|
func (c *Sys) RaftJoin(opts *RaftJoinRequest) (*RaftJoinResponse, error) {
|
|
return c.RaftJoinWithContext(context.Background(), opts)
|
|
}
|
|
|
|
// RaftJoinWithContext adds the node from which this call is invoked from to the raft
|
|
// cluster represented by the leader address in the parameter.
|
|
func (c *Sys) RaftJoinWithContext(ctx context.Context, opts *RaftJoinRequest) (*RaftJoinResponse, error) {
|
|
ctx, cancelFunc := c.c.withConfiguredTimeout(ctx)
|
|
defer cancelFunc()
|
|
|
|
r := c.c.NewRequest(http.MethodPost, "/v1/sys/storage/raft/join")
|
|
|
|
if err := r.SetJSONBody(opts); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
resp, err := c.c.rawRequestWithContext(ctx, r)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
var result RaftJoinResponse
|
|
err = resp.DecodeJSON(&result)
|
|
return &result, err
|
|
}
|
|
|
|
// RaftSnapshot wraps RaftSnapshotWithContext using context.Background.
|
|
func (c *Sys) RaftSnapshot(snapWriter io.Writer) error {
|
|
return c.RaftSnapshotWithContext(context.Background(), snapWriter)
|
|
}
|
|
|
|
// RaftSnapshotWithContext invokes the API that takes the snapshot of the raft cluster and
|
|
// writes it to the supplied io.Writer.
|
|
func (c *Sys) RaftSnapshotWithContext(ctx context.Context, snapWriter io.Writer) error {
|
|
r := c.c.NewRequest(http.MethodGet, "/v1/sys/storage/raft/snapshot")
|
|
r.URL.RawQuery = r.Params.Encode()
|
|
|
|
resp, err := c.c.httpRequestWithContext(ctx, r)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
// Make sure that the last file in the archive, SHA256SUMS.sealed, is present
|
|
// and non-empty. This is to catch cases where the snapshot failed midstream,
|
|
// e.g. due to a problem with the seal that prevented encryption of that file.
|
|
var wg sync.WaitGroup
|
|
wg.Add(1)
|
|
var verified bool
|
|
|
|
rPipe, wPipe := io.Pipe()
|
|
dup := io.TeeReader(resp.Body, wPipe)
|
|
go func() {
|
|
defer func() {
|
|
io.Copy(ioutil.Discard, rPipe)
|
|
rPipe.Close()
|
|
wg.Done()
|
|
}()
|
|
|
|
uncompressed, err := gzip.NewReader(rPipe)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
t := tar.NewReader(uncompressed)
|
|
var h *tar.Header
|
|
for {
|
|
h, err = t.Next()
|
|
if err != nil {
|
|
return
|
|
}
|
|
if h.Name != "SHA256SUMS.sealed" {
|
|
continue
|
|
}
|
|
var b []byte
|
|
b, err = ioutil.ReadAll(t)
|
|
if err != nil || len(b) == 0 {
|
|
return
|
|
}
|
|
verified = true
|
|
return
|
|
}
|
|
}()
|
|
|
|
// Copy bytes from dup to snapWriter. This will have a side effect that
|
|
// everything read from dup will be written to wPipe.
|
|
_, err = io.Copy(snapWriter, dup)
|
|
wPipe.Close()
|
|
if err != nil {
|
|
rPipe.CloseWithError(err)
|
|
return err
|
|
}
|
|
wg.Wait()
|
|
|
|
if !verified {
|
|
return ErrIncompleteSnapshot
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// RaftSnapshotRestore wraps RaftSnapshotRestoreWithContext using context.Background.
|
|
func (c *Sys) RaftSnapshotRestore(snapReader io.Reader, force bool) error {
|
|
return c.RaftSnapshotRestoreWithContext(context.Background(), snapReader, force)
|
|
}
|
|
|
|
// RaftSnapshotRestoreWithContext reads the snapshot from the io.Reader and installs that
|
|
// snapshot, returning the cluster to the state defined by it.
|
|
func (c *Sys) RaftSnapshotRestoreWithContext(ctx context.Context, snapReader io.Reader, force bool) error {
|
|
path := "/v1/sys/storage/raft/snapshot"
|
|
if force {
|
|
path = "/v1/sys/storage/raft/snapshot-force"
|
|
}
|
|
|
|
r := c.c.NewRequest(http.MethodPost, path)
|
|
r.Body = snapReader
|
|
|
|
resp, err := c.c.httpRequestWithContext(ctx, r)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
return nil
|
|
}
|
|
|
|
// RaftAutopilotState wraps RaftAutopilotStateWithContext using context.Background.
|
|
func (c *Sys) RaftAutopilotState() (*AutopilotState, error) {
|
|
return c.RaftAutopilotStateWithContext(context.Background())
|
|
}
|
|
|
|
// RaftAutopilotStateWithToken wraps RaftAutopilotStateWithContext using the given token.
|
|
func (c *Sys) RaftAutopilotStateWithDRToken(drToken string) (*AutopilotState, error) {
|
|
return c.RaftAutopilotStateWithContext(context.WithValue(context.Background(), "dr-token", drToken))
|
|
}
|
|
|
|
// RaftAutopilotStateWithContext returns the state of the raft cluster as seen by autopilot.
|
|
func (c *Sys) RaftAutopilotStateWithContext(ctx context.Context) (*AutopilotState, error) {
|
|
ctx, cancelFunc := c.c.withConfiguredTimeout(ctx)
|
|
defer cancelFunc()
|
|
|
|
if ctx.Value("dr-token") != nil {
|
|
c.c.SetToken(ctx.Value("dr-token").(string))
|
|
}
|
|
r := c.c.NewRequest(http.MethodGet, "/v1/sys/storage/raft/autopilot/state")
|
|
|
|
resp, err := c.c.rawRequestWithContext(ctx, r)
|
|
if resp != nil {
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode == 404 {
|
|
return nil, nil
|
|
}
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
secret, err := ParseSecret(resp.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if secret == nil || secret.Data == nil {
|
|
return nil, errors.New("data from server response is empty")
|
|
}
|
|
|
|
var result AutopilotState
|
|
err = mapstructure.Decode(secret.Data, &result)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &result, err
|
|
}
|
|
|
|
// RaftAutopilotConfiguration wraps RaftAutopilotConfigurationWithContext using context.Background.
|
|
func (c *Sys) RaftAutopilotConfiguration() (*AutopilotConfig, error) {
|
|
return c.RaftAutopilotConfigurationWithContext(context.Background())
|
|
}
|
|
|
|
// RaftAutopilotConfigurationWithDRToken wraps RaftAutopilotConfigurationWithContext using the given token.
|
|
func (c *Sys) RaftAutopilotConfigurationWithDRToken(drToken string) (*AutopilotConfig, error) {
|
|
return c.RaftAutopilotConfigurationWithContext(context.WithValue(context.Background(), "dr-token", drToken))
|
|
}
|
|
|
|
// RaftAutopilotConfigurationWithContext fetches the autopilot config.
|
|
func (c *Sys) RaftAutopilotConfigurationWithContext(ctx context.Context) (*AutopilotConfig, error) {
|
|
ctx, cancelFunc := c.c.withConfiguredTimeout(ctx)
|
|
defer cancelFunc()
|
|
|
|
if ctx.Value("dr-token") != nil {
|
|
c.c.SetToken(ctx.Value("dr-token").(string))
|
|
}
|
|
|
|
r := c.c.NewRequest(http.MethodGet, "/v1/sys/storage/raft/autopilot/configuration")
|
|
|
|
resp, err := c.c.rawRequestWithContext(ctx, r)
|
|
if resp != nil {
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode == 404 {
|
|
return nil, nil
|
|
}
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
secret, err := ParseSecret(resp.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if secret == nil {
|
|
return nil, errors.New("data from server response is empty")
|
|
}
|
|
|
|
var result AutopilotConfig
|
|
if err = mapstructure.Decode(secret.Data, &result); err != nil {
|
|
return nil, err
|
|
}
|
|
if result.LastContactThreshold, err = parseutil.ParseDurationSecond(secret.Data["last_contact_threshold"]); err != nil {
|
|
return nil, err
|
|
}
|
|
if result.DeadServerLastContactThreshold, err = parseutil.ParseDurationSecond(secret.Data["dead_server_last_contact_threshold"]); err != nil {
|
|
return nil, err
|
|
}
|
|
if result.ServerStabilizationTime, err = parseutil.ParseDurationSecond(secret.Data["server_stabilization_time"]); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &result, err
|
|
}
|
|
|
|
// PutRaftAutopilotConfiguration wraps PutRaftAutopilotConfigurationWithContext using context.Background.
|
|
func (c *Sys) PutRaftAutopilotConfiguration(opts *AutopilotConfig) error {
|
|
return c.PutRaftAutopilotConfigurationWithContext(context.Background(), opts)
|
|
}
|
|
|
|
// PutRaftAutopilotConfigurationWithContext allows modifying the raft autopilot configuration
|
|
func (c *Sys) PutRaftAutopilotConfigurationWithContext(ctx context.Context, opts *AutopilotConfig) error {
|
|
ctx, cancelFunc := c.c.withConfiguredTimeout(ctx)
|
|
defer cancelFunc()
|
|
|
|
r := c.c.NewRequest(http.MethodPost, "/v1/sys/storage/raft/autopilot/configuration")
|
|
|
|
if err := r.SetJSONBody(opts); err != nil {
|
|
return err
|
|
}
|
|
|
|
resp, err := c.c.rawRequestWithContext(ctx, r)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
return nil
|
|
}
|