2023-03-15 19:00:52 +03:00
|
|
|
// Copyright (c) HashiCorp, Inc.
|
2024-01-02 21:36:20 +03:00
|
|
|
// SPDX-License-Identifier: BUSL-1.1
|
2023-03-15 19:00:52 +03:00
|
|
|
|
2018-09-18 06:03:00 +03:00
|
|
|
package vault
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"net/http"
|
2021-12-10 17:59:52 +03:00
|
|
|
"os"
|
2022-11-30 20:59:05 +03:00
|
|
|
"runtime/debug"
|
2018-09-18 06:03:00 +03:00
|
|
|
"sync/atomic"
|
|
|
|
"time"
|
|
|
|
|
2022-10-07 19:09:08 +03:00
|
|
|
"github.com/armon/go-metrics"
|
2018-09-18 06:03:00 +03:00
|
|
|
"github.com/hashicorp/vault/helper/forwarding"
|
2021-03-03 21:59:50 +03:00
|
|
|
"github.com/hashicorp/vault/physical/raft"
|
2022-05-20 23:49:11 +03:00
|
|
|
"github.com/hashicorp/vault/sdk/helper/consts"
|
2019-02-15 05:14:56 +03:00
|
|
|
"github.com/hashicorp/vault/vault/replication"
|
2018-09-18 06:03:00 +03:00
|
|
|
)
|
|
|
|
|
|
|
|
type forwardedRequestRPCServer struct {
|
2021-09-30 04:25:15 +03:00
|
|
|
UnimplementedRequestForwardingServer
|
|
|
|
|
2018-09-18 06:03:00 +03:00
|
|
|
core *Core
|
|
|
|
handler http.Handler
|
|
|
|
perfStandbySlots chan struct{}
|
2019-02-15 05:14:56 +03:00
|
|
|
perfStandbyRepCluster *replication.Cluster
|
2021-03-03 21:59:50 +03:00
|
|
|
raftFollowerStates *raft.FollowerStates
|
2018-09-18 06:03:00 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
func (s *forwardedRequestRPCServer) ForwardRequest(ctx context.Context, freq *forwarding.Request) (*forwarding.Response, error) {
|
|
|
|
// Parse an http.Request out of it
|
|
|
|
req, err := forwarding.ParseForwardedRequest(freq)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// A very dummy response writer that doesn't follow normal semantics, just
|
|
|
|
// lets you write a status code (last written wins) and a body. But it
|
|
|
|
// meets the interface requirements.
|
|
|
|
w := forwarding.NewRPCResponseWriter()
|
|
|
|
|
|
|
|
resp := &forwarding.Response{}
|
|
|
|
|
|
|
|
runRequest := func() {
|
|
|
|
defer func() {
|
|
|
|
if err := recover(); err != nil {
|
2022-11-30 20:59:05 +03:00
|
|
|
s.core.logger.Error("panic serving forwarded request", "path", req.URL.Path, "error", err, "stacktrace", string(debug.Stack()))
|
2018-09-18 06:03:00 +03:00
|
|
|
}
|
|
|
|
}()
|
|
|
|
s.handler.ServeHTTP(w, req)
|
|
|
|
}
|
|
|
|
runRequest()
|
|
|
|
resp.StatusCode = uint32(w.StatusCode())
|
|
|
|
resp.Body = w.Body().Bytes()
|
|
|
|
|
|
|
|
header := w.Header()
|
|
|
|
if header != nil {
|
|
|
|
resp.HeaderEntries = make(map[string]*forwarding.HeaderEntry, len(header))
|
|
|
|
for k, v := range header {
|
|
|
|
resp.HeaderEntries[k] = &forwarding.HeaderEntry{
|
|
|
|
Values: v,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-02-19 23:03:02 +03:00
|
|
|
// Performance standby nodes will use this value to do wait for WALs to ship
|
2019-07-19 04:10:15 +03:00
|
|
|
// in order to do a best-effort read after write guarantee
|
2019-02-19 23:03:02 +03:00
|
|
|
resp.LastRemoteWal = LastWAL(s.core)
|
2018-09-18 06:03:00 +03:00
|
|
|
|
|
|
|
return resp, nil
|
|
|
|
}
|
|
|
|
|
2021-11-30 22:49:58 +03:00
|
|
|
type nodeHAConnectionInfo struct {
|
2022-05-20 23:49:11 +03:00
|
|
|
nodeInfo *NodeInformation
|
|
|
|
lastHeartbeat time.Time
|
|
|
|
version string
|
|
|
|
upgradeVersion string
|
|
|
|
redundancyZone string
|
2021-11-30 22:49:58 +03:00
|
|
|
}
|
|
|
|
|
2018-09-18 06:03:00 +03:00
|
|
|
func (s *forwardedRequestRPCServer) Echo(ctx context.Context, in *EchoRequest) (*EchoReply, error) {
|
2021-11-30 22:49:58 +03:00
|
|
|
incomingNodeConnectionInfo := nodeHAConnectionInfo{
|
2022-05-20 23:49:11 +03:00
|
|
|
nodeInfo: in.NodeInfo,
|
|
|
|
lastHeartbeat: time.Now(),
|
|
|
|
version: in.SdkVersion,
|
|
|
|
upgradeVersion: in.RaftUpgradeVersion,
|
|
|
|
redundancyZone: in.RaftRedundancyZone,
|
2021-11-30 22:49:58 +03:00
|
|
|
}
|
2018-09-18 06:03:00 +03:00
|
|
|
if in.ClusterAddr != "" {
|
2021-11-30 22:49:58 +03:00
|
|
|
s.core.clusterPeerClusterAddrsCache.Set(in.ClusterAddr, incomingNodeConnectionInfo, 0)
|
2018-09-18 06:03:00 +03:00
|
|
|
}
|
2019-06-20 22:14:58 +03:00
|
|
|
|
2019-06-27 20:00:03 +03:00
|
|
|
if in.RaftAppliedIndex > 0 && len(in.RaftNodeID) > 0 && s.raftFollowerStates != nil {
|
2022-05-20 23:49:11 +03:00
|
|
|
s.raftFollowerStates.Update(&raft.EchoRequestUpdate{
|
|
|
|
NodeID: in.RaftNodeID,
|
|
|
|
AppliedIndex: in.RaftAppliedIndex,
|
|
|
|
Term: in.RaftTerm,
|
|
|
|
DesiredSuffrage: in.RaftDesiredSuffrage,
|
2022-06-06 18:44:24 +03:00
|
|
|
SDKVersion: in.SdkVersion,
|
2022-05-20 23:49:11 +03:00
|
|
|
UpgradeVersion: in.RaftUpgradeVersion,
|
|
|
|
RedundancyZone: in.RaftRedundancyZone,
|
|
|
|
})
|
2019-06-20 22:14:58 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
reply := &EchoReply{
|
2018-09-18 06:03:00 +03:00
|
|
|
Message: "pong",
|
|
|
|
ReplicationState: uint32(s.core.ReplicationState()),
|
2019-06-20 22:14:58 +03:00
|
|
|
}
|
|
|
|
|
2020-06-30 01:36:22 +03:00
|
|
|
if raftBackend := s.core.getRaftBackend(); raftBackend != nil {
|
2021-04-12 16:33:21 +03:00
|
|
|
reply.RaftAppliedIndex = raftBackend.AppliedIndex()
|
|
|
|
reply.RaftNodeID = raftBackend.NodeID()
|
2019-06-20 22:14:58 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
return reply, nil
|
2018-09-18 06:03:00 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
type forwardingClient struct {
|
|
|
|
RequestForwardingClient
|
2022-05-20 23:49:11 +03:00
|
|
|
core *Core
|
2018-09-18 06:03:00 +03:00
|
|
|
echoTicker *time.Ticker
|
|
|
|
echoContext context.Context
|
|
|
|
}
|
|
|
|
|
|
|
|
// NOTE: we also take advantage of gRPC's keepalive bits, but as we send data
|
|
|
|
// with these requests it's useful to keep this as well
|
|
|
|
func (c *forwardingClient) startHeartbeat() {
|
|
|
|
go func() {
|
2021-11-30 22:49:58 +03:00
|
|
|
clusterAddr := c.core.ClusterAddr()
|
2021-12-10 17:59:52 +03:00
|
|
|
hostname, _ := os.Hostname()
|
2021-11-30 22:49:58 +03:00
|
|
|
ni := NodeInformation{
|
|
|
|
ApiAddr: c.core.redirectAddr,
|
2021-12-10 17:59:52 +03:00
|
|
|
Hostname: hostname,
|
2021-11-30 22:49:58 +03:00
|
|
|
Mode: "standby",
|
|
|
|
}
|
2018-09-18 06:03:00 +03:00
|
|
|
tick := func() {
|
2022-10-07 19:09:08 +03:00
|
|
|
labels := make([]metrics.Label, 0, 1)
|
|
|
|
defer metrics.MeasureSinceWithLabels([]string{"ha", "rpc", "client", "echo"}, time.Now(), labels)
|
|
|
|
|
2019-06-20 22:14:58 +03:00
|
|
|
req := &EchoRequest{
|
2018-09-18 06:03:00 +03:00
|
|
|
Message: "ping",
|
|
|
|
ClusterAddr: clusterAddr,
|
2021-11-30 22:49:58 +03:00
|
|
|
NodeInfo: &ni,
|
2022-10-06 21:24:16 +03:00
|
|
|
SdkVersion: c.core.effectiveSDKVersion,
|
2019-06-20 22:14:58 +03:00
|
|
|
}
|
|
|
|
|
2020-06-30 01:36:22 +03:00
|
|
|
if raftBackend := c.core.getRaftBackend(); raftBackend != nil {
|
2021-04-12 16:33:21 +03:00
|
|
|
req.RaftAppliedIndex = raftBackend.AppliedIndex()
|
|
|
|
req.RaftNodeID = raftBackend.NodeID()
|
|
|
|
req.RaftTerm = raftBackend.Term()
|
|
|
|
req.RaftDesiredSuffrage = raftBackend.DesiredSuffrage()
|
2022-05-20 23:49:11 +03:00
|
|
|
req.RaftRedundancyZone = raftBackend.RedundancyZone()
|
2024-04-17 00:36:25 +03:00
|
|
|
req.RaftUpgradeVersion = raftBackend.UpgradeVersion()
|
2022-10-07 19:09:08 +03:00
|
|
|
labels = append(labels, metrics.Label{Name: "peer_id", Value: raftBackend.NodeID()})
|
2019-06-20 22:14:58 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
ctx, cancel := context.WithTimeout(c.echoContext, 2*time.Second)
|
|
|
|
resp, err := c.RequestForwardingClient.Echo(ctx, req)
|
2018-09-18 06:03:00 +03:00
|
|
|
cancel()
|
|
|
|
if err != nil {
|
2022-10-07 19:09:08 +03:00
|
|
|
metrics.IncrCounter([]string{"ha", "rpc", "client", "echo", "errors"}, 1)
|
2018-09-18 06:03:00 +03:00
|
|
|
c.core.logger.Debug("forwarding: error sending echo request to active node", "error", err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if resp == nil {
|
|
|
|
c.core.logger.Debug("forwarding: empty echo response from active node")
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if resp.Message != "pong" {
|
|
|
|
c.core.logger.Debug("forwarding: unexpected echo response from active node", "message", resp.Message)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
// Store the active node's replication state to display in
|
|
|
|
// sys/health calls
|
|
|
|
atomic.StoreUint32(c.core.activeNodeReplicationState, resp.ReplicationState)
|
|
|
|
}
|
|
|
|
|
|
|
|
tick()
|
|
|
|
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-c.echoContext.Done():
|
|
|
|
c.echoTicker.Stop()
|
|
|
|
c.core.logger.Debug("forwarding: stopping heartbeating")
|
|
|
|
atomic.StoreUint32(c.core.activeNodeReplicationState, uint32(consts.ReplicationUnknown))
|
|
|
|
return
|
|
|
|
case <-c.echoTicker.C:
|
|
|
|
tick()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
}
|