2023-03-15 19:00:52 +03:00
|
|
|
// Copyright (c) HashiCorp, Inc.
|
|
|
|
// SPDX-License-Identifier: MPL-2.0
|
|
|
|
|
2019-10-29 03:28:59 +03:00
|
|
|
package api
|
|
|
|
|
|
|
|
import (
|
|
|
|
"errors"
|
|
|
|
"math/rand"
|
|
|
|
"sync"
|
|
|
|
"time"
|
2021-05-06 21:04:26 +03:00
|
|
|
|
|
|
|
"github.com/cenkalti/backoff/v3"
|
2019-10-29 03:28:59 +03:00
|
|
|
)
|
|
|
|
|
|
|
|
var (
|
|
|
|
ErrLifetimeWatcherMissingInput = errors.New("missing input")
|
|
|
|
ErrLifetimeWatcherMissingSecret = errors.New("missing secret")
|
|
|
|
ErrLifetimeWatcherNotRenewable = errors.New("secret is not renewable")
|
|
|
|
ErrLifetimeWatcherNoSecretData = errors.New("returned empty secret data")
|
|
|
|
|
|
|
|
// Deprecated; kept for compatibility
|
|
|
|
ErrRenewerMissingInput = errors.New("missing input to renewer")
|
|
|
|
ErrRenewerMissingSecret = errors.New("missing secret to renew")
|
|
|
|
ErrRenewerNotRenewable = errors.New("secret is not renewable")
|
|
|
|
ErrRenewerNoSecretData = errors.New("returned empty secret data")
|
|
|
|
|
|
|
|
// DefaultLifetimeWatcherRenewBuffer is the default size of the buffer for renew
|
|
|
|
// messages on the channel.
|
|
|
|
DefaultLifetimeWatcherRenewBuffer = 5
|
|
|
|
// Deprecated: kept for backwards compatibility
|
|
|
|
DefaultRenewerRenewBuffer = 5
|
|
|
|
)
|
|
|
|
|
|
|
|
type RenewBehavior uint
|
|
|
|
|
|
|
|
const (
|
|
|
|
// RenewBehaviorIgnoreErrors means we will attempt to keep renewing until
|
|
|
|
// we hit the lifetime threshold. It also ignores errors stemming from
|
|
|
|
// passing a non-renewable lease in. In practice, this means you simply
|
|
|
|
// reauthenticate/refetch credentials when the watcher exits. This is the
|
|
|
|
// default.
|
|
|
|
RenewBehaviorIgnoreErrors RenewBehavior = iota
|
|
|
|
|
|
|
|
// RenewBehaviorRenewDisabled turns off renewal attempts entirely. This
|
|
|
|
// allows you to simply watch lifetime and have the watcher return at a
|
|
|
|
// reasonable threshold without actually making Vault calls.
|
|
|
|
RenewBehaviorRenewDisabled
|
|
|
|
|
|
|
|
// RenewBehaviorErrorOnErrors is the "legacy" behavior which always exits
|
|
|
|
// on some kind of error
|
|
|
|
RenewBehaviorErrorOnErrors
|
|
|
|
)
|
|
|
|
|
|
|
|
// LifetimeWatcher is a process for watching lifetime of a secret.
|
|
|
|
//
|
2022-09-08 03:31:20 +03:00
|
|
|
// watcher, err := client.NewLifetimeWatcher(&LifetimeWatcherInput{
|
|
|
|
// Secret: mySecret,
|
|
|
|
// })
|
|
|
|
// go watcher.Start()
|
|
|
|
// defer watcher.Stop()
|
2019-10-29 03:28:59 +03:00
|
|
|
//
|
2022-09-08 03:31:20 +03:00
|
|
|
// for {
|
|
|
|
// select {
|
|
|
|
// case err := <-watcher.DoneCh():
|
|
|
|
// if err != nil {
|
|
|
|
// log.Fatal(err)
|
|
|
|
// }
|
2019-10-29 03:28:59 +03:00
|
|
|
//
|
2022-09-08 03:31:20 +03:00
|
|
|
// // Renewal is now over
|
|
|
|
// case renewal := <-watcher.RenewCh():
|
|
|
|
// log.Printf("Successfully renewed: %#v", renewal)
|
|
|
|
// }
|
|
|
|
// }
|
2019-10-29 03:28:59 +03:00
|
|
|
//
|
|
|
|
// `DoneCh` will return if renewal fails, or if the remaining lease duration is
|
|
|
|
// under a built-in threshold and either renewing is not extending it or
|
|
|
|
// renewing is disabled. In both cases, the caller should attempt a re-read of
|
|
|
|
// the secret. Clients should check the return value of the channel to see if
|
|
|
|
// renewal was successful.
|
|
|
|
type LifetimeWatcher struct {
|
|
|
|
l sync.Mutex
|
|
|
|
|
|
|
|
client *Client
|
|
|
|
secret *Secret
|
|
|
|
grace time.Duration
|
|
|
|
random *rand.Rand
|
|
|
|
increment int
|
|
|
|
doneCh chan error
|
|
|
|
renewCh chan *RenewOutput
|
|
|
|
renewBehavior RenewBehavior
|
|
|
|
|
|
|
|
stopped bool
|
|
|
|
stopCh chan struct{}
|
|
|
|
|
|
|
|
errLifetimeWatcherNotRenewable error
|
|
|
|
errLifetimeWatcherNoSecretData error
|
|
|
|
}
|
|
|
|
|
|
|
|
// LifetimeWatcherInput is used as input to the renew function.
|
|
|
|
type LifetimeWatcherInput struct {
|
|
|
|
// Secret is the secret to renew
|
|
|
|
Secret *Secret
|
|
|
|
|
|
|
|
// DEPRECATED: this does not do anything.
|
|
|
|
Grace time.Duration
|
|
|
|
|
|
|
|
// Rand is the randomizer to use for underlying randomization. If not
|
|
|
|
// provided, one will be generated and seeded automatically. If provided, it
|
|
|
|
// is assumed to have already been seeded.
|
|
|
|
Rand *rand.Rand
|
|
|
|
|
|
|
|
// RenewBuffer is the size of the buffered channel where renew messages are
|
|
|
|
// dispatched.
|
|
|
|
RenewBuffer int
|
|
|
|
|
|
|
|
// The new TTL, in seconds, that should be set on the lease. The TTL set
|
|
|
|
// here may or may not be honored by the vault server, based on Vault
|
2022-04-06 20:04:45 +03:00
|
|
|
// configuration or any associated max TTL values. If specified, the
|
|
|
|
// minimum of this value and the remaining lease duration will be used
|
|
|
|
// for grace period calculations.
|
2019-10-29 03:28:59 +03:00
|
|
|
Increment int
|
|
|
|
|
|
|
|
// RenewBehavior controls what happens when a renewal errors or the
|
|
|
|
// passed-in secret is not renewable.
|
|
|
|
RenewBehavior RenewBehavior
|
|
|
|
}
|
|
|
|
|
|
|
|
// RenewOutput is the metadata returned to the client (if it's listening) to
|
|
|
|
// renew messages.
|
|
|
|
type RenewOutput struct {
|
|
|
|
// RenewedAt is the timestamp when the renewal took place (UTC).
|
|
|
|
RenewedAt time.Time
|
|
|
|
|
|
|
|
// Secret is the underlying renewal data. It's the same struct as all data
|
|
|
|
// that is returned from Vault, but since this is renewal data, it will not
|
|
|
|
// usually include the secret itself.
|
|
|
|
Secret *Secret
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewLifetimeWatcher creates a new renewer from the given input.
|
|
|
|
func (c *Client) NewLifetimeWatcher(i *LifetimeWatcherInput) (*LifetimeWatcher, error) {
|
|
|
|
if i == nil {
|
|
|
|
return nil, ErrLifetimeWatcherMissingInput
|
|
|
|
}
|
|
|
|
|
|
|
|
secret := i.Secret
|
|
|
|
if secret == nil {
|
|
|
|
return nil, ErrLifetimeWatcherMissingSecret
|
|
|
|
}
|
|
|
|
|
|
|
|
random := i.Rand
|
|
|
|
if random == nil {
|
2023-03-14 16:17:26 +03:00
|
|
|
// NOTE:
|
|
|
|
// Rather than a cryptographically secure random number generator (RNG),
|
|
|
|
// the default behavior uses the math/rand package. The random number is
|
|
|
|
// used to introduce a slight jitter when calculating the grace period
|
|
|
|
// for a monitored secret monitoring. This is intended to stagger renewal
|
|
|
|
// requests to the Vault server, but in a semi-predictable way, so there
|
|
|
|
// is no need to use a cryptographically secure RNG.
|
2019-10-29 03:28:59 +03:00
|
|
|
random = rand.New(rand.NewSource(int64(time.Now().Nanosecond())))
|
|
|
|
}
|
|
|
|
|
|
|
|
renewBuffer := i.RenewBuffer
|
|
|
|
if renewBuffer == 0 {
|
|
|
|
renewBuffer = DefaultLifetimeWatcherRenewBuffer
|
|
|
|
}
|
|
|
|
|
|
|
|
return &LifetimeWatcher{
|
|
|
|
client: c,
|
|
|
|
secret: secret,
|
|
|
|
increment: i.Increment,
|
|
|
|
random: random,
|
|
|
|
doneCh: make(chan error, 1),
|
|
|
|
renewCh: make(chan *RenewOutput, renewBuffer),
|
|
|
|
renewBehavior: i.RenewBehavior,
|
|
|
|
|
|
|
|
stopped: false,
|
|
|
|
stopCh: make(chan struct{}),
|
|
|
|
|
|
|
|
errLifetimeWatcherNotRenewable: ErrLifetimeWatcherNotRenewable,
|
|
|
|
errLifetimeWatcherNoSecretData: ErrLifetimeWatcherNoSecretData,
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Deprecated: exists only for backwards compatibility. Calls
|
|
|
|
// NewLifetimeWatcher, and sets compatibility flags.
|
|
|
|
func (c *Client) NewRenewer(i *LifetimeWatcherInput) (*LifetimeWatcher, error) {
|
|
|
|
if i == nil {
|
|
|
|
return nil, ErrRenewerMissingInput
|
|
|
|
}
|
|
|
|
|
|
|
|
secret := i.Secret
|
|
|
|
if secret == nil {
|
|
|
|
return nil, ErrRenewerMissingSecret
|
|
|
|
}
|
|
|
|
|
|
|
|
renewer, err := c.NewLifetimeWatcher(i)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
renewer.renewBehavior = RenewBehaviorErrorOnErrors
|
|
|
|
renewer.errLifetimeWatcherNotRenewable = ErrRenewerNotRenewable
|
|
|
|
renewer.errLifetimeWatcherNoSecretData = ErrRenewerNoSecretData
|
|
|
|
return renewer, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// DoneCh returns the channel where the renewer will publish when renewal stops.
|
|
|
|
// If there is an error, this will be an error.
|
|
|
|
func (r *LifetimeWatcher) DoneCh() <-chan error {
|
|
|
|
return r.doneCh
|
|
|
|
}
|
|
|
|
|
|
|
|
// RenewCh is a channel that receives a message when a successful renewal takes
|
|
|
|
// place and includes metadata about the renewal.
|
|
|
|
func (r *LifetimeWatcher) RenewCh() <-chan *RenewOutput {
|
|
|
|
return r.renewCh
|
|
|
|
}
|
|
|
|
|
|
|
|
// Stop stops the renewer.
|
|
|
|
func (r *LifetimeWatcher) Stop() {
|
|
|
|
r.l.Lock()
|
|
|
|
defer r.l.Unlock()
|
|
|
|
|
|
|
|
if !r.stopped {
|
|
|
|
close(r.stopCh)
|
|
|
|
r.stopped = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Start starts a background process for watching the lifetime of this secret.
|
|
|
|
// If renewal is enabled, when the secret has auth data, this attempts to renew
|
|
|
|
// the auth (token); When the secret has a lease, this attempts to renew the
|
|
|
|
// lease.
|
|
|
|
func (r *LifetimeWatcher) Start() {
|
|
|
|
r.doneCh <- r.doRenew()
|
|
|
|
}
|
|
|
|
|
2022-02-18 04:10:26 +03:00
|
|
|
// Renew is for compatibility with the legacy api.Renewer. Calling Renew
|
2019-10-29 03:28:59 +03:00
|
|
|
// simply chains to Start.
|
|
|
|
func (r *LifetimeWatcher) Renew() {
|
|
|
|
r.Start()
|
|
|
|
}
|
|
|
|
|
2021-05-06 21:04:26 +03:00
|
|
|
type renewFunc func(string, int) (*Secret, error)
|
2019-10-29 03:28:59 +03:00
|
|
|
|
2021-05-06 21:04:26 +03:00
|
|
|
// doRenew is a helper for renewing authentication.
|
|
|
|
func (r *LifetimeWatcher) doRenew() error {
|
|
|
|
defaultInitialRetryInterval := 10 * time.Second
|
2019-10-29 03:28:59 +03:00
|
|
|
switch {
|
|
|
|
case r.secret.Auth != nil:
|
2021-05-06 21:04:26 +03:00
|
|
|
return r.doRenewWithOptions(true, !r.secret.Auth.Renewable,
|
|
|
|
r.secret.Auth.LeaseDuration, r.secret.Auth.ClientToken,
|
|
|
|
r.client.Auth().Token().RenewTokenAsSelf, defaultInitialRetryInterval)
|
2019-10-29 03:28:59 +03:00
|
|
|
default:
|
2021-05-06 21:04:26 +03:00
|
|
|
return r.doRenewWithOptions(false, !r.secret.Renewable,
|
|
|
|
r.secret.LeaseDuration, r.secret.LeaseID,
|
|
|
|
r.client.Sys().Renew, defaultInitialRetryInterval)
|
2019-10-29 03:28:59 +03:00
|
|
|
}
|
2021-05-06 21:04:26 +03:00
|
|
|
}
|
2019-10-29 03:28:59 +03:00
|
|
|
|
2021-05-06 21:04:26 +03:00
|
|
|
func (r *LifetimeWatcher) doRenewWithOptions(tokenMode bool, nonRenewable bool, initLeaseDuration int, credString string,
|
2022-09-08 03:31:20 +03:00
|
|
|
renew renewFunc, initialRetryInterval time.Duration,
|
|
|
|
) error {
|
2019-10-29 03:28:59 +03:00
|
|
|
if credString == "" ||
|
|
|
|
(nonRenewable && r.renewBehavior == RenewBehaviorErrorOnErrors) {
|
|
|
|
return r.errLifetimeWatcherNotRenewable
|
|
|
|
}
|
|
|
|
|
|
|
|
initialTime := time.Now()
|
|
|
|
priorDuration := time.Duration(initLeaseDuration) * time.Second
|
2022-04-06 20:04:45 +03:00
|
|
|
r.calculateGrace(priorDuration, time.Duration(r.increment)*time.Second)
|
2021-05-06 21:04:26 +03:00
|
|
|
var errorBackoff backoff.BackOff
|
2019-10-29 03:28:59 +03:00
|
|
|
|
|
|
|
for {
|
|
|
|
// Check if we are stopped.
|
|
|
|
select {
|
|
|
|
case <-r.stopCh:
|
|
|
|
return nil
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
|
2021-05-06 21:04:26 +03:00
|
|
|
var remainingLeaseDuration time.Duration
|
2019-10-29 03:28:59 +03:00
|
|
|
fallbackLeaseDuration := initialTime.Add(priorDuration).Sub(time.Now())
|
2021-05-06 21:04:26 +03:00
|
|
|
var renewal *Secret
|
|
|
|
var err error
|
2019-10-29 03:28:59 +03:00
|
|
|
|
|
|
|
switch {
|
|
|
|
case nonRenewable || r.renewBehavior == RenewBehaviorRenewDisabled:
|
|
|
|
// Can't or won't renew, just keep the same expiration so we exit
|
|
|
|
// when it's reauthentication time
|
2021-05-06 21:04:26 +03:00
|
|
|
remainingLeaseDuration = fallbackLeaseDuration
|
2019-10-29 03:28:59 +03:00
|
|
|
|
|
|
|
default:
|
|
|
|
// Renew the token
|
2021-05-06 21:04:26 +03:00
|
|
|
renewal, err = renew(credString, r.increment)
|
2019-10-29 03:28:59 +03:00
|
|
|
if err != nil || renewal == nil || (tokenMode && renewal.Auth == nil) {
|
|
|
|
if r.renewBehavior == RenewBehaviorErrorOnErrors {
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if renewal == nil || (tokenMode && renewal.Auth == nil) {
|
|
|
|
return r.errLifetimeWatcherNoSecretData
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-05-06 21:04:26 +03:00
|
|
|
// Calculate remaining duration until initial token lease expires
|
|
|
|
remainingLeaseDuration = initialTime.Add(time.Duration(initLeaseDuration) * time.Second).Sub(time.Now())
|
|
|
|
if errorBackoff == nil {
|
|
|
|
errorBackoff = &backoff.ExponentialBackOff{
|
|
|
|
MaxElapsedTime: remainingLeaseDuration,
|
|
|
|
RandomizationFactor: backoff.DefaultRandomizationFactor,
|
|
|
|
InitialInterval: initialRetryInterval,
|
|
|
|
MaxInterval: 5 * time.Minute,
|
|
|
|
Multiplier: 2,
|
|
|
|
Clock: backoff.SystemClock,
|
|
|
|
}
|
|
|
|
errorBackoff.Reset()
|
|
|
|
}
|
2019-10-29 03:28:59 +03:00
|
|
|
break
|
|
|
|
}
|
2021-05-06 21:04:26 +03:00
|
|
|
errorBackoff = nil
|
2019-10-29 03:28:59 +03:00
|
|
|
|
|
|
|
// Push a message that a renewal took place.
|
|
|
|
select {
|
|
|
|
case r.renewCh <- &RenewOutput{time.Now().UTC(), renewal}:
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
|
|
|
|
// Possibly error if we are not renewable
|
|
|
|
if ((tokenMode && !renewal.Auth.Renewable) || (!tokenMode && !renewal.Renewable)) &&
|
|
|
|
r.renewBehavior == RenewBehaviorErrorOnErrors {
|
|
|
|
return r.errLifetimeWatcherNotRenewable
|
|
|
|
}
|
|
|
|
|
2021-05-06 21:04:26 +03:00
|
|
|
// Reset initial time
|
|
|
|
initialTime = time.Now()
|
|
|
|
|
2019-10-29 03:28:59 +03:00
|
|
|
// Grab the lease duration
|
2021-05-06 21:04:26 +03:00
|
|
|
initLeaseDuration = renewal.LeaseDuration
|
2019-10-29 03:28:59 +03:00
|
|
|
if tokenMode {
|
2021-05-06 21:04:26 +03:00
|
|
|
initLeaseDuration = renewal.Auth.LeaseDuration
|
2019-10-29 03:28:59 +03:00
|
|
|
}
|
|
|
|
|
2021-05-06 21:04:26 +03:00
|
|
|
remainingLeaseDuration = time.Duration(initLeaseDuration) * time.Second
|
2019-10-29 03:28:59 +03:00
|
|
|
}
|
|
|
|
|
2021-05-06 21:04:26 +03:00
|
|
|
var sleepDuration time.Duration
|
2019-10-29 03:28:59 +03:00
|
|
|
|
2023-02-14 17:57:25 +03:00
|
|
|
if errorBackoff == nil {
|
|
|
|
sleepDuration = r.calculateSleepDuration(remainingLeaseDuration, priorDuration)
|
|
|
|
} else if errorBackoff.NextBackOff() == backoff.Stop {
|
|
|
|
return err
|
2021-05-06 21:04:26 +03:00
|
|
|
}
|
2019-10-29 03:28:59 +03:00
|
|
|
|
2023-02-14 17:57:25 +03:00
|
|
|
// remainingLeaseDuration becomes the priorDuration for the next loop
|
|
|
|
priorDuration = remainingLeaseDuration
|
|
|
|
|
2019-10-29 03:28:59 +03:00
|
|
|
// If we are within grace, return now; or, if the amount of time we
|
|
|
|
// would sleep would land us in the grace period. This helps with short
|
|
|
|
// tokens; for example, you don't want a current lease duration of 4
|
|
|
|
// seconds, a grace period of 3 seconds, and end up sleeping for more
|
|
|
|
// than three of those seconds and having a very small budget of time
|
|
|
|
// to renew.
|
2021-05-06 21:04:26 +03:00
|
|
|
if remainingLeaseDuration <= r.grace || remainingLeaseDuration-sleepDuration <= r.grace {
|
2019-10-29 03:28:59 +03:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-02-06 19:49:01 +03:00
|
|
|
timer := time.NewTimer(sleepDuration)
|
2019-10-29 03:28:59 +03:00
|
|
|
select {
|
|
|
|
case <-r.stopCh:
|
2023-02-06 19:49:01 +03:00
|
|
|
timer.Stop()
|
2019-10-29 03:28:59 +03:00
|
|
|
return nil
|
2023-02-06 19:49:01 +03:00
|
|
|
case <-timer.C:
|
2019-10-29 03:28:59 +03:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-02-14 17:57:25 +03:00
|
|
|
// calculateSleepDuration calculates the amount of time the LifeTimeWatcher should sleep
|
|
|
|
// before re-entering its loop.
|
|
|
|
func (r *LifetimeWatcher) calculateSleepDuration(remainingLeaseDuration, priorDuration time.Duration) time.Duration {
|
|
|
|
// We keep evaluating a new grace period so long as the lease is
|
|
|
|
// extending. Once it stops extending, we've hit the max and need to
|
|
|
|
// rely on the grace duration.
|
|
|
|
if remainingLeaseDuration > priorDuration {
|
|
|
|
r.calculateGrace(remainingLeaseDuration, time.Duration(r.increment)*time.Second)
|
|
|
|
}
|
|
|
|
|
|
|
|
// The sleep duration is set to 2/3 of the current lease duration plus
|
|
|
|
// 1/3 of the current grace period, which adds jitter.
|
|
|
|
return time.Duration(float64(remainingLeaseDuration.Nanoseconds())*2/3 + float64(r.grace.Nanoseconds())/3)
|
|
|
|
}
|
|
|
|
|
2022-04-06 20:04:45 +03:00
|
|
|
// calculateGrace calculates the grace period based on the minimum of the
|
|
|
|
// remaining lease duration and the token increment value; it also adds some
|
|
|
|
// jitter to not have clients be in sync.
|
|
|
|
func (r *LifetimeWatcher) calculateGrace(leaseDuration, increment time.Duration) {
|
|
|
|
minDuration := leaseDuration
|
|
|
|
if minDuration > increment && increment > 0 {
|
|
|
|
minDuration = increment
|
|
|
|
}
|
|
|
|
|
|
|
|
if minDuration <= 0 {
|
2019-10-29 03:28:59 +03:00
|
|
|
r.grace = 0
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2022-04-06 20:04:45 +03:00
|
|
|
leaseNanos := float64(minDuration.Nanoseconds())
|
2019-10-29 03:28:59 +03:00
|
|
|
jitterMax := 0.1 * leaseNanos
|
|
|
|
|
|
|
|
// For a given lease duration, we want to allow 80-90% of that to elapse,
|
|
|
|
// so the remaining amount is the grace period
|
|
|
|
r.grace = time.Duration(jitterMax) + time.Duration(uint64(r.random.Int63())%uint64(jitterMax))
|
|
|
|
}
|
|
|
|
|
2021-04-08 19:43:39 +03:00
|
|
|
type (
|
|
|
|
Renewer = LifetimeWatcher
|
|
|
|
RenewerInput = LifetimeWatcherInput
|
|
|
|
)
|