2023-03-15 19:00:52 +03:00
|
|
|
// Copyright (c) HashiCorp, Inc.
|
2024-01-02 21:36:20 +03:00
|
|
|
// SPDX-License-Identifier: BUSL-1.1
|
2023-03-15 19:00:52 +03:00
|
|
|
|
2015-03-18 02:16:04 +03:00
|
|
|
package vault
|
|
|
|
|
|
|
|
import (
|
2018-01-08 21:31:38 +03:00
|
|
|
"context"
|
2018-09-18 06:03:00 +03:00
|
|
|
"errors"
|
2023-10-17 15:33:54 +03:00
|
|
|
"fmt"
|
|
|
|
"os"
|
|
|
|
"strconv"
|
2015-04-09 02:43:17 +03:00
|
|
|
"strings"
|
2015-04-02 08:11:48 +03:00
|
|
|
"sync"
|
2015-03-18 02:16:04 +03:00
|
|
|
"time"
|
|
|
|
|
2019-01-09 03:48:57 +03:00
|
|
|
metrics "github.com/armon/go-metrics"
|
2023-10-17 15:33:54 +03:00
|
|
|
"github.com/gammazero/workerpool"
|
2018-04-03 03:46:59 +03:00
|
|
|
log "github.com/hashicorp/go-hclog"
|
2018-09-18 06:03:00 +03:00
|
|
|
"github.com/hashicorp/vault/helper/namespace"
|
2019-04-13 00:54:35 +03:00
|
|
|
"github.com/hashicorp/vault/sdk/logical"
|
2015-03-18 02:16:04 +03:00
|
|
|
)
|
|
|
|
|
2023-10-17 15:33:54 +03:00
|
|
|
const (
|
|
|
|
RollbackDefaultNumWorkers = 256
|
|
|
|
RollbackWorkersEnvVar = "VAULT_ROLLBACK_WORKERS"
|
|
|
|
)
|
|
|
|
|
2024-02-21 19:22:47 +03:00
|
|
|
var rollbackCanceled = errors.New("rollback attempt canceled")
|
|
|
|
|
2015-03-18 02:16:04 +03:00
|
|
|
// RollbackManager is responsible for performing rollbacks of partial
|
|
|
|
// secrets within logical backends.
|
|
|
|
//
|
|
|
|
// During normal operations, it is possible for logical backends to
|
|
|
|
// error partially through an operation. These are called "partial secrets":
|
|
|
|
// they are never sent back to a user, but they do need to be cleaned up.
|
|
|
|
// This manager handles that by periodically (on a timer) requesting that the
|
|
|
|
// backends clean up.
|
|
|
|
//
|
2015-04-02 08:11:48 +03:00
|
|
|
// The RollbackManager periodically initiates a logical.RollbackOperation
|
|
|
|
// on every mounted logical backend. It ensures that only one rollback operation
|
|
|
|
// is in-flight at any given time within a single seal/unseal phase.
|
2015-03-18 02:16:04 +03:00
|
|
|
type RollbackManager struct {
|
2016-08-19 23:45:17 +03:00
|
|
|
logger log.Logger
|
2015-11-11 19:44:07 +03:00
|
|
|
|
2016-04-19 04:06:26 +03:00
|
|
|
// This gives the current mount table of both logical and credential backends,
|
|
|
|
// plus a RWMutex that is locked for reading. It is up to the caller to RUnlock
|
|
|
|
// it when done with the mount table.
|
|
|
|
backends func() []*MountEntry
|
2015-11-11 19:44:07 +03:00
|
|
|
|
2015-04-02 08:11:48 +03:00
|
|
|
router *Router
|
|
|
|
period time.Duration
|
|
|
|
|
|
|
|
inflightAll sync.WaitGroup
|
|
|
|
inflight map[string]*rollbackState
|
2016-09-13 18:50:14 +03:00
|
|
|
inflightLock sync.RWMutex
|
2015-04-02 08:11:48 +03:00
|
|
|
|
2023-05-04 17:14:06 +03:00
|
|
|
doneCh chan struct{}
|
|
|
|
shutdown bool
|
|
|
|
shutdownCh chan struct{}
|
|
|
|
shutdownLock sync.Mutex
|
|
|
|
stopTicker chan struct{}
|
|
|
|
tickerIsStopped bool
|
|
|
|
quitContext context.Context
|
2023-10-17 15:33:54 +03:00
|
|
|
runner *workerpool.WorkerPool
|
|
|
|
core *Core
|
|
|
|
// This channel is used for testing
|
|
|
|
rollbacksDoneCh chan struct{}
|
2015-04-02 08:11:48 +03:00
|
|
|
}
|
2015-03-18 04:46:10 +03:00
|
|
|
|
2015-04-02 08:11:48 +03:00
|
|
|
// rollbackState is used to track the state of a single rollback attempt
|
|
|
|
type rollbackState struct {
|
|
|
|
lastError error
|
|
|
|
sync.WaitGroup
|
2019-03-18 21:12:48 +03:00
|
|
|
cancelLockGrabCtx context.Context
|
|
|
|
cancelLockGrabCtxCancel context.CancelFunc
|
2023-10-17 15:33:54 +03:00
|
|
|
// scheduled is the time that this job was created and submitted to the
|
|
|
|
// rollbackRunner
|
2024-02-21 19:22:47 +03:00
|
|
|
scheduled time.Time
|
|
|
|
isRunning chan struct{}
|
|
|
|
isCanceled chan struct{}
|
2015-04-02 08:11:48 +03:00
|
|
|
}
|
2015-03-18 02:16:04 +03:00
|
|
|
|
2015-04-02 08:11:48 +03:00
|
|
|
// NewRollbackManager is used to create a new rollback manager
|
2018-07-31 04:55:43 +03:00
|
|
|
func NewRollbackManager(ctx context.Context, logger log.Logger, backendsFunc func() []*MountEntry, router *Router, core *Core) *RollbackManager {
|
2015-04-02 08:11:48 +03:00
|
|
|
r := &RollbackManager{
|
2018-01-08 21:31:38 +03:00
|
|
|
logger: logger,
|
|
|
|
backends: backendsFunc,
|
|
|
|
router: router,
|
2022-10-13 16:59:07 +03:00
|
|
|
period: core.rollbackPeriod,
|
2018-01-08 21:31:38 +03:00
|
|
|
inflight: make(map[string]*rollbackState),
|
|
|
|
doneCh: make(chan struct{}),
|
|
|
|
shutdownCh: make(chan struct{}),
|
2023-03-29 15:59:01 +03:00
|
|
|
stopTicker: make(chan struct{}),
|
2018-01-08 21:31:38 +03:00
|
|
|
quitContext: ctx,
|
2018-07-31 04:55:43 +03:00
|
|
|
core: core,
|
2015-04-02 08:11:48 +03:00
|
|
|
}
|
2023-10-17 15:33:54 +03:00
|
|
|
numWorkers := r.numRollbackWorkers()
|
|
|
|
r.logger.Info(fmt.Sprintf("Starting the rollback manager with %d workers", numWorkers))
|
|
|
|
r.runner = workerpool.New(numWorkers)
|
2015-04-02 08:11:48 +03:00
|
|
|
return r
|
2015-03-18 02:16:04 +03:00
|
|
|
}
|
|
|
|
|
2023-10-17 15:33:54 +03:00
|
|
|
func (m *RollbackManager) numRollbackWorkers() int {
|
|
|
|
numWorkers := m.core.numRollbackWorkers
|
|
|
|
envOverride := os.Getenv(RollbackWorkersEnvVar)
|
|
|
|
if envOverride != "" {
|
|
|
|
envVarWorkers, err := strconv.Atoi(envOverride)
|
|
|
|
if err != nil || envVarWorkers < 1 {
|
|
|
|
m.logger.Warn(fmt.Sprintf("%s must be a positive integer, but was %s", RollbackWorkersEnvVar, envOverride))
|
|
|
|
} else {
|
|
|
|
numWorkers = envVarWorkers
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return numWorkers
|
|
|
|
}
|
|
|
|
|
2015-04-02 08:11:48 +03:00
|
|
|
// Start starts the rollback manager
|
2015-03-18 02:16:04 +03:00
|
|
|
func (m *RollbackManager) Start() {
|
2015-04-02 08:11:48 +03:00
|
|
|
go m.run()
|
|
|
|
}
|
|
|
|
|
|
|
|
// Stop stops the running manager. This will wait for any in-flight
|
|
|
|
// rollbacks to complete.
|
|
|
|
func (m *RollbackManager) Stop() {
|
|
|
|
m.shutdownLock.Lock()
|
|
|
|
defer m.shutdownLock.Unlock()
|
|
|
|
if !m.shutdown {
|
|
|
|
m.shutdown = true
|
|
|
|
close(m.shutdownCh)
|
|
|
|
<-m.doneCh
|
2015-03-18 02:16:04 +03:00
|
|
|
}
|
2023-10-17 15:33:54 +03:00
|
|
|
m.runner.StopWait()
|
2015-04-02 08:11:48 +03:00
|
|
|
}
|
2015-03-18 02:16:04 +03:00
|
|
|
|
2023-04-10 20:24:03 +03:00
|
|
|
// StopTicker stops the automatic Rollback manager's ticker, causing us
|
|
|
|
// to not do automatic rollbacks. This is useful for testing plugin's
|
|
|
|
// periodic function's behavior, without trying to race against the
|
|
|
|
// rollback manager proper.
|
|
|
|
//
|
|
|
|
// THIS SHOULD ONLY BE CALLED FROM TEST HELPERS.
|
2023-03-29 15:59:01 +03:00
|
|
|
func (m *RollbackManager) StopTicker() {
|
2023-05-04 17:14:06 +03:00
|
|
|
if !m.tickerIsStopped {
|
|
|
|
close(m.stopTicker)
|
|
|
|
m.tickerIsStopped = true
|
|
|
|
}
|
2023-03-29 15:59:01 +03:00
|
|
|
}
|
|
|
|
|
2015-04-02 08:11:48 +03:00
|
|
|
// run is a long running routine to periodically invoke rollback
|
|
|
|
func (m *RollbackManager) run() {
|
2018-04-03 03:46:59 +03:00
|
|
|
m.logger.Info("starting rollback manager")
|
2015-04-02 08:11:48 +03:00
|
|
|
tick := time.NewTicker(m.period)
|
2023-04-10 20:24:03 +03:00
|
|
|
logTestStopOnce := false
|
2015-03-18 02:16:04 +03:00
|
|
|
defer tick.Stop()
|
2015-04-02 08:11:48 +03:00
|
|
|
defer close(m.doneCh)
|
2015-03-18 02:16:04 +03:00
|
|
|
for {
|
2015-04-02 08:11:48 +03:00
|
|
|
select {
|
|
|
|
case <-tick.C:
|
|
|
|
m.triggerRollbacks()
|
2015-03-18 02:16:04 +03:00
|
|
|
|
2015-04-02 08:11:48 +03:00
|
|
|
case <-m.shutdownCh:
|
2018-04-03 03:46:59 +03:00
|
|
|
m.logger.Info("stopping rollback manager")
|
2015-03-18 02:16:04 +03:00
|
|
|
return
|
2023-03-29 15:59:01 +03:00
|
|
|
|
|
|
|
case <-m.stopTicker:
|
2023-04-10 20:24:03 +03:00
|
|
|
if !logTestStopOnce {
|
|
|
|
m.logger.Info("stopping rollback manager ticker for tests")
|
|
|
|
logTestStopOnce = true
|
|
|
|
}
|
2023-03-29 15:59:01 +03:00
|
|
|
tick.Stop()
|
2015-03-18 02:16:04 +03:00
|
|
|
}
|
2015-04-02 08:11:48 +03:00
|
|
|
}
|
|
|
|
}
|
2015-03-18 02:16:04 +03:00
|
|
|
|
2015-04-02 08:11:48 +03:00
|
|
|
// triggerRollbacks is used to trigger the rollbacks across all the backends
|
|
|
|
func (m *RollbackManager) triggerRollbacks() {
|
2016-04-19 04:06:26 +03:00
|
|
|
backends := m.backends()
|
2015-11-11 19:44:07 +03:00
|
|
|
|
2016-04-19 04:06:26 +03:00
|
|
|
for _, e := range backends {
|
2016-05-26 21:29:41 +03:00
|
|
|
path := e.Path
|
|
|
|
if e.Table == credentialTableType {
|
2017-07-12 23:12:55 +03:00
|
|
|
path = credentialRoutePrefix + path
|
2016-05-26 21:29:41 +03:00
|
|
|
}
|
2017-07-12 23:12:55 +03:00
|
|
|
|
|
|
|
// When the mount is filtered, the backend will be nil
|
2018-09-18 06:03:00 +03:00
|
|
|
ctx := namespace.ContextWithNamespace(m.quitContext, e.namespace)
|
|
|
|
backend := m.router.MatchingBackend(ctx, path)
|
2017-07-12 23:12:55 +03:00
|
|
|
if backend == nil {
|
|
|
|
continue
|
|
|
|
}
|
2018-09-18 06:03:00 +03:00
|
|
|
fullPath := e.namespace.Path + path
|
2017-07-12 23:12:55 +03:00
|
|
|
|
2019-03-18 21:12:48 +03:00
|
|
|
// Start a rollback if necessary
|
|
|
|
m.startOrLookupRollback(ctx, fullPath, true)
|
2015-03-18 02:16:04 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-02-21 19:22:47 +03:00
|
|
|
// lookupRollbackLocked checks if there's an inflight rollback with the given
|
|
|
|
// path. Callers must have the inflightLock. The function also reports metrics,
|
|
|
|
// since it is regularly called as part of the scheduled rollbacks.
|
|
|
|
func (m *RollbackManager) lookupRollbackLocked(fullPath string) *rollbackState {
|
2023-10-17 15:33:54 +03:00
|
|
|
defer metrics.SetGauge([]string{"rollback", "queued"}, float32(m.runner.WaitingQueueSize()))
|
|
|
|
defer metrics.SetGauge([]string{"rollback", "inflight"}, float32(len(m.inflight)))
|
2024-02-21 19:22:47 +03:00
|
|
|
rsInflight := m.inflight[fullPath]
|
|
|
|
return rsInflight
|
|
|
|
}
|
2019-03-18 21:12:48 +03:00
|
|
|
|
2024-02-21 19:22:47 +03:00
|
|
|
// newRollbackLocked creates a new rollback state and adds it to the inflight
|
|
|
|
// rollback map. Callers must have the inflightLock
|
|
|
|
func (m *RollbackManager) newRollbackLocked(fullPath string) *rollbackState {
|
2019-03-18 21:12:48 +03:00
|
|
|
cancelCtx, cancelFunc := context.WithCancel(context.Background())
|
|
|
|
rs := &rollbackState{
|
|
|
|
cancelLockGrabCtx: cancelCtx,
|
|
|
|
cancelLockGrabCtxCancel: cancelFunc,
|
2024-02-21 19:22:47 +03:00
|
|
|
isRunning: make(chan struct{}),
|
|
|
|
isCanceled: make(chan struct{}),
|
|
|
|
scheduled: time.Now(),
|
2019-03-18 21:12:48 +03:00
|
|
|
}
|
2018-09-18 06:03:00 +03:00
|
|
|
m.inflight[fullPath] = rs
|
2019-03-18 21:12:48 +03:00
|
|
|
rs.Add(1)
|
|
|
|
m.inflightAll.Add(1)
|
2024-02-21 19:22:47 +03:00
|
|
|
return rs
|
|
|
|
}
|
|
|
|
|
|
|
|
// startOrLookupRollback is used to start an async rollback attempt.
|
|
|
|
func (m *RollbackManager) startOrLookupRollback(ctx context.Context, fullPath string, grabStatelock bool) *rollbackState {
|
|
|
|
m.inflightLock.Lock()
|
|
|
|
defer m.inflightLock.Unlock()
|
|
|
|
rs := m.lookupRollbackLocked(fullPath)
|
|
|
|
if rs != nil {
|
|
|
|
return rs
|
|
|
|
}
|
|
|
|
|
|
|
|
// If no inflight rollback is already running, kick one off
|
|
|
|
rs = m.newRollbackLocked(fullPath)
|
|
|
|
|
2023-10-17 15:33:54 +03:00
|
|
|
select {
|
|
|
|
case <-m.doneCh:
|
|
|
|
// if we've already shut down, then don't submit the task to avoid a panic
|
|
|
|
// we should still call finishRollback for the rollback state in order to remove
|
|
|
|
// it from the map and decrement the waitgroup.
|
|
|
|
|
|
|
|
// we already have the inflight lock, so we can't grab it here
|
|
|
|
m.finishRollback(rs, errors.New("rollback manager is stopped"), fullPath, false)
|
|
|
|
default:
|
|
|
|
m.runner.Submit(func() {
|
|
|
|
m.attemptRollback(ctx, fullPath, rs, grabStatelock)
|
|
|
|
select {
|
|
|
|
case m.rollbacksDoneCh <- struct{}{}:
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
})
|
|
|
|
|
|
|
|
}
|
2015-04-02 08:11:48 +03:00
|
|
|
return rs
|
2015-03-18 02:16:04 +03:00
|
|
|
}
|
|
|
|
|
2023-10-17 15:33:54 +03:00
|
|
|
func (m *RollbackManager) finishRollback(rs *rollbackState, err error, fullPath string, grabInflightLock bool) {
|
|
|
|
rs.lastError = err
|
|
|
|
rs.Done()
|
|
|
|
m.inflightAll.Done()
|
|
|
|
if grabInflightLock {
|
|
|
|
m.inflightLock.Lock()
|
|
|
|
defer m.inflightLock.Unlock()
|
|
|
|
}
|
2024-02-21 19:22:47 +03:00
|
|
|
if _, ok := m.inflight[fullPath]; ok {
|
|
|
|
delete(m.inflight, fullPath)
|
|
|
|
}
|
2023-10-17 15:33:54 +03:00
|
|
|
}
|
|
|
|
|
2015-04-02 08:11:48 +03:00
|
|
|
// attemptRollback invokes a RollbackOperation for the given path
|
2018-09-18 06:03:00 +03:00
|
|
|
func (m *RollbackManager) attemptRollback(ctx context.Context, fullPath string, rs *rollbackState, grabStatelock bool) (err error) {
|
2024-02-21 19:22:47 +03:00
|
|
|
close(rs.isRunning)
|
|
|
|
defer m.finishRollback(rs, err, fullPath, true)
|
|
|
|
select {
|
|
|
|
case <-rs.isCanceled:
|
|
|
|
return rollbackCanceled
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
|
2023-10-17 15:33:54 +03:00
|
|
|
metrics.MeasureSince([]string{"rollback", "waiting"}, rs.scheduled)
|
2022-08-03 22:22:48 +03:00
|
|
|
defer metrics.MeasureSince([]string{"rollback", "attempt", strings.ReplaceAll(fullPath, "/", "-")}, time.Now())
|
2015-04-02 08:11:48 +03:00
|
|
|
|
2018-09-18 06:03:00 +03:00
|
|
|
ns, err := namespace.FromContext(ctx)
|
|
|
|
if err != nil {
|
2019-07-19 04:05:21 +03:00
|
|
|
m.logger.Error("rollback failed to derive namespace from context", "path", fullPath)
|
2018-09-18 06:03:00 +03:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
if ns == nil {
|
2019-07-19 04:05:21 +03:00
|
|
|
m.logger.Error("rollback found no namespace", "path", fullPath)
|
2018-09-18 06:03:00 +03:00
|
|
|
return namespace.ErrNoNamespace
|
|
|
|
}
|
|
|
|
|
2015-04-02 08:11:48 +03:00
|
|
|
// Invoke a RollbackOperation
|
2015-03-18 02:16:04 +03:00
|
|
|
req := &logical.Request{
|
|
|
|
Operation: logical.RollbackOperation,
|
2018-09-18 06:03:00 +03:00
|
|
|
Path: ns.TrimmedPath(fullPath),
|
2015-03-18 02:16:04 +03:00
|
|
|
}
|
2018-09-18 06:03:00 +03:00
|
|
|
|
2019-03-18 21:12:48 +03:00
|
|
|
releaseLock := true
|
2018-08-08 03:48:43 +03:00
|
|
|
if grabStatelock {
|
2019-03-18 21:12:48 +03:00
|
|
|
doneCh := make(chan struct{})
|
|
|
|
defer close(doneCh)
|
|
|
|
|
|
|
|
stopCh := make(chan struct{})
|
|
|
|
go func() {
|
|
|
|
defer close(stopCh)
|
|
|
|
|
|
|
|
select {
|
|
|
|
case <-m.shutdownCh:
|
|
|
|
case <-rs.cancelLockGrabCtx.Done():
|
|
|
|
case <-doneCh:
|
2024-02-21 19:22:47 +03:00
|
|
|
case <-rs.isCanceled:
|
2019-03-18 21:12:48 +03:00
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
2018-09-18 06:03:00 +03:00
|
|
|
// Grab the statelock or stop
|
2022-09-20 18:03:16 +03:00
|
|
|
l := newLockGrabber(m.core.stateLock.RLock, m.core.stateLock.RUnlock, stopCh)
|
|
|
|
go l.grab()
|
|
|
|
if stopped := l.lockOrStop(); stopped {
|
2019-03-18 21:12:48 +03:00
|
|
|
// If we stopped due to shutdown, return. Otherwise another thread
|
|
|
|
// is holding the lock for us, continue on.
|
|
|
|
select {
|
|
|
|
case <-m.shutdownCh:
|
|
|
|
return errors.New("rollback shutting down")
|
2024-02-21 19:22:47 +03:00
|
|
|
case <-rs.isCanceled:
|
|
|
|
return rollbackCanceled
|
2019-03-18 21:12:48 +03:00
|
|
|
default:
|
|
|
|
releaseLock = false
|
|
|
|
}
|
2018-09-18 06:03:00 +03:00
|
|
|
}
|
2018-08-08 03:48:43 +03:00
|
|
|
}
|
2018-09-18 06:03:00 +03:00
|
|
|
|
|
|
|
var cancelFunc context.CancelFunc
|
|
|
|
ctx, cancelFunc = context.WithTimeout(ctx, DefaultMaxRequestDuration)
|
2019-07-06 01:12:34 +03:00
|
|
|
resp, err := m.router.Route(ctx, req)
|
2019-03-18 21:12:48 +03:00
|
|
|
if grabStatelock && releaseLock {
|
2018-08-08 03:48:43 +03:00
|
|
|
m.core.stateLock.RUnlock()
|
|
|
|
}
|
2018-07-25 00:50:49 +03:00
|
|
|
cancelFunc()
|
2015-03-18 02:24:42 +03:00
|
|
|
|
2015-04-02 08:11:48 +03:00
|
|
|
// If the error is an unsupported operation, then it doesn't
|
|
|
|
// matter, the backend doesn't support it.
|
|
|
|
if err == logical.ErrUnsupportedOperation {
|
|
|
|
err = nil
|
|
|
|
}
|
2018-02-03 04:28:25 +03:00
|
|
|
// If we failed due to read-only storage, we can't do anything; ignore
|
2019-07-06 01:12:34 +03:00
|
|
|
if (err != nil && strings.Contains(err.Error(), logical.ErrReadOnly.Error())) ||
|
|
|
|
(resp.IsError() && strings.Contains(resp.Error().Error(), logical.ErrReadOnly.Error())) {
|
2018-02-03 04:28:25 +03:00
|
|
|
err = nil
|
|
|
|
}
|
2015-04-02 08:11:48 +03:00
|
|
|
if err != nil {
|
2018-09-18 06:03:00 +03:00
|
|
|
m.logger.Error("error rolling back", "path", fullPath, "error", err)
|
2015-03-18 02:16:04 +03:00
|
|
|
}
|
2015-04-02 08:11:48 +03:00
|
|
|
return
|
2015-03-18 02:16:04 +03:00
|
|
|
}
|
2015-03-18 02:23:58 +03:00
|
|
|
|
2015-04-02 08:11:48 +03:00
|
|
|
// Rollback is used to trigger an immediate rollback of the path,
|
2018-08-08 03:48:43 +03:00
|
|
|
// or to join an existing rollback operation if in flight. Caller should have
|
2019-03-18 21:12:48 +03:00
|
|
|
// core's statelock held (write OR read). If an already inflight rollback is
|
|
|
|
// happening this function will simply wait for it to complete
|
2018-09-18 06:03:00 +03:00
|
|
|
func (m *RollbackManager) Rollback(ctx context.Context, path string) error {
|
|
|
|
ns, err := namespace.FromContext(ctx)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
fullPath := ns.Path + path
|
|
|
|
|
2024-02-21 19:22:47 +03:00
|
|
|
m.inflightLock.Lock()
|
|
|
|
rs := m.lookupRollbackLocked(fullPath)
|
|
|
|
if rs != nil {
|
|
|
|
// Since we have the statelock held, tell any inflight rollback to give up
|
|
|
|
// trying to acquire it. This will prevent deadlocks in the case where we
|
|
|
|
// have the write lock. In the case where it was waiting to grab
|
|
|
|
// a read lock it will then simply continue with the rollback
|
|
|
|
// operation under the protection of our write lock.
|
|
|
|
rs.cancelLockGrabCtxCancel()
|
|
|
|
|
|
|
|
select {
|
|
|
|
case <-rs.isRunning:
|
|
|
|
// if the rollback has started then we should wait for it to complete
|
|
|
|
m.inflightLock.Unlock()
|
|
|
|
rs.Wait()
|
|
|
|
return rs.lastError
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
|
|
|
|
// if the rollback hasn't started and there's no capacity, we could
|
|
|
|
// end up deadlocking. Cancel the existing rollback and start a new
|
|
|
|
// one.
|
|
|
|
close(rs.isCanceled)
|
|
|
|
}
|
|
|
|
rs = m.newRollbackLocked(fullPath)
|
|
|
|
m.inflightLock.Unlock()
|
2019-03-18 21:12:48 +03:00
|
|
|
|
2024-02-21 19:22:47 +03:00
|
|
|
// we can ignore the error, since it's going to be set in rs.lastError
|
|
|
|
m.attemptRollback(ctx, fullPath, rs, false)
|
2015-03-18 02:23:58 +03:00
|
|
|
|
2015-04-02 08:11:48 +03:00
|
|
|
rs.Wait()
|
|
|
|
return rs.lastError
|
|
|
|
}
|
2015-03-18 02:23:58 +03:00
|
|
|
|
2015-04-02 08:11:48 +03:00
|
|
|
// The methods below are the hooks from core that are called pre/post seal.
|
|
|
|
|
|
|
|
// startRollback is used to start the rollback manager after unsealing
|
|
|
|
func (c *Core) startRollback() error {
|
2016-04-19 04:06:26 +03:00
|
|
|
backendsFunc := func() []*MountEntry {
|
2015-11-16 01:32:57 +03:00
|
|
|
ret := []*MountEntry{}
|
|
|
|
c.mountsLock.RLock()
|
|
|
|
defer c.mountsLock.RUnlock()
|
2016-07-19 05:02:39 +03:00
|
|
|
// During teardown/setup after a leader change or unseal there could be
|
|
|
|
// something racy here so make sure the table isn't nil
|
|
|
|
if c.mounts != nil {
|
|
|
|
for _, entry := range c.mounts.Entries {
|
|
|
|
ret = append(ret, entry)
|
|
|
|
}
|
2015-11-16 01:32:57 +03:00
|
|
|
}
|
2016-05-26 21:29:41 +03:00
|
|
|
c.authLock.RLock()
|
|
|
|
defer c.authLock.RUnlock()
|
2016-07-19 05:02:39 +03:00
|
|
|
// During teardown/setup after a leader change or unseal there could be
|
|
|
|
// something racy here so make sure the table isn't nil
|
|
|
|
if c.auth != nil {
|
|
|
|
for _, entry := range c.auth.Entries {
|
|
|
|
ret = append(ret, entry)
|
|
|
|
}
|
2016-05-26 21:29:41 +03:00
|
|
|
}
|
2015-11-16 01:32:57 +03:00
|
|
|
return ret
|
2015-11-11 19:44:07 +03:00
|
|
|
}
|
2018-09-05 22:52:54 +03:00
|
|
|
rollbackLogger := c.baseLogger.Named("rollback")
|
2023-11-04 00:40:17 +03:00
|
|
|
c.AddLogger(rollbackLogger)
|
2018-09-05 22:52:54 +03:00
|
|
|
c.rollback = NewRollbackManager(c.activeContext, rollbackLogger, backendsFunc, c.router, c)
|
2015-04-02 08:11:48 +03:00
|
|
|
c.rollback.Start()
|
2015-03-18 02:23:58 +03:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2015-04-02 08:11:48 +03:00
|
|
|
// stopRollback is used to stop running the rollback manager before sealing
|
2015-03-18 02:23:58 +03:00
|
|
|
func (c *Core) stopRollback() error {
|
|
|
|
if c.rollback != nil {
|
|
|
|
c.rollback.Stop()
|
|
|
|
c.rollback = nil
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|