1326 lines
36 KiB
Go
1326 lines
36 KiB
Go
// Copyright (c) HashiCorp, Inc.
|
|
// SPDX-License-Identifier: BUSL-1.1
|
|
|
|
package quotas
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"path"
|
|
"strings"
|
|
|
|
log "github.com/hashicorp/go-hclog"
|
|
"github.com/hashicorp/go-memdb"
|
|
"github.com/hashicorp/vault/helper/locking"
|
|
"github.com/hashicorp/vault/helper/metricsutil"
|
|
"github.com/hashicorp/vault/helper/namespace"
|
|
"github.com/hashicorp/vault/sdk/helper/pathmanager"
|
|
"github.com/hashicorp/vault/sdk/logical"
|
|
)
|
|
|
|
// Type represents the quota kind
|
|
type Type string
|
|
|
|
const (
|
|
// TypeRateLimit represents the rate limiting quota type
|
|
TypeRateLimit Type = "rate-limit"
|
|
|
|
// TypeLeaseCount represents the lease count limiting quota type
|
|
TypeLeaseCount Type = "lease-count"
|
|
)
|
|
|
|
// LeaseAction is the action taken by the expiration manager on the lease. The
|
|
// quota manager will use this information to update the lease path cache and
|
|
// updating counters for relevant quota rules.
|
|
type LeaseAction uint32
|
|
|
|
// String converts each lease action into its string equivalent value
|
|
func (la LeaseAction) String() string {
|
|
switch la {
|
|
case LeaseActionLoaded:
|
|
return "loaded"
|
|
case LeaseActionCreated:
|
|
return "created"
|
|
case LeaseActionDeleted:
|
|
return "deleted"
|
|
case LeaseActionAllow:
|
|
return "allow"
|
|
}
|
|
return "unknown"
|
|
}
|
|
|
|
const (
|
|
_ LeaseAction = iota
|
|
|
|
// LeaseActionLoaded indicates loading of lease in the expiration manager after
|
|
// unseal.
|
|
LeaseActionLoaded
|
|
|
|
// LeaseActionCreated indicates that a lease is created in the expiration manager.
|
|
LeaseActionCreated
|
|
|
|
// LeaseActionDeleted indicates that is lease is expired and deleted in the
|
|
// expiration manager.
|
|
LeaseActionDeleted
|
|
|
|
// LeaseActionAllow will be used to indicate the lease count checker that
|
|
// incCounter is called from Allow(). All the rest of the actions indicate the
|
|
// action took place on the lease in the expiration manager.
|
|
LeaseActionAllow
|
|
)
|
|
|
|
type leaseWalkFunc func(context.Context, func(request *Request) bool) error
|
|
|
|
// String converts each quota type into its string equivalent value
|
|
func (q Type) String() string {
|
|
switch q {
|
|
case TypeLeaseCount:
|
|
return "lease-count"
|
|
case TypeRateLimit:
|
|
return "rate-limit"
|
|
}
|
|
return "unknown"
|
|
}
|
|
|
|
const (
|
|
indexID = "id"
|
|
indexName = "name"
|
|
indexNamespace = "ns"
|
|
indexNamespaceMount = "ns_mount"
|
|
indexNamespaceMountPath = "ns_mount_path"
|
|
indexNamespaceMountRole = "ns_mount_role"
|
|
)
|
|
|
|
const (
|
|
// StoragePrefix is the prefix for the physical location where quota rules are
|
|
// persisted.
|
|
StoragePrefix = "quotas/"
|
|
|
|
// ConfigPath is the physical location where the quota configuration is
|
|
// persisted.
|
|
ConfigPath = StoragePrefix + "config"
|
|
|
|
// DefaultRateLimitExemptPathsToggle is the path to a toggle that allows us to
|
|
// determine if a Vault operator explicitly modified the exempt paths set for
|
|
// rate limit resource quotas. Specifically, when this toggle is false, we can
|
|
// infer a Vault node is operating with an initial default set and on a subsequent
|
|
// update to that set, we should not overwrite it on Setup.
|
|
DefaultRateLimitExemptPathsToggle = StoragePrefix + "default_rate_limit_exempt_paths_toggle"
|
|
)
|
|
|
|
var (
|
|
// ErrLeaseCountQuotaExceeded is returned when a request is rejected due to a lease
|
|
// count quota being exceeded.
|
|
ErrLeaseCountQuotaExceeded = errors.New("lease count quota exceeded")
|
|
|
|
// ErrRateLimitQuotaExceeded is returned when a request is rejected due to a
|
|
// rate limit quota being exceeded.
|
|
ErrRateLimitQuotaExceeded = errors.New("rate limit quota exceeded")
|
|
)
|
|
|
|
var defaultExemptPaths = []string{
|
|
"sys/generate-recovery-token/attempt",
|
|
"sys/generate-recovery-token/update",
|
|
"sys/generate-root/attempt",
|
|
"sys/generate-root/update",
|
|
"sys/health",
|
|
"sys/seal-status",
|
|
"sys/unseal",
|
|
}
|
|
|
|
// Access provides information to reach back to the quota checker.
|
|
type Access interface {
|
|
// QuotaID is the identifier of the quota that issued this access.
|
|
QuotaID() string
|
|
}
|
|
|
|
// Ensure that access implements the Access interface.
|
|
var _ Access = (*access)(nil)
|
|
|
|
// access implements the Access interface
|
|
type access struct {
|
|
quotaID string
|
|
}
|
|
|
|
// QuotaID returns the identifier of the quota rule to which this access refers
|
|
// to.
|
|
func (a *access) QuotaID() string {
|
|
return a.quotaID
|
|
}
|
|
|
|
// Manager holds all the existing quota rules. For any given input. the manager
|
|
// checks them against any applicable quota rules.
|
|
type Manager struct {
|
|
entManager
|
|
|
|
// db holds the in memory instances of all active quota rules indexed by
|
|
// some of the quota properties.
|
|
db *memdb.MemDB
|
|
|
|
// config containing operator preferences and quota behaviors
|
|
config *Config
|
|
|
|
rateLimitPathManager *pathmanager.PathManager
|
|
|
|
storage logical.Storage
|
|
ctx context.Context
|
|
|
|
logger log.Logger
|
|
metricSink *metricsutil.ClusterMetricSink
|
|
|
|
// quotaLock is a lock for manipulating quotas and anything not covered by a more specific lock
|
|
quotaLock locking.RWMutex
|
|
|
|
// quotaConfigLock is a lock for accessing config items, such as RateLimitExemptPaths
|
|
quotaConfigLock locking.RWMutex
|
|
|
|
// dbAndCacheLock is a lock for db and path caches that need to be reset during Reset()
|
|
dbAndCacheLock locking.RWMutex
|
|
}
|
|
|
|
// QuotaLeaseInformation contains all of the information lease-count quotas require
|
|
// from a lease to uniquely identify the lease-count quota to increment/decrement
|
|
type QuotaLeaseInformation struct {
|
|
// We can determine path and namespace from leaseId
|
|
LeaseId string
|
|
|
|
// We need the role as it's not part of the leaseId, and is required
|
|
// to uniquely identify a lease count quota
|
|
Role string
|
|
}
|
|
|
|
// Quota represents the common properties of every quota type
|
|
type Quota interface {
|
|
// allow checks the if the request is allowed by the quota type implementation.
|
|
allow(context.Context, *Request) (Response, error)
|
|
|
|
// quotaID is the identifier of the quota rule
|
|
quotaID() string
|
|
|
|
// QuotaName is the name of the quota rule
|
|
QuotaName() string
|
|
|
|
// initialize sets up the fields in the quota type to begin operating
|
|
initialize(log.Logger, *metricsutil.ClusterMetricSink) error
|
|
|
|
// close defines any cleanup behavior that needs to be executed when a quota
|
|
// rule is deleted.
|
|
close(context.Context) error
|
|
|
|
// Clone creates a clone of the calling quota
|
|
Clone() Quota
|
|
|
|
// handleRemount updates the mount and namesapce paths of the quota
|
|
handleRemount(string, string)
|
|
}
|
|
|
|
// Response holds information about the result of the Allow() call. The response
|
|
// can optionally have the Access field set, which is used to reach back into
|
|
// the quota rule that sent this response.
|
|
type Response struct {
|
|
// Allowed is set if the quota allows the request
|
|
Allowed bool
|
|
|
|
// Access is the handle to reach back into the quota rule that processed the
|
|
// quota request. This may not be set all the time.
|
|
Access Access
|
|
|
|
// Headers defines any optional headers that may be returned by the quota rule
|
|
// to clients.
|
|
Headers map[string]string
|
|
}
|
|
|
|
// Config holds operator preferences around quota behaviors
|
|
type Config struct {
|
|
// EnableRateLimitAuditLogging, if set, starts audit logging of the
|
|
// request rejections that arise due to rate limit quota violations.
|
|
EnableRateLimitAuditLogging bool `json:"enable_rate_limit_audit_logging"`
|
|
|
|
// EnableRateLimitResponseHeaders dictates if rate limit quota HTTP headers
|
|
// should be added to responses.
|
|
EnableRateLimitResponseHeaders bool `json:"enable_rate_limit_response_headers"`
|
|
|
|
// RateLimitExemptPaths defines the set of exempt paths used for all rate limit
|
|
// quotas. Any request path that exists in this set is exempt from rate limiting.
|
|
// If the set is empty, no paths are exempt.
|
|
RateLimitExemptPaths []string `json:"rate_limit_exempt_paths"`
|
|
}
|
|
|
|
// Request contains information required by the quota manager to query and
|
|
// apply the quota rules.
|
|
type Request struct {
|
|
// Type is the quota type
|
|
Type Type
|
|
|
|
// Path is the request path to which quota rules are being queried for
|
|
Path string
|
|
|
|
// Role is the role given as part of the request to a login endpoint
|
|
Role string
|
|
|
|
// NamespacePath is the namespace path to which the request belongs
|
|
NamespacePath string
|
|
|
|
// MountPath is the mount path to which the request is made
|
|
MountPath string
|
|
|
|
// ClientAddress is client unique addressable string (e.g. IP address). It can
|
|
// be empty if the quota type does not need it.
|
|
ClientAddress string
|
|
}
|
|
|
|
// NewManager creates and initializes a new quota manager to hold all the quota
|
|
// rules and to process incoming requests.
|
|
func NewManager(logger log.Logger, walkFunc leaseWalkFunc, ms *metricsutil.ClusterMetricSink, detectDeadlocks bool) (*Manager, error) {
|
|
db, err := memdb.NewMemDB(dbSchema())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
manager := &Manager{
|
|
db: db,
|
|
logger: logger,
|
|
metricSink: ms,
|
|
rateLimitPathManager: pathmanager.New(),
|
|
config: new(Config),
|
|
quotaLock: &locking.SyncRWMutex{},
|
|
quotaConfigLock: &locking.SyncRWMutex{},
|
|
dbAndCacheLock: &locking.SyncRWMutex{},
|
|
}
|
|
|
|
if detectDeadlocks {
|
|
logger.Debug("enabling deadlock detection")
|
|
manager.quotaLock = &locking.DeadlockRWMutex{}
|
|
manager.quotaConfigLock = &locking.DeadlockRWMutex{}
|
|
manager.dbAndCacheLock = &locking.DeadlockRWMutex{}
|
|
}
|
|
|
|
manager.init(walkFunc)
|
|
|
|
return manager, nil
|
|
}
|
|
|
|
// SetQuota adds or updates a quota rule.
|
|
func (m *Manager) SetQuota(ctx context.Context, qType string, quota Quota, loading bool) error {
|
|
m.quotaLock.Lock()
|
|
m.dbAndCacheLock.RLock()
|
|
defer m.quotaLock.Unlock()
|
|
defer m.dbAndCacheLock.RUnlock()
|
|
return m.setQuotaLocked(ctx, qType, quota, loading)
|
|
}
|
|
|
|
// setQuotaLocked creates a transaction, passes it into setQuotaLockedWithTxn and manages its lifecycle
|
|
// along with updating lease quota counts
|
|
func (m *Manager) setQuotaLocked(ctx context.Context, qType string, quota Quota, loading bool) error {
|
|
txn := m.db.Txn(true)
|
|
defer txn.Abort()
|
|
|
|
err := m.setQuotaLockedWithTxn(ctx, qType, quota, loading, txn)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if loading {
|
|
txn.Commit()
|
|
return nil
|
|
}
|
|
|
|
// For the lease count type, recompute the counters
|
|
if !loading && qType == TypeLeaseCount.String() {
|
|
if err := m.recomputeLeaseCounts(ctx, txn); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
txn.Commit()
|
|
return nil
|
|
}
|
|
|
|
// setQuotaLockedWithTxn adds or updates a quota rule, modifying the db as well as
|
|
// any runtime elements such as goroutines, using the transaction passed in
|
|
// It should be called with the write lock held.
|
|
func (m *Manager) setQuotaLockedWithTxn(ctx context.Context, qType string, quota Quota, loading bool, txn *memdb.Txn) error {
|
|
if qType == TypeLeaseCount.String() {
|
|
m.setIsPerfStandby(quota)
|
|
}
|
|
|
|
raw, err := txn.First(qType, indexID, quota.quotaID())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// If there already exists an entry in the db, remove that first.
|
|
if raw != nil {
|
|
quota := raw.(Quota)
|
|
if err := quota.close(ctx); err != nil {
|
|
return err
|
|
}
|
|
err = txn.Delete(qType, raw)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// Initialize the quota type implementation
|
|
if err := quota.initialize(m.logger, m.metricSink); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Add the initialized quota type implementation to the db
|
|
if err := txn.Insert(qType, quota); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// QuotaNames returns the names of all the quota rules for a given type
|
|
func (m *Manager) QuotaNames(qType Type) ([]string, error) {
|
|
m.dbAndCacheLock.RLock()
|
|
defer m.dbAndCacheLock.RUnlock()
|
|
|
|
return m.quotaNamesLocked(qType)
|
|
}
|
|
|
|
// quotaNamesLocked returns the names of all the quota rules for a given type, and must be called with the lock
|
|
func (m *Manager) quotaNamesLocked(qType Type) ([]string, error) {
|
|
txn := m.db.Txn(false)
|
|
iter, err := txn.Get(qType.String(), indexID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var names []string
|
|
for raw := iter.Next(); raw != nil; raw = iter.Next() {
|
|
names = append(names, raw.(Quota).QuotaName())
|
|
}
|
|
return names, nil
|
|
}
|
|
|
|
// QuotaByID queries for a quota rule in the db for a given quota ID
|
|
func (m *Manager) QuotaByID(qType string, id string) (Quota, error) {
|
|
m.dbAndCacheLock.RLock()
|
|
defer m.dbAndCacheLock.RUnlock()
|
|
|
|
txn := m.db.Txn(false)
|
|
|
|
quotaRaw, err := txn.First(qType, indexID, id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if quotaRaw == nil {
|
|
return nil, nil
|
|
}
|
|
|
|
return quotaRaw.(Quota), nil
|
|
}
|
|
|
|
// QuotaByName queries for a quota rule in the db for a given quota name
|
|
func (m *Manager) QuotaByName(qType string, name string) (Quota, error) {
|
|
m.dbAndCacheLock.RLock()
|
|
defer m.dbAndCacheLock.RUnlock()
|
|
|
|
return m.quotaByNameLocked(qType, name)
|
|
}
|
|
|
|
// quotaByNameLocked queries for a quota rule in the db for a given quota name, and must be called with the lock
|
|
func (m *Manager) quotaByNameLocked(qType string, name string) (Quota, error) {
|
|
txn := m.db.Txn(false)
|
|
|
|
quotaRaw, err := txn.First(qType, indexName, name)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if quotaRaw == nil {
|
|
return nil, nil
|
|
}
|
|
|
|
return quotaRaw.(Quota), nil
|
|
}
|
|
|
|
// QuotaByFactors returns the quota rule that matches the provided factors
|
|
func (m *Manager) QuotaByFactors(ctx context.Context, qType, nsPath, mountPath, pathSuffix, role string) (Quota, error) {
|
|
m.dbAndCacheLock.RLock()
|
|
defer m.dbAndCacheLock.RUnlock()
|
|
|
|
// nsPath would have been made non-empty during insertion. Use non-empty value
|
|
// during query as well.
|
|
if nsPath == "" {
|
|
nsPath = "root"
|
|
}
|
|
|
|
idx := indexNamespace
|
|
args := []interface{}{nsPath, false, false, false}
|
|
if mountPath != "" {
|
|
if pathSuffix != "" {
|
|
idx = indexNamespaceMountPath
|
|
args = []interface{}{nsPath, mountPath, pathSuffix, false}
|
|
} else if role != "" {
|
|
idx = indexNamespaceMountRole
|
|
args = []interface{}{nsPath, mountPath, false, role}
|
|
} else {
|
|
idx = indexNamespaceMount
|
|
args = []interface{}{nsPath, mountPath, false, false}
|
|
}
|
|
}
|
|
|
|
txn := m.db.Txn(false)
|
|
iter, err := txn.Get(qType, idx, args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var quotas []Quota
|
|
for raw := iter.Next(); raw != nil; raw = iter.Next() {
|
|
quotas = append(quotas, raw.(Quota))
|
|
}
|
|
if len(quotas) > 1 {
|
|
m.logger.Debug("conflicting quotas in QuotaByFactors", "matching_quotas", quotas)
|
|
return nil, fmt.Errorf("conflicting quota definitions detected")
|
|
}
|
|
if len(quotas) == 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
return quotas[0], nil
|
|
}
|
|
|
|
// QueryQuota returns the most specific applicable quota for a given request.
|
|
func (m *Manager) QueryQuota(req *Request) (Quota, error) {
|
|
m.dbAndCacheLock.RLock()
|
|
defer m.dbAndCacheLock.RUnlock()
|
|
|
|
return m.queryQuota(nil, req)
|
|
}
|
|
|
|
// queryQuota returns the quota rule that is applicable for the given request. It
|
|
// queries all the quota rules that are defined against request values and finds
|
|
// the quota rule that takes priority.
|
|
//
|
|
// Priority rules are as follows:
|
|
// - namespace specific quota takes precedence over global quota
|
|
// - mount specific quota takes precedence over namespace specific quota
|
|
// - path suffix specific quota takes precedence over mount specific quota
|
|
// - role based quota takes precedence over path suffix/mount specific quota
|
|
func (m *Manager) queryQuota(txn *memdb.Txn, req *Request) (Quota, error) {
|
|
if txn == nil {
|
|
txn = m.db.Txn(false)
|
|
}
|
|
|
|
// ns would have been made non-empty during insertion. Use non-empty
|
|
// value during query as well.
|
|
if req.NamespacePath == "" {
|
|
req.NamespacePath = "root"
|
|
}
|
|
|
|
//
|
|
// Find a match from most specific applicable quota rule to less specific one.
|
|
//
|
|
quotaFetchFunc := func(idx string, args ...interface{}) (Quota, error) {
|
|
iter, err := txn.Get(req.Type.String(), idx, args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var quotas []Quota
|
|
for raw := iter.Next(); raw != nil; raw = iter.Next() {
|
|
quota := raw.(Quota)
|
|
quotas = append(quotas, quota)
|
|
}
|
|
if len(quotas) > 1 {
|
|
m.logger.Debug("conflicting quotas in queryQuota", "matching_quotas", quotas, "args", args)
|
|
return nil, fmt.Errorf("conflicting quota definitions detected")
|
|
}
|
|
if len(quotas) == 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
return quotas[0], nil
|
|
}
|
|
|
|
// Fetch role suffix quota
|
|
quota, err := quotaFetchFunc(indexNamespaceMountRole, req.NamespacePath, req.MountPath, false, req.Role)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if quota != nil {
|
|
return quota, nil
|
|
}
|
|
|
|
// Fetch path suffix quota
|
|
pathSuffix := strings.TrimSuffix(strings.TrimPrefix(strings.TrimPrefix(req.Path, req.NamespacePath), req.MountPath), "/")
|
|
quota, err = quotaFetchFunc(indexNamespaceMountPath, req.NamespacePath, req.MountPath, pathSuffix, false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if quota != nil {
|
|
return quota, nil
|
|
}
|
|
|
|
// Fetch path suffix quotas with globbing
|
|
// Request paths which match the resulting glob (i.e. share the same prefix prior to the glob) are in scope for the quota
|
|
for i := 0; i <= len(pathSuffix); i++ {
|
|
trimmedSuffixWithGlob := pathSuffix[:len(pathSuffix)-i] + "*"
|
|
// Check to see if a quota exists with this particular pattern
|
|
quota, err = quotaFetchFunc(indexNamespaceMountPath, req.NamespacePath, req.MountPath, trimmedSuffixWithGlob, false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if quota != nil {
|
|
return quota, nil
|
|
}
|
|
}
|
|
|
|
// Fetch mount quota
|
|
quota, err = quotaFetchFunc(indexNamespaceMount, req.NamespacePath, req.MountPath, false, false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if quota != nil {
|
|
return quota, nil
|
|
}
|
|
|
|
// Fetch ns quota. If NamespacePath is root, this will return the global quota.
|
|
quota, err = quotaFetchFunc(indexNamespace, req.NamespacePath, false, false, false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if quota != nil {
|
|
return quota, nil
|
|
}
|
|
|
|
// If the request belongs to "root" namespace, then we have already looked at
|
|
// global quotas when fetching namespace specific quota rule. When the request
|
|
// belongs to a non-root namespace, and when there are no namespace specific
|
|
// quota rules present, we fallback on the global quotas.
|
|
if req.NamespacePath == "root" {
|
|
return nil, nil
|
|
}
|
|
|
|
// Fetch global quota
|
|
quota, err = quotaFetchFunc(indexNamespace, "root", false, false, false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if quota != nil {
|
|
return quota, nil
|
|
}
|
|
|
|
return nil, nil
|
|
}
|
|
|
|
// QueryResolveRoleQuotas checks if there's a quota for the request mount path
|
|
// which requires ResolveRoleOperation.
|
|
func (m *Manager) QueryResolveRoleQuotas(req *Request) (bool, error) {
|
|
m.dbAndCacheLock.RLock()
|
|
defer m.dbAndCacheLock.RUnlock()
|
|
|
|
txn := m.db.Txn(false)
|
|
|
|
// ns would have been made non-empty during insertion. Use non-empty
|
|
// value during query as well.
|
|
if req.NamespacePath == "" {
|
|
req.NamespacePath = "root"
|
|
}
|
|
|
|
// Check for any role-based quotas on the request namespaces/mount path.
|
|
for _, qType := range quotaTypes() {
|
|
// Use the namespace and mount as indexes and find all matches with a
|
|
// set Role field (see: 'true' as the last argument). We can't use
|
|
// indexNamespaceMountRole for this, because Role is a StringFieldIndex,
|
|
// which won't match on an empty string.
|
|
quota, err := txn.First(qType, indexNamespaceMount, req.NamespacePath, req.MountPath, false, true)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if quota != nil {
|
|
return true, nil
|
|
}
|
|
}
|
|
return false, nil
|
|
}
|
|
|
|
// DeleteQuota removes a quota rule from the db for a given name
|
|
func (m *Manager) DeleteQuota(ctx context.Context, qType string, name string) error {
|
|
m.quotaLock.Lock()
|
|
m.dbAndCacheLock.RLock()
|
|
defer m.quotaLock.Unlock()
|
|
defer m.dbAndCacheLock.RUnlock()
|
|
|
|
txn := m.db.Txn(true)
|
|
defer txn.Abort()
|
|
|
|
raw, err := txn.First(qType, indexName, name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if raw == nil {
|
|
return nil
|
|
}
|
|
|
|
quota := raw.(Quota)
|
|
if err := quota.close(ctx); err != nil {
|
|
return err
|
|
}
|
|
|
|
err = txn.Delete(qType, raw)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// For the lease count type, recompute the counters
|
|
if qType == TypeLeaseCount.String() {
|
|
if err := m.recomputeLeaseCounts(ctx, txn); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
txn.Commit()
|
|
return nil
|
|
}
|
|
|
|
// ApplyQuota runs the request against any quota rule that is applicable to it. If
|
|
// there are multiple quota rule that matches the request parameters, rule that
|
|
// takes precedence will be used to allow/reject the request.
|
|
func (m *Manager) ApplyQuota(ctx context.Context, req *Request) (Response, error) {
|
|
var resp Response
|
|
|
|
quota, err := m.QueryQuota(req)
|
|
if err != nil {
|
|
return resp, err
|
|
}
|
|
|
|
// If there is no quota defined, allow the request.
|
|
if quota == nil {
|
|
resp.Allowed = true
|
|
return resp, nil
|
|
}
|
|
|
|
// If the quota type is lease count, and if the path is not known to
|
|
// generate leases, allow the request.
|
|
if req.Type == TypeLeaseCount && !m.inLeasePathCache(req.Path) {
|
|
resp.Allowed = true
|
|
return resp, nil
|
|
}
|
|
|
|
return quota.allow(ctx, req)
|
|
}
|
|
|
|
// SetEnableRateLimitAuditLogging updates the operator preference regarding the
|
|
// audit logging behavior.
|
|
func (m *Manager) SetEnableRateLimitAuditLogging(val bool) {
|
|
m.quotaLock.Lock()
|
|
defer m.quotaLock.Unlock()
|
|
m.setEnableRateLimitAuditLoggingLocked(val)
|
|
}
|
|
|
|
func (m *Manager) setEnableRateLimitAuditLoggingLocked(val bool) {
|
|
m.config.EnableRateLimitAuditLogging = val
|
|
}
|
|
|
|
// SetEnableRateLimitResponseHeaders updates the operator preference regarding
|
|
// the rate limit quota HTTP header behavior.
|
|
func (m *Manager) SetEnableRateLimitResponseHeaders(val bool) {
|
|
m.quotaConfigLock.Lock()
|
|
defer m.quotaConfigLock.Unlock()
|
|
m.setEnableRateLimitResponseHeadersLocked(val)
|
|
}
|
|
|
|
func (m *Manager) setEnableRateLimitResponseHeadersLocked(val bool) {
|
|
m.config.EnableRateLimitResponseHeaders = val
|
|
}
|
|
|
|
// SetRateLimitExemptPaths updates the rate limit exempt paths in the Manager's
|
|
// configuration in addition to updating the path manager. Every call to
|
|
// SetRateLimitExemptPaths will wipe out the existing path manager and set the
|
|
// paths based on the provided argument.
|
|
func (m *Manager) SetRateLimitExemptPaths(vals []string) {
|
|
m.quotaConfigLock.Lock()
|
|
defer m.quotaConfigLock.Unlock()
|
|
m.setRateLimitExemptPathsLocked(vals)
|
|
}
|
|
|
|
func (m *Manager) setRateLimitExemptPathsLocked(vals []string) {
|
|
if vals == nil {
|
|
vals = []string{}
|
|
}
|
|
m.config.RateLimitExemptPaths = vals
|
|
m.rateLimitPathManager = pathmanager.New()
|
|
m.rateLimitPathManager.AddPaths(vals)
|
|
}
|
|
|
|
// RateLimitAuditLoggingEnabled returns if the quota configuration allows audit
|
|
// logging of request rejections due to rate limiting quota rule violations.
|
|
func (m *Manager) RateLimitAuditLoggingEnabled() bool {
|
|
m.quotaConfigLock.RLock()
|
|
defer m.quotaConfigLock.RUnlock()
|
|
|
|
return m.config.EnableRateLimitAuditLogging
|
|
}
|
|
|
|
// RateLimitResponseHeadersEnabled returns if the quota configuration allows for
|
|
// rate limit quota HTTP headers to be added to responses.
|
|
func (m *Manager) RateLimitResponseHeadersEnabled() bool {
|
|
m.quotaConfigLock.RLock()
|
|
defer m.quotaConfigLock.RUnlock()
|
|
|
|
return m.config.EnableRateLimitResponseHeaders
|
|
}
|
|
|
|
// RateLimitPathExempt returns a boolean dictating if a given path is exempt from
|
|
// any rate limit quota. If not rate limit path manager is defined, false is
|
|
// returned.
|
|
func (m *Manager) RateLimitPathExempt(path string) bool {
|
|
m.quotaConfigLock.RLock()
|
|
defer m.quotaConfigLock.RUnlock()
|
|
|
|
if m.rateLimitPathManager == nil {
|
|
return false
|
|
}
|
|
|
|
return m.rateLimitPathManager.HasPath(path)
|
|
}
|
|
|
|
// Config returns the operator preferences in the quota manager
|
|
func (m *Manager) Config() *Config {
|
|
return m.config
|
|
}
|
|
|
|
// Reset will clear all the quotas from the db and clear the lease path cache.
|
|
func (m *Manager) Reset() error {
|
|
m.quotaLock.Lock()
|
|
defer m.quotaLock.Unlock()
|
|
m.dbAndCacheLock.Lock()
|
|
defer m.dbAndCacheLock.Unlock()
|
|
|
|
err := m.resetCache()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
m.storage = nil
|
|
m.ctx = nil
|
|
|
|
return m.entManager.Reset()
|
|
}
|
|
|
|
// Must be called with the lock held
|
|
func (m *Manager) resetCache() error {
|
|
names, err := m.quotaNamesLocked(TypeRateLimit)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, name := range names {
|
|
quota, err := m.quotaByNameLocked(TypeRateLimit.String(), name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if quota != nil {
|
|
rlq := quota.(*RateLimitQuota)
|
|
|
|
// Cancel the quota's purgeBlockedClients goroutine
|
|
err = rlq.close(context.Background())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = rlq.store.Close(context.Background())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
db, err := memdb.NewMemDB(dbSchema())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
m.db = db
|
|
return nil
|
|
}
|
|
|
|
// dbSchema creates a DB schema for holding all the quota rules. It creates
|
|
// table for each supported type of quota.
|
|
func dbSchema() *memdb.DBSchema {
|
|
schema := &memdb.DBSchema{
|
|
Tables: make(map[string]*memdb.TableSchema),
|
|
}
|
|
|
|
commonSchema := func(name string) *memdb.TableSchema {
|
|
return &memdb.TableSchema{
|
|
Name: name,
|
|
Indexes: map[string]*memdb.IndexSchema{
|
|
indexID: {
|
|
Name: indexID,
|
|
Unique: true,
|
|
Indexer: &memdb.StringFieldIndex{
|
|
Field: "ID",
|
|
},
|
|
},
|
|
indexName: {
|
|
Name: indexName,
|
|
Unique: true,
|
|
Indexer: &memdb.StringFieldIndex{
|
|
Field: "Name",
|
|
},
|
|
},
|
|
indexNamespace: {
|
|
Name: indexNamespace,
|
|
Indexer: &memdb.CompoundMultiIndex{
|
|
Indexes: []memdb.Indexer{
|
|
&memdb.StringFieldIndex{
|
|
Field: "NamespacePath",
|
|
},
|
|
// By sending false as the query parameter, we can
|
|
// query just the namespace specific quota.
|
|
&memdb.FieldSetIndex{
|
|
Field: "MountPath",
|
|
},
|
|
// By sending false as the query parameter, we can
|
|
// query just the namespace specific quota.
|
|
&memdb.FieldSetIndex{
|
|
Field: "PathSuffix",
|
|
},
|
|
// By sending false as the query parameter, we can
|
|
// query just the namespace specific quota.
|
|
&memdb.FieldSetIndex{
|
|
Field: "Role",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
indexNamespaceMount: {
|
|
Name: indexNamespaceMount,
|
|
AllowMissing: true,
|
|
Indexer: &memdb.CompoundMultiIndex{
|
|
Indexes: []memdb.Indexer{
|
|
&memdb.StringFieldIndex{
|
|
Field: "NamespacePath",
|
|
},
|
|
&memdb.StringFieldIndex{
|
|
Field: "MountPath",
|
|
},
|
|
// By sending false as the query parameter, we can
|
|
// query just the namespace specific quota.
|
|
&memdb.FieldSetIndex{
|
|
Field: "PathSuffix",
|
|
},
|
|
// By sending false as the query parameter, we can
|
|
// query just the namespace specific quota.
|
|
&memdb.FieldSetIndex{
|
|
Field: "Role",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
indexNamespaceMountRole: {
|
|
Name: indexNamespaceMountRole,
|
|
AllowMissing: true,
|
|
Indexer: &memdb.CompoundMultiIndex{
|
|
Indexes: []memdb.Indexer{
|
|
&memdb.StringFieldIndex{
|
|
Field: "NamespacePath",
|
|
},
|
|
&memdb.StringFieldIndex{
|
|
Field: "MountPath",
|
|
},
|
|
// By sending false as the query parameter, we can
|
|
// query just the role specific quota.
|
|
&memdb.FieldSetIndex{
|
|
Field: "PathSuffix",
|
|
},
|
|
&memdb.StringFieldIndex{
|
|
Field: "Role",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
indexNamespaceMountPath: {
|
|
Name: indexNamespaceMountPath,
|
|
AllowMissing: true,
|
|
Indexer: &memdb.CompoundMultiIndex{
|
|
Indexes: []memdb.Indexer{
|
|
&memdb.StringFieldIndex{
|
|
Field: "NamespacePath",
|
|
},
|
|
&memdb.StringFieldIndex{
|
|
Field: "MountPath",
|
|
},
|
|
&memdb.StringFieldIndex{
|
|
Field: "PathSuffix",
|
|
},
|
|
// By sending false as the query parameter, we can
|
|
// query just the namespace specific quota.
|
|
&memdb.FieldSetIndex{
|
|
Field: "Role",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// Create a table per quota type. This allows names to be reused between
|
|
// different quota types and querying a bit easier.
|
|
for _, name := range quotaTypes() {
|
|
schema.Tables[name] = commonSchema(name)
|
|
}
|
|
|
|
return schema
|
|
}
|
|
|
|
// Invalidate receives notifications from the replication sub-system when a key
|
|
// is updated in the storage. This function will read the key from storage and
|
|
// updates the caches and data structures to reflect those updates.
|
|
func (m *Manager) Invalidate(key string) {
|
|
switch key {
|
|
case "config":
|
|
config, err := LoadConfig(m.ctx, m.storage)
|
|
if err != nil {
|
|
m.logger.Error("failed to invalidate quota config", "error", err)
|
|
return
|
|
}
|
|
|
|
m.SetEnableRateLimitAuditLogging(config.EnableRateLimitAuditLogging)
|
|
m.SetEnableRateLimitResponseHeaders(config.EnableRateLimitResponseHeaders)
|
|
m.SetRateLimitExemptPaths(config.RateLimitExemptPaths)
|
|
|
|
default:
|
|
splitKeys := strings.Split(key, "/")
|
|
if len(splitKeys) != 2 {
|
|
m.logger.Error("incorrect key while invalidating quota rule", "key", key)
|
|
return
|
|
}
|
|
qType := splitKeys[0]
|
|
name := splitKeys[1]
|
|
|
|
if qType == TypeLeaseCount.String() && m.isDRSecondary {
|
|
// lease count invalidation not supported on DR Secondary
|
|
return
|
|
}
|
|
|
|
// Read quota rule from storage
|
|
quota, err := Load(m.ctx, m.storage, qType, name)
|
|
if err != nil {
|
|
m.logger.Error("failed to read invalidated quota rule", "error", err)
|
|
return
|
|
}
|
|
|
|
switch {
|
|
case quota == nil:
|
|
// Handle quota deletion
|
|
if err := m.DeleteQuota(m.ctx, qType, name); err != nil {
|
|
m.logger.Error("failed to delete invalidated quota rule", "error", err)
|
|
return
|
|
}
|
|
default:
|
|
// Handle quota update
|
|
if err := m.SetQuota(m.ctx, qType, quota, false); err != nil {
|
|
m.logger.Error("failed to update invalidated quota rule", "error", err)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// LoadConfig reads the quota configuration from the underlying storage
|
|
func LoadConfig(ctx context.Context, storage logical.Storage) (*Config, error) {
|
|
var config Config
|
|
entry, err := storage.Get(ctx, ConfigPath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if entry == nil {
|
|
return &config, nil
|
|
}
|
|
|
|
err = entry.DecodeJSON(&config)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &config, nil
|
|
}
|
|
|
|
// Load reads the quota rule from the underlying storage
|
|
func Load(ctx context.Context, storage logical.Storage, qType, name string) (Quota, error) {
|
|
var quota Quota
|
|
entry, err := storage.Get(ctx, QuotaStoragePath(qType, name))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if entry == nil {
|
|
return nil, nil
|
|
}
|
|
|
|
switch qType {
|
|
case TypeRateLimit.String():
|
|
quota = &RateLimitQuota{}
|
|
case TypeLeaseCount.String():
|
|
quota = &LeaseCountQuota{}
|
|
default:
|
|
return nil, fmt.Errorf("unsupported type: %v", qType)
|
|
}
|
|
|
|
err = entry.DecodeJSON(quota)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return quota, nil
|
|
}
|
|
|
|
// Setup loads the quota configuration and all the quota rules into the
|
|
// quota manager.
|
|
func (m *Manager) Setup(ctx context.Context, storage logical.Storage, isPerfStandby, isDRSecondary bool) error {
|
|
m.quotaLock.Lock()
|
|
m.quotaConfigLock.Lock()
|
|
m.dbAndCacheLock.Lock()
|
|
defer m.quotaLock.Unlock()
|
|
defer m.quotaConfigLock.Unlock()
|
|
defer m.dbAndCacheLock.Unlock()
|
|
|
|
m.storage = storage
|
|
m.ctx = ctx
|
|
m.isPerfStandby = isPerfStandby
|
|
m.isDRSecondary = isDRSecondary
|
|
|
|
// Load the quota configuration from storage and load it into the quota
|
|
// manager.
|
|
config, err := LoadConfig(ctx, storage)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
entry, err := storage.Get(ctx, DefaultRateLimitExemptPathsToggle)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Determine if we need to set the default set of exempt paths for rate limit
|
|
// resource quotas. We use a default set introduced in 1.5 when the toggle
|
|
// entry does not exist in storage or is false. The toggle is flipped , i.e.
|
|
// set to true when SetRateLimitExemptPaths is called during a config update.
|
|
var toggle bool
|
|
if entry != nil {
|
|
if err := entry.DecodeJSON(&toggle); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
exemptPaths := defaultExemptPaths
|
|
if toggle {
|
|
exemptPaths = config.RateLimitExemptPaths
|
|
}
|
|
|
|
m.setEnableRateLimitAuditLoggingLocked(config.EnableRateLimitAuditLogging)
|
|
m.setEnableRateLimitResponseHeadersLocked(config.EnableRateLimitResponseHeaders)
|
|
m.setRateLimitExemptPathsLocked(exemptPaths)
|
|
if err = m.resetCache(); err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, qType := range quotaTypes() {
|
|
m.setupQuotaType(ctx, storage, qType)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (m *Manager) setupQuotaType(ctx context.Context, storage logical.Storage, quotaType string) error {
|
|
if quotaType == TypeLeaseCount.String() && m.isDRSecondary {
|
|
m.logger.Trace("lease count quotas are not processed on DR Secondaries")
|
|
return nil
|
|
}
|
|
|
|
names, err := logical.CollectKeys(ctx, logical.NewStorageView(storage, StoragePrefix+quotaType+"/"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, name := range names {
|
|
quota, err := Load(ctx, m.storage, quotaType, name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if quota == nil {
|
|
continue
|
|
}
|
|
|
|
err = m.setQuotaLocked(ctx, quotaType, quota, true)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// QuotaStoragePath returns the storage path suffix for persisting the quota
|
|
// rule.
|
|
func QuotaStoragePath(quotaType, name string) string {
|
|
return path.Join(StoragePrefix+quotaType, name)
|
|
}
|
|
|
|
// HandleRemount updates the quota subsystem about the remount operation that
|
|
// took place. Quota manager will trigger the quota specific updates including
|
|
// the mount path update and the namespace update
|
|
func (m *Manager) HandleRemount(ctx context.Context, from, to namespace.MountPathDetails) error {
|
|
m.quotaLock.Lock()
|
|
m.dbAndCacheLock.RLock()
|
|
defer m.quotaLock.Unlock()
|
|
defer m.dbAndCacheLock.RUnlock()
|
|
|
|
// Grab a write transaction, as we want to save the updated quota in memdb
|
|
txn := m.db.Txn(true)
|
|
defer txn.Abort()
|
|
|
|
// quota namespace would have been made non-empty during insertion. Use non-empty value
|
|
// during query as well.
|
|
fromNs := from.Namespace.Path
|
|
if fromNs == "" {
|
|
fromNs = namespace.RootNamespaceID
|
|
}
|
|
|
|
toNs := to.Namespace.Path
|
|
if toNs == "" {
|
|
toNs = namespace.RootNamespaceID
|
|
}
|
|
|
|
leaseQuotaUpdated := false
|
|
|
|
updateMounts := func(idx string, args ...interface{}) error {
|
|
for _, quotaType := range quotaTypes() {
|
|
iter, err := txn.Get(quotaType, idx, args...)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for raw := iter.Next(); raw != nil; raw = iter.Next() {
|
|
quota := raw.(Quota)
|
|
|
|
// Clone the object and update it
|
|
clonedQuota := quota.Clone()
|
|
clonedQuota.handleRemount(to.MountPath, toNs)
|
|
// Update both underlying storage and memdb with the quota change
|
|
entry, err := logical.StorageEntryJSON(QuotaStoragePath(quotaType, quota.QuotaName()), quota)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := m.storage.Put(ctx, entry); err != nil {
|
|
return err
|
|
}
|
|
if err := m.setQuotaLockedWithTxn(ctx, quotaType, clonedQuota, false, txn); err != nil {
|
|
return err
|
|
}
|
|
if quotaType == TypeLeaseCount.String() {
|
|
leaseQuotaUpdated = true
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Update mounts for everything without a path prefix or role
|
|
err := updateMounts(indexNamespaceMount, fromNs, from.MountPath, false, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Update mounts for everything with a path prefix
|
|
err = updateMounts(indexNamespaceMount, fromNs, from.MountPath, true, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Update mounts for everything with a role
|
|
err = updateMounts(indexNamespaceMount, fromNs, from.MountPath, false, true)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if leaseQuotaUpdated {
|
|
if err := m.recomputeLeaseCounts(ctx, txn); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
txn.Commit()
|
|
|
|
return nil
|
|
}
|
|
|
|
// HandleBackendDisabling updates the quota subsystem with the disabling of auth
|
|
// or secret engine disabling. This should only be called on the primary cluster
|
|
// node.
|
|
func (m *Manager) HandleBackendDisabling(ctx context.Context, nsPath, mountPath string) error {
|
|
m.quotaLock.Lock()
|
|
m.dbAndCacheLock.RLock()
|
|
defer m.quotaLock.Unlock()
|
|
defer m.dbAndCacheLock.RUnlock()
|
|
|
|
txn := m.db.Txn(true)
|
|
defer txn.Abort()
|
|
|
|
// nsPath would have been made non-empty during insertion. Use non-empty value
|
|
// during query as well.
|
|
if nsPath == "" {
|
|
nsPath = "root"
|
|
}
|
|
|
|
leaseQuotaDeleted := false
|
|
|
|
updateMounts := func(idx string, args ...interface{}) error {
|
|
for _, quotaType := range quotaTypes() {
|
|
iter, err := txn.Get(quotaType, idx, args...)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for raw := iter.Next(); raw != nil; raw = iter.Next() {
|
|
if err := txn.Delete(quotaType, raw); err != nil {
|
|
return fmt.Errorf("failed to delete quota from db after mount disabling; namespace %q, err %v", nsPath, err)
|
|
}
|
|
quota := raw.(Quota)
|
|
if err := m.storage.Delete(ctx, QuotaStoragePath(quotaType, quota.QuotaName())); err != nil {
|
|
return fmt.Errorf("failed to delete quota from storage after mount disabling; namespace %q, err %v", nsPath, err)
|
|
}
|
|
if quotaType == TypeLeaseCount.String() {
|
|
leaseQuotaDeleted = true
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Update mounts for everything without a path prefix or role
|
|
err := updateMounts(indexNamespaceMount, nsPath, mountPath, false, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Update mounts for everything with a path prefix
|
|
err = updateMounts(indexNamespaceMount, nsPath, mountPath, true, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Update mounts for everything with a role
|
|
err = updateMounts(indexNamespaceMount, nsPath, mountPath, false, true)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if leaseQuotaDeleted {
|
|
if err := m.recomputeLeaseCounts(ctx, txn); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
txn.Commit()
|
|
|
|
return nil
|
|
}
|
|
|
|
func (m *Manager) DetectDeadlocks() bool {
|
|
if _, ok := m.quotaLock.(*locking.DeadlockRWMutex); ok {
|
|
return true
|
|
}
|
|
return false
|
|
}
|