
worker
Index
- Constants
- func CleanupBackoff(attempts int32) time.Duration
- func ExceedsThreshold(stats map[string]core.QuotaStat, order []string, threshold float64) bool
- func NewCleanupQueueService(cleanup *CleanupWorker, locker tickrunner.AdvisoryLocker) lifecycle.Runner
- func NewOverReplicationService(manager tickrunner.QuotaMetricsRefresher, overRep *OverReplicationCleaner, locker tickrunner.AdvisoryLocker) lifecycle.Runner
- func NewPendingReaperService(reaper *PendingReaper, locker tickrunner.AdvisoryLocker, tick time.Duration) lifecycle.Runner
- func NewRebalancerService(manager tickrunner.QuotaMetricsRefresher, rebalancer *Rebalancer, locker tickrunner.AdvisoryLocker) lifecycle.Runner
- func NewReconcileService(reconciler *Reconciler, locker tickrunner.AdvisoryLocker, interval time.Duration) lifecycle.Runner
- func NewReplicatorService(manager tickrunner.QuotaMetricsRefresher, replicator *Replicator, locker tickrunner.AdvisoryLocker) lifecycle.Runner
- func NewScrubberService(scrubber *Scrubber, locker tickrunner.AdvisoryLocker) lifecycle.Runner
- func WithAdmission(ctx context.Context, ac AdmissionControl, name string, fn func())
- type AdmissionControl
- type BackendAccess
- type BackendSyncer
- type BatchRunner
- type CleanupOps
- type CleanupWorker
- type CleanupWorkerDeps
- type CleanupWorkerStore
- type DataMover
- type ItemOutcome
- type ItemResult
- type Ops
- type OverReplicationCleaner
- func NewOverReplicationCleaner(ops Ops, placement Placement, store OverReplicationCleanerStore) *OverReplicationCleaner
- func (c *OverReplicationCleaner) Clean(ctx context.Context, cfg config.ReplicationConfig, observer progress.Observer) (int, error)
- func (c *OverReplicationCleaner) Config() *config.ReplicationConfig
- func (c *OverReplicationCleaner) CountPending(ctx context.Context, factor int) (int64, error)
- func (c *OverReplicationCleaner) ScoreCopy(loc *core.ObjectLocation, stats map[string]core.QuotaStat) float64
- func (c *OverReplicationCleaner) SetConfig(cfg *config.ReplicationConfig)
- type OverReplicationCleanerStore
- type PendingReaper
- type PendingReaperDeps
- type PendingReaperStore
- type Placement
- type PlacementSet
- type RebalanceMove
- type Rebalancer
- func NewRebalancer(ops Ops, placement Placement, store RebalancerStore) *Rebalancer
- func (r *Rebalancer) Config() *config.RebalanceConfig
- func (r *Rebalancer) ExecuteMoves(ctx context.Context, plan []RebalanceMove, strategy string, concurrency int) WorkSummary
- func (r *Rebalancer) ExecuteOneMove(ctx context.Context, move RebalanceMove, strategy string) bool
- func (r *Rebalancer) PlanPackTight(ctx context.Context, stats map[string]core.QuotaStat, batchSize int) ([]RebalanceMove, error)
- func (r *Rebalancer) PlanSpreadEven(ctx context.Context, stats map[string]core.QuotaStat, batchSize int) ([]RebalanceMove, error)
- func (r *Rebalancer) Rebalance(ctx context.Context, cfg config.RebalanceConfig) (WorkSummary, error)
- func (r *Rebalancer) SetConfig(cfg *config.RebalanceConfig)
- type RebalancerStore
- type ReconcileResult
- type Reconciler
- func NewReconciler(syncer BackendSyncer, bucketNames []string) *Reconciler
- func (r *Reconciler) Reconcile(ctx context.Context, backendName string) (*ReconcileResult, error)
- func (r *Reconciler) ReconcileStreaming(ctx context.Context, backendName string, observer progress.Observer) (*ReconcileResult, error)
- func (r *Reconciler) Run(ctx context.Context)
- type RecorderProvider
- type ReplicationOutcome
- type Replicator
- func NewReplicator(ops Ops, placement Placement, store ReplicatorStore) *Replicator
- func (r *Replicator) CleanupOrphan(ctx context.Context, backendName, key string, sizeBytes int64)
- func (r *Replicator) Config() *config.ReplicationConfig
- func (r *Replicator) CopyToReplica(ctx context.Context, key string, copies []core.ObjectLocation, target string) (string, int64, error)
- func (r *Replicator) FindReplicaTarget(ctx context.Context, key string, size int64, exclusion map[string]bool) string
- func (r *Replicator) IsBackendHealthy(name string) bool
- func (r *Replicator) Replicate(ctx context.Context, cfg config.ReplicationConfig, observer progress.Observer) (int, error)
- func (r *Replicator) ReplicateObject(ctx context.Context, key string, existingCopies []core.ObjectLocation, needed int) ReplicationOutcome
- func (r *Replicator) SetConfig(cfg *config.ReplicationConfig)
- func (r *Replicator) UnhealthyBackends(threshold time.Duration) []string
- type ReplicatorStore
- type Scrubber
- func NewScrubber(deps ScrubberDeps) *Scrubber
- func (s *Scrubber) Backfill(ctx context.Context, batchSize, offset int, observer progress.Observer) (WorkSummary, int)
- func (s *Scrubber) Config() *config.IntegrityConfig
- func (s *Scrubber) Scrub(ctx context.Context, batchSize int, observer progress.Observer) WorkSummary
- func (s *Scrubber) SetConfig(cfg *config.IntegrityConfig)
- type ScrubberDeps
- type ScrubberOps
- type ScrubberStore
- type SourceCandidates
- type UsageAccessor
- type WorkSummary
Constants
Worker name labels used by admission-rejection metric and other worker-scoped telemetry. Constants prevent the silent label drift that would otherwise split a metric’s time series in Grafana when a typo lands.
DefaultCleanupQueueTick is the cleanup-queue worker’s per-tick cadence when the config does not override it.
DefaultOverReplicationTick is the over-replication cleaner’s per-tick cadence when the config does not specify one.
DefaultPendingReaperTick is the pending reaper’s per-tick cadence when the config does not specify one.
DefaultRebalanceInterval is the rebalancer’s per-tick cadence when the config does not specify one.
DefaultReplicatorTick is the replicator’s per-tick cadence when the config does not specify one.
DefaultScrubberInterval is the integrity scrubber’s per-tick cadence when the integrity config does not specify one.
func CleanupBackoff
CleanupBackoff returns the backoff duration for the given attempt number. Uses exponential backoff: min(1m * 2^attempts, 24h). Short-circuits the shift for attempts >= 11 (where the doubling already exceeds the cap) and for negative inputs, since shifting by a negative or out-of-range count is undefined in Go.
func ExceedsThreshold
ExceedsThreshold reports whether the utilization spread across backends (max ratio minus min ratio) is at least the configured threshold.
func NewCleanupQueueService
NewCleanupQueueService constructs the cleanup-queue background service.
func NewOverReplicationService
NewOverReplicationService constructs the over-replication cleanup service.
func NewPendingReaperService
NewPendingReaperService constructs the pending-objects reaper background service. The reaper resolves abandoned PUT intents by HEADing the destination backend and either promoting the intent into object_locations (bytes present) or dropping it (bytes absent). Returns nil when no pending reaper is configured.
func NewRebalancerService
NewRebalancerService constructs the rebalancer background service.
func NewReconcileService
NewReconcileService constructs the reconcile background service.
func NewReplicatorService
NewReplicatorService constructs the replication background service.
func NewScrubberService
NewScrubberService constructs the integrity scrubber background service.
func WithAdmission
WithAdmission acquires an admission slot, runs fn if granted, and releases the slot afterward. When admission is rejected, increments the per-worker rejection counter and returns without invoking fn. name must be one of the WorkerName* constants above.
type AdmissionControl
AdmissionControl gates concurrent access to backends.
type BackendAccess
BackendAccess provides backend fleet discovery and drain-awareness.
type BackendSyncer
BackendSyncer is the interface the reconciler needs from the proxy layer. Defined here to avoid a worker->proxy import cycle.
type BatchRunner
BatchRunner drives one worker cycle. Concurrency is passed through to workerpool.Run (1 = sequential); Key, when set, supplies each item’s progress label; Observer, when set, receives the per-item progress steps.
func (BatchRunner[T]) Run
Run processes items through fn with the configured concurrency, brackets each in a progress step, and returns the tallied WorkSummary. Items not reached because ctx was cancelled mid-cycle are left out of the tally, so Planned - (Attempted + Skipped) reflects the cancelled remainder.
type CleanupOps
CleanupOps is the dependency contract for CleanupWorker. It omits BackendAccess because cleanup operates on specific named backends.
type CleanupWorker
CleanupWorker processes the retry queue for failed object deletions.
func NewCleanupWorker
NewCleanupWorker creates a CleanupWorker with the given dependencies.
func (*CleanupWorker) ProcessCleanupQueue
ProcessCleanupQueue fetches pending cleanup items and attempts to delete the orphaned objects from their respective backends.
type CleanupWorkerDeps
CleanupWorkerDeps groups the cleanup worker’s constructor parameters. InstanceID is stamped into cleanup_queue.claimed_by for observability; ClaimGracePeriod is the threshold past which an outstanding claim becomes reclaimable by another worker tick (typically 5m).
type CleanupWorkerStore
CleanupWorkerStore is the narrow persistence surface the cleanup worker needs: the cleanup queue + DLQ operations. Declared locally so the worker does not pull in the full MetadataStore.
type DataMover
DataMover provides object data movement primitives.
type ItemOutcome
ItemOutcome classifies how a single work item finished, for the batch tally.
type ItemResult
ItemResult is what a per-item function returns: the outcome for the tally and a human-readable status for the progress stream (ignored when no observer is attached).
type Ops
Ops combines fleet access, admission control, data movement, and usage tracking. Used by Rebalancer, Replicator, and OverReplicationCleaner.
type OverReplicationCleaner
OverReplicationCleaner removes excess copies of objects that exceed the configured replication factor. Embeds *backendCore for access to shared infrastructure.
func NewOverReplicationCleaner
NewOverReplicationCleaner creates a cleaner with fleet operations, write-path placement, and a metadata store.
func (*OverReplicationCleaner) Clean
Clean finds over-replicated objects and removes excess copies to reach the target replication factor. Returns the number of copies removed. observer, when non-nil, receives a start and end step per object cleaned.
func (*OverReplicationCleaner) Config
Config returns the current replication configuration.
func (*OverReplicationCleaner) CountPending
CountPending returns the number of objects exceeding the replication factor.
func (*OverReplicationCleaner) ScoreCopy
ScoreCopy assigns a retention score to a copy based on its backend’s state:
- draining backend: 0 (always remove first)
- circuit-broken backend: 1 (remove next)
- healthy backend: 2 + (1 - utilization_ratio), range [2..3]
Among healthy backends, the most utilized backend gets the lowest score, making its copy the first candidate for removal -- freeing space where it is scarcest.
func (*OverReplicationCleaner) SetConfig
SetConfig atomically stores the replication configuration.
type OverReplicationCleanerStore
OverReplicationCleanerStore is the narrow persistence surface the over-replication cleaner needs: over-replicated discovery + excess copy removal + per-backend quota stats for the deletion target. Declared locally so the worker does not pull in the full MetadataStore.
type PendingReaper
PendingReaper resolves abandoned PUT intents by inspecting the destination backend. The min-age window protects in-flight PUTs whose commit has not yet had a chance to clear the intent on the synchronous path.
func NewPendingReaper
NewPendingReaper creates a PendingReaper with the given dependencies.
func (*PendingReaper) ProcessPendingQueue
ProcessPendingQueue runs one reaper tick: fetch stale intents, HEAD their destinations, and promote or drop based on what the backend reports. Returns the number of intents that completed (committed or dropped) and the number that failed (left for the next tick).
type PendingReaperDeps
PendingReaperDeps groups the pending reaper’s constructor parameters. Concurrency, MinAge, and BatchSize fall back to safe defaults when zero or negative.
type PendingReaperStore
PendingReaperStore is the narrow persistence surface the pending reaper needs. Declared locally so the worker does not pull in the full MetadataStore.
type Placement
Placement is the store-coupled write-path facet (target selection, move, delete-or-enqueue) workers get from the BackendManager, kept apart from the runtime roles so the manager need not re-export runtime methods.
type PlacementSet
PlacementSet maps an object key to the set of backends that already hold it. Built once from a copy map (key -> backend names) so planners can ask “is key already on backend?” without rescanning a slice or allocating a lookup map per object.
func NewPlacementSet
NewPlacementSet builds a PlacementSet from a key -> backend-names copy map.
func (PlacementSet) Has
Has reports whether key already lives on backend.
type RebalanceMove
RebalanceMove describes a single object move from one backend to another.
type Rebalancer
Rebalancer moves objects between backends to optimize space distribution.
func NewRebalancer
NewRebalancer creates a Rebalancer with fleet operations, write-path placement, and a metadata store.
func (*Rebalancer) Config
Config returns the current rebalance configuration.
func (*Rebalancer) ExecuteMoves
ExecuteMoves runs the planned object moves with bounded concurrency. Skips individual moves that fail and continues with the rest, returning the count of successful moves.
func (*Rebalancer) ExecuteOneMove
ExecuteOneMove performs a single object move: stream the bytes from source to destination, swap the DB location with compare-and-swap, and delete the source copy. Returns true when all steps succeed.
func (*Rebalancer) PlanPackTight
PlanPackTight consolidates objects onto the most-utilized backends by pulling from the least-utilized. Sorts by percent full descending and only moves an object from a less-full source to a more-full destination. Skips moves that would not increase the destination’s packing ratio.
func (*Rebalancer) PlanSpreadEven
PlanSpreadEven equalizes utilization ratios across backends by moving objects from over-utilized backends to under-utilized ones. Returns nil when no backend has a usable byte limit.
func (*Rebalancer) Rebalance
Rebalance moves objects between backends to optimize space distribution. Returns the number of objects successfully moved.
func (*Rebalancer) SetConfig
SetConfig atomically stores the rebalance configuration.
type RebalancerStore
RebalancerStore is the narrow persistence surface the rebalancer needs: per-backend listings + quota stats for the source/dest fill ratio. Declared locally so the worker does not pull in the full MetadataStore.
type ReconcileResult
ReconcileResult holds the outcome of a reconciliation pass for one backend.
type Reconciler
Reconciler scans backends for untracked objects and imports them into the metadata database.
func NewReconciler
NewReconciler creates a reconciler that uses the syncer’s SyncBackend to import untracked objects.
func (*Reconciler) Reconcile
Reconcile performs a full reconciliation for the given backend (or all backends if backendName is empty). Lists objects on each backend, diffs against DB entries, imports untracked objects, and removes stale entries.
func (*Reconciler) ReconcileStreaming
ReconcileStreaming is Reconcile with a per-backend observer. onBackend, when non-nil, is called after each backend is reconciled so a streaming caller can report incremental progress; pass nil for the non-streaming path.
func (*Reconciler) Run
Run performs a full reconciliation pass: for each backend, list all objects and import any that are not tracked in the metadata database.
type RecorderProvider
RecorderProvider provides the shared per-backend accounting recorder. Prefer this over UsageAccessor for per-attempt API call / byte counters: APICall / Egress / Ingress / Operation document intent at the call site and prevent argument-order bugs.
type ReplicationOutcome
ReplicationOutcome captures the per-object result of one ReplicateObject invocation. Counts are populated regardless of success so the reporter and unit tests can reason about retry behaviour without parsing log lines. NoTarget reflects whether the loop exited because target selection ran out, not whether any individual attempt failed.
func (ReplicationOutcome) Failed
Failed reports the total number of failed copy attempts for this object, irrespective of failure stage.
type Replicator
Replicator creates additional copies of under-replicated objects across backends.
func NewReplicator
NewReplicator creates a Replicator with fleet operations, write-path placement, and a metadata store.
func (*Replicator) CleanupOrphan
CleanupOrphan deletes an object from a backend when the DB record was not created (e.g. source was deleted during replication). Looks up the backend by name and dispatches to DeleteOrEnqueue, which handles its own API accounting and orphan-byte tracking.
func (*Replicator) Config
Config returns the current replication configuration.
func (*Replicator) CopyToReplica
copyToReplica reads the object from an existing copy and writes it to the target backend. Tries each existing copy in order for failover. Returns the source backend name that was successfully read from and the size_bytes recorded on that source’s ObjectLocation row (the size of the bytes that were actually transferred). The input slice is cloned before sorting so callers retain their original ordering — without the clone, sort.Slice reorders the caller’s slice in place and the caller’s later reads see a different element at each index.
func (*Replicator) FindReplicaTarget
FindReplicaTarget selects a backend for a replication copy using the same routing strategy as normal writes. Returns empty string if no suitable target exists.
func (*Replicator) IsBackendHealthy
IsBackendHealthy returns true if the backend has a closed circuit breaker or has no circuit breaker wrapper.
func (*Replicator) Replicate
Replicate finds under-replicated objects and creates additional copies to reach the target replication factor. Returns the number of copies created. observer, when non-nil, receives a start and end step per object replicated.
func (*Replicator) ReplicateObject
ReplicateObject creates up to `needed` additional copies of a single object. Returns a ReplicationOutcome the caller can use to drive metrics, audit, and log reporting without re-parsing logs.
Per-attempt diagnostic logs are preserved so incident responders can still trace each failed retry; the outcome is a *structured summary* on top of those, not a replacement.
func (*Replicator) SetConfig
SetConfig atomically stores the replication configuration.
func (*Replicator) UnhealthyBackends
UnhealthyBackends returns backend names whose circuit breakers have been open longer than the given threshold. Returns nil when all backends are healthy or circuit breakers are not enabled.
type ReplicatorStore
ReplicatorStore is the narrow persistence surface the replicator needs: under-replicated discovery + per-replica record / location updates. Declared locally so the worker does not pull in the full MetadataStore.
type Scrubber
Scrubber periodically verifies stored object integrity by reading objects from backends, computing their SHA-256 hash, and comparing against the stored content hash. Also supports backfilling hashes for objects that were written before integrity was enabled.
func NewScrubber
NewScrubber creates a Scrubber with the given dependencies.
func (*Scrubber) Backfill
Backfill reads objects that have no stored content hash, computes the SHA-256 digest, and stores it in the database. Processes up to batchSize objects starting at the given offset. observer, when non-nil, receives a start step before each object is hashed and an end step after, carrying the per-object outcome and duration. Returns the cycle summary and the next offset for pagination (0 when done).
func (*Scrubber) Config
Config returns the current integrity configuration.
func (*Scrubber) Scrub
Scrub verifies a batch of objects with stored content hashes. Returns the number of objects checked and the number of hash mismatches found.
func (*Scrubber) SetConfig
SetConfig atomically stores the integrity configuration.
type ScrubberDeps
ScrubberDeps groups the scrubber’s constructor dependencies. Encryptor is optional (nil when encryption is disabled).
type ScrubberOps
ScrubberOps is the dependency contract for Scrubber. It omits AdmissionControl and BackendAccess because integrity checks are best-effort background work.
type ScrubberStore
ScrubberStore is the narrow persistence surface the scrubber needs: integrity row reads/writes. Declared locally so the worker does not pull in the full MetadataStore.
type SourceCandidates
SourceCandidates bundles a source backend’s candidate objects with their placement set, fetched together so a planner pays the two store queries once per source.
type UsageAccessor
UsageAccessor provides usage tracking.
type WorkSummary
WorkSummary is the uniform result of one worker cycle. It replaces the ad-hoc count tuples (checked/failed, processed/failed, moved) each worker used to return, so partial-failure reporting and metric labels are consistent.
func (WorkSummary) Outcome
Outcome classifies the cycle for metric labels: success (work done, no failures), partial (some succeeded, some failed), failed (only failures), or empty (nothing succeeded or failed).
Generated by gomarkdoc