refactor(outbox): share domain backoff policy
This commit is contained in:
@@ -148,8 +148,8 @@ func (p *OutboxProcessor) handleFailure(ctx context.Context, event *OutboxEvent,
|
|||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// 计算下次重试时间(指数退避)
|
// 计算下次重试时间(指数退避)
|
||||||
backoffSeconds := calculateBackoff(event.RetryCount, event.MaxRetries)
|
backoffSeconds := CalculateOutboxBackoff(event.RetryCount, event.MaxRetries)
|
||||||
nextRetry := time.Now().Add(time.Duration(backoffSeconds) * time.Second)
|
nextRetry := time.Now().Add(time.Duration(backoffSeconds) * time.Second)
|
||||||
|
|
||||||
// 在存储层更新重试状态(这里简化处理)
|
// 在存储层更新重试状态(这里简化处理)
|
||||||
if err := p.eventStore.MarkFailed(ctx, event.EventID, publishErr.Error()); err != nil {
|
if err := p.eventStore.MarkFailed(ctx, event.EventID, publishErr.Error()); err != nil {
|
||||||
@@ -162,8 +162,8 @@ func (p *OutboxProcessor) handleFailure(ctx context.Context, event *OutboxEvent,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// calculateBackoff 计算指数退避时间
|
// CalculateOutboxBackoff 计算指数退避时间
|
||||||
func calculateBackoff(retryCount, maxRetries int) int {
|
func CalculateOutboxBackoff(retryCount, maxRetries int) int {
|
||||||
backoff := DefaultInitialBackoffSeconds * int(math.Pow(2, float64(retryCount-1)))
|
backoff := DefaultInitialBackoffSeconds * int(math.Pow(2, float64(retryCount-1)))
|
||||||
if backoff > DefaultMaxBackoffSeconds {
|
if backoff > DefaultMaxBackoffSeconds {
|
||||||
backoff = DefaultMaxBackoffSeconds
|
backoff = DefaultMaxBackoffSeconds
|
||||||
|
|||||||
@@ -56,7 +56,7 @@ func (m *mockOutboxEventStore) MarkFailed(ctx context.Context, eventID string, e
|
|||||||
if e, ok := m.events[eventID]; ok {
|
if e, ok := m.events[eventID]; ok {
|
||||||
e.Status = OutboxStatusFailed
|
e.Status = OutboxStatusFailed
|
||||||
e.ErrorMessage = errorMsg
|
e.ErrorMessage = errorMsg
|
||||||
backoff := calculateBackoff(e.RetryCount, e.MaxRetries)
|
backoff := CalculateOutboxBackoff(e.RetryCount, e.MaxRetries)
|
||||||
nextRetry := time.Now().Add(time.Duration(backoff) * time.Second)
|
nextRetry := time.Now().Add(time.Duration(backoff) * time.Second)
|
||||||
e.NextRetryAt = &nextRetry
|
e.NextRetryAt = &nextRetry
|
||||||
m.failed = append(m.failed, e)
|
m.failed = append(m.failed, e)
|
||||||
@@ -269,7 +269,7 @@ func TestP006_ExponentialBackoff(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
backoff := calculateBackoff(tt.retryCount, tt.maxRetries)
|
backoff := CalculateOutboxBackoff(tt.retryCount, tt.maxRetries)
|
||||||
if backoff < tt.expectedMin || backoff > tt.expectedMax {
|
if backoff < tt.expectedMin || backoff > tt.expectedMax {
|
||||||
t.Errorf("retry %d: expected backoff %d-%d, got %d",
|
t.Errorf("retry %d: expected backoff %d-%d, got %d",
|
||||||
tt.retryCount, tt.expectedMin, tt.expectedMax, backoff)
|
tt.retryCount, tt.expectedMin, tt.expectedMax, backoff)
|
||||||
@@ -280,7 +280,7 @@ func TestP006_ExponentialBackoff(t *testing.T) {
|
|||||||
// TestP006_MaxBackoffCap 验证退避时间上限
|
// TestP006_MaxBackoffCap 验证退避时间上限
|
||||||
func TestP006_MaxBackoffCap(t *testing.T) {
|
func TestP006_MaxBackoffCap(t *testing.T) {
|
||||||
// 即使重试很多次,退避时间也不应超过60秒
|
// 即使重试很多次,退避时间也不应超过60秒
|
||||||
backoff := calculateBackoff(100, 100)
|
backoff := CalculateOutboxBackoff(100, 100)
|
||||||
if backoff > DefaultMaxBackoffSeconds {
|
if backoff > DefaultMaxBackoffSeconds {
|
||||||
t.Errorf("backoff should be capped at %d, got %d", DefaultMaxBackoffSeconds, backoff)
|
t.Errorf("backoff should be capped at %d, got %d", DefaultMaxBackoffSeconds, backoff)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package outbox
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"math"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"lijiaoqiao/supply-api/internal/domain"
|
"lijiaoqiao/supply-api/internal/domain"
|
||||||
@@ -147,8 +146,8 @@ func (r *OutboxProcessorRunner) handleFailure(ctx context.Context, event *domain
|
|||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// 计算下次重试时间(指数退避)
|
// 计算下次重试时间(指数退避)
|
||||||
backoffSeconds := CalculateOutboxBackoff(event.RetryCount, event.MaxRetries)
|
backoffSeconds := domain.CalculateOutboxBackoff(event.RetryCount, event.MaxRetries)
|
||||||
nextRetry := time.Now().Add(time.Duration(backoffSeconds) * time.Second)
|
nextRetry := time.Now().Add(time.Duration(backoffSeconds) * time.Second)
|
||||||
|
|
||||||
if err := r.repo.MarkFailed(ctx, event.EventID, publishErr.Error(), &nextRetry); err != nil {
|
if err := r.repo.MarkFailed(ctx, event.EventID, publishErr.Error(), &nextRetry); err != nil {
|
||||||
r.stats.RecordOutboxFailure("mark_failed_failed")
|
r.stats.RecordOutboxFailure("mark_failed_failed")
|
||||||
@@ -157,14 +156,3 @@ func (r *OutboxProcessorRunner) handleFailure(ctx context.Context, event *domain
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// CalculateOutboxBackoff 计算指数退避时间
|
|
||||||
func CalculateOutboxBackoff(retryCount, maxRetries int) int {
|
|
||||||
initialBackoff := 1.0
|
|
||||||
maxBackoff := 60.0
|
|
||||||
backoff := initialBackoff * math.Pow(2, float64(retryCount-1))
|
|
||||||
if backoff > maxBackoff {
|
|
||||||
backoff = maxBackoff
|
|
||||||
}
|
|
||||||
return int(backoff)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -3,16 +3,21 @@ package outbox
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"lijiaoqiao/supply-api/internal/domain"
|
||||||
"lijiaoqiao/supply-api/internal/messaging"
|
"lijiaoqiao/supply-api/internal/messaging"
|
||||||
"lijiaoqiao/supply-api/internal/repository"
|
"lijiaoqiao/supply-api/internal/repository"
|
||||||
)
|
)
|
||||||
|
|
||||||
type stubRunnerRepo struct {
|
type stubRunnerRepo struct {
|
||||||
events []*repository.OutboxEvent
|
events []*repository.OutboxEvent
|
||||||
|
failedEventID string
|
||||||
|
failedErrorMsg string
|
||||||
|
failedNextRetryAt *time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *stubRunnerRepo) FetchAndLock(ctx context.Context, limit int) ([]*repository.OutboxEvent, error) {
|
func (r *stubRunnerRepo) FetchAndLock(ctx context.Context, limit int) ([]*repository.OutboxEvent, error) {
|
||||||
@@ -24,6 +29,9 @@ func (r *stubRunnerRepo) MarkCompleted(ctx context.Context, eventID string) erro
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (r *stubRunnerRepo) MarkFailed(ctx context.Context, eventID string, errorMsg string, nextRetryAt *time.Time) error {
|
func (r *stubRunnerRepo) MarkFailed(ctx context.Context, eventID string, errorMsg string, nextRetryAt *time.Time) error {
|
||||||
|
r.failedEventID = eventID
|
||||||
|
r.failedErrorMsg = errorMsg
|
||||||
|
r.failedNextRetryAt = nextRetryAt
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -56,3 +64,50 @@ func TestOutboxProcessorRunner_ProcessRejectsNilMessageBroker(t *testing.T) {
|
|||||||
t.Fatalf("expected error to mention message broker, got %v", err)
|
t.Fatalf("expected error to mention message broker, got %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type failingBroker struct {
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *failingBroker) Publish(ctx context.Context, event *repository.OutboxEvent) error {
|
||||||
|
return b.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOutboxProcessorRunner_HandleFailureUsesDomainBackoff(t *testing.T) {
|
||||||
|
payload := json.RawMessage(`{"event":"created"}`)
|
||||||
|
repo := &stubRunnerRepo{
|
||||||
|
events: []*repository.OutboxEvent{
|
||||||
|
{
|
||||||
|
ID: 1,
|
||||||
|
AggregateType: "account",
|
||||||
|
AggregateID: "acc-1",
|
||||||
|
EventType: "created",
|
||||||
|
EventID: "evt-1",
|
||||||
|
Payload: payload,
|
||||||
|
Status: repository.OutboxStatusProcessing,
|
||||||
|
RetryCount: 0,
|
||||||
|
MaxRetries: 5,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
runner := NewOutboxProcessorRunner(repo, &failingBroker{
|
||||||
|
err: errors.New("publish failed"),
|
||||||
|
}, &messaging.NoOpOutboxStats{})
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
if err := runner.process(context.Background()); err != nil {
|
||||||
|
t.Fatalf("expected runner to handle publish failure, got %v", err)
|
||||||
|
}
|
||||||
|
if repo.failedEventID != "evt-1" {
|
||||||
|
t.Fatalf("expected failed event evt-1, got %s", repo.failedEventID)
|
||||||
|
}
|
||||||
|
if repo.failedNextRetryAt == nil {
|
||||||
|
t.Fatal("expected failed retry timestamp to be recorded")
|
||||||
|
}
|
||||||
|
expectedBackoff := time.Duration(domain.CalculateOutboxBackoff(1, 5)) * time.Second
|
||||||
|
actualBackoff := repo.failedNextRetryAt.Sub(start)
|
||||||
|
if actualBackoff < expectedBackoff-time.Second || actualBackoff > expectedBackoff+time.Second {
|
||||||
|
t.Fatalf("expected retry backoff around %s, got %s", expectedBackoff, actualBackoff)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user