diff --git a/go.mod b/go.mod index 631923c..b2698ec 100644 --- a/go.mod +++ b/go.mod @@ -9,16 +9,21 @@ require ( ) require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/netbox-community/go-netbox/v4 v4.3.0 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/redis/go-redis/v9 v9.18.0 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/subosito/gotenv v1.6.0 // indirect + go.uber.org/atomic v1.11.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.28.0 // indirect diff --git a/go.sum b/go.sum index 798f566..dd066ba 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= @@ -10,6 +14,8 @@ github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9L github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -25,6 +31,8 @@ github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs= +github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= @@ -43,6 +51,8 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= diff --git a/internal/queue/waq.go b/internal/queue/waq.go new file mode 100644 index 0000000..9eeadbe --- /dev/null +++ b/internal/queue/waq.go @@ -0,0 +1,154 @@ +package queue + +import ( + "context" + "encoding/json" + "fmt" + "log" + "regexp" + "strconv" + "time" + + "github.com/google/uuid" + "github.com/redis/go-redis/v9" +) + +const queueKey = "hwlab:netbox:pending_ops" + +// PendingOp represents a queued NetBox operation. +// Serialized as JSON in DragonFlyDB LIST. +type PendingOp struct { + ID string `json:"id"` // UUID + Type string `json:"type"` // "create_device", "patch_custom_fields", "sync_tags", etc. + Payload json.RawMessage `json:"payload"` // operation-specific data + CreatedAt time.Time `json:"created_at"` + Attempts int `json:"attempts"` // retry count +} + +// NewPendingOp creates a new PendingOp with a generated ID and current timestamp. +func NewPendingOp(opType string, payload interface{}) (PendingOp, error) { + data, err := json.Marshal(payload) + if err != nil { + return PendingOp{}, fmt.Errorf("marshal payload: %w", err) + } + return PendingOp{ + ID: uuid.New().String(), + Type: opType, + Payload: json.RawMessage(data), + CreatedAt: time.Now().UTC(), + Attempts: 0, + }, nil +} + +// WAQ is a write-ahead queue backed by DragonFlyDB (Redis-compatible). +// Operations are stored as JSON in a Redis LIST using RPUSH/BLPOP for FIFO ordering. +type WAQ struct { + rdb *redis.Client +} + +// redisURLRegexp matches redis://[:password@]host[:port][/db] +// The password group is non-greedy up to the last '@' before host:port. +// This handles passwords containing forward slashes, which url.Parse cannot. +var redisURLRegexp = regexp.MustCompile(`^rediss?://:?([^@]*)@([^:/]+)(?::(\d+))?(?:/(\d+))?$`) + +// parseRedisURL parses a redis:// URL into redis.Options, correctly handling +// passwords that contain forward slashes (which break standard url.Parse). +func parseRedisURL(rawURL string) (*redis.Options, error) { + // Try standard ParseURL first (works when password has no special chars). + if opt, err := redis.ParseURL(rawURL); err == nil { + return opt, nil + } + + // Fall back to manual regex parsing for passwords with slashes. + m := redisURLRegexp.FindStringSubmatch(rawURL) + if m == nil { + return nil, fmt.Errorf("invalid redis URL format: %s", rawURL) + } + password := m[1] + host := m[2] + portStr := m[3] + dbStr := m[4] + + port := 6379 + if portStr != "" { + p, err := strconv.Atoi(portStr) + if err != nil { + return nil, fmt.Errorf("invalid port in redis URL: %s", portStr) + } + port = p + } + + db := 0 + if dbStr != "" { + d, err := strconv.Atoi(dbStr) + if err != nil { + return nil, fmt.Errorf("invalid db in redis URL: %s", dbStr) + } + db = d + } + + return &redis.Options{ + Addr: fmt.Sprintf("%s:%d", host, port), + Password: password, + DB: db, + }, nil +} + +// NewWAQ creates a new WAQ connected to the given Redis/DragonFlyDB URL. +// Returns error if the URL is invalid or the server is unreachable. +// Passwords containing forward slashes (or other special characters) are supported. +func NewWAQ(redisURL string) (*WAQ, error) { + opt, err := parseRedisURL(redisURL) + if err != nil { + return nil, fmt.Errorf("parse dragonfly url: %w", err) + } + client := redis.NewClient(opt) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := client.Ping(ctx).Err(); err != nil { + client.Close() + return nil, fmt.Errorf("dragonfly unreachable at %s: %w", redisURL, err) + } + log.Printf("WAQ connected to DragonFlyDB") + return &WAQ{rdb: client}, nil +} + +// Enqueue pushes a PendingOp onto the right end of the queue (FIFO). +func (q *WAQ) Enqueue(ctx context.Context, op PendingOp) error { + data, err := json.Marshal(op) + if err != nil { + return fmt.Errorf("marshal op: %w", err) + } + return q.rdb.RPush(ctx, queueKey, data).Err() +} + +// Dequeue pops the oldest operation (blocking for up to timeout). +// Returns nil, nil if timeout elapses with no item. +func (q *WAQ) Dequeue(ctx context.Context, timeout time.Duration) (*PendingOp, error) { + result, err := q.rdb.BLPop(ctx, timeout, queueKey).Result() + if err == redis.Nil { + return nil, nil // timeout — no items + } + if err != nil { + return nil, fmt.Errorf("blpop: %w", err) + } + var op PendingOp + if err := json.Unmarshal([]byte(result[1]), &op); err != nil { + return nil, fmt.Errorf("unmarshal op: %w", err) + } + return &op, nil +} + +// Len returns the current number of pending operations in the queue. +func (q *WAQ) Len(ctx context.Context) (int64, error) { + n, err := q.rdb.LLen(ctx, queueKey).Result() + if err != nil { + return 0, fmt.Errorf("llen: %w", err) + } + return n, nil +} + +// Close releases the DragonFlyDB connection. +func (q *WAQ) Close() error { + return q.rdb.Close() +} diff --git a/internal/queue/waq_test.go b/internal/queue/waq_test.go new file mode 100644 index 0000000..f9f8e75 --- /dev/null +++ b/internal/queue/waq_test.go @@ -0,0 +1,85 @@ +package queue_test + +import ( + "context" + "encoding/json" + "os" + "testing" + "time" + + "git.georgsen.dk/hwlab/internal/queue" +) + +func TestPendingOpJSON(t *testing.T) { + payload := map[string]string{"device_id": "42", "hw_id": "HW-00001"} + op, err := queue.NewPendingOp("create_device", payload) + if err != nil { + t.Fatalf("NewPendingOp: %v", err) + } + if op.ID == "" { + t.Error("ID should be a UUID") + } + if op.Type != "create_device" { + t.Errorf("Type: want create_device, got %s", op.Type) + } + // Round-trip JSON + data, _ := json.Marshal(op) + var op2 queue.PendingOp + if err := json.Unmarshal(data, &op2); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if op2.ID != op.ID { + t.Errorf("ID mismatch after round-trip: %s != %s", op2.ID, op.ID) + } +} + +func TestNewWAQInvalidURL(t *testing.T) { + _, err := queue.NewWAQ("not-a-redis-url") + if err == nil { + t.Error("expected error for invalid URL") + } +} + +// dragonflyURL returns the DragonFlyDB URL from env, or the default from .env. +func dragonflyURL(t *testing.T) string { + t.Helper() + url := os.Getenv("HWLAB_DRAGONFLY_URL") + if url == "" { + url = "redis://:nUq/IfoIQJf/kouckKHRQOk7vV0NwCuI@10.5.0.10:6379" + } + return url +} + +func TestWAQEnqueueDequeue(t *testing.T) { + waq, err := queue.NewWAQ(dragonflyURL(t)) + if err != nil { + t.Skipf("DragonFlyDB unavailable: %v", err) + } + defer waq.Close() + + ctx := context.Background() + + op, _ := queue.NewPendingOp("test_op", map[string]string{"test": "value"}) + if err := waq.Enqueue(ctx, op); err != nil { + t.Fatalf("Enqueue: %v", err) + } + + n, err := waq.Len(ctx) + if err != nil { + t.Fatalf("Len: %v", err) + } + if n < 1 { + t.Error("expected at least 1 item after enqueue") + } + + got, err := waq.Dequeue(ctx, 2*time.Second) + if err != nil { + t.Fatalf("Dequeue: %v", err) + } + if got == nil { + t.Fatal("expected op from dequeue, got nil") + } + if got.ID != op.ID { + t.Errorf("dequeued op ID mismatch: want %s, got %s", op.ID, got.ID) + } +}