From b252ba1c30de753cfb3f6f722eeda96d6138c91e Mon Sep 17 00:00:00 2001
From: Shekhar Vaidya <shekhar.vaidya2@hotmail.com>
Date: Mon, 13 Jul 2026 12:37:01 +0530
Subject: [PATCH] feat(backend): approximate distinct-source count for Sentinel
 rejections

CLI telemetry showed a growing count of Sentinel rejections but no way
to tell whether they come from one user retrying repeatedly or many
distinct users independently getting blocked.

Adds a per-day Redis HyperLogLog (msdl:telemetry:sentinel_hll:<date>)
seeded from the reporting IP whenever a CLI error contains "Sentinel",
plus a parallel daily counter. /metrics now exposes sentinel_today:
{date, errors_today, distinct_sources_est}. No raw IPs are stored --
HLL is a one-way cardinality estimator, and both keys expire after 48h.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 backend/main.go | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/backend/main.go b/backend/main.go
index 3fa9785..e4a7b7c 100644
--- a/backend/main.go
+++ b/backend/main.go
@@ -1675,6 +1675,20 @@ func handleTelemetry(w http.ResponseWriter, r *http.Request) {
 		rdb.HIncrBy(ctx, "msdl:telemetry:results", result, 1)
 		if !p.Success && p.Error != "" && len(p.Error) <= 200 {
 			rdb.HIncrBy(ctx, "msdl:telemetry:errors", p.Error, 1)
+			if strings.Contains(p.Error, "Sentinel") {
+				// Track today's Sentinel-rejection count and an approximate
+				// distinct-source count (via HyperLogLog on the reporting
+				// IP) so /metrics can show whether rejections are
+				// concentrated in a few sources or spread across many,
+				// without ever storing raw IPs.
+				day := time.Now().UTC().Format("2006-01-02")
+				countKey := "msdl:telemetry:sentinel_count:" + day
+				hllKey := "msdl:telemetry:sentinel_hll:" + day
+				rdb.Incr(ctx, countKey)
+				rdb.Expire(ctx, countKey, 48*time.Hour)
+				rdb.PFAdd(ctx, hllKey, ip)
+				rdb.Expire(ctx, hllKey, 48*time.Hour)
+			}
 		}
 	}
 
@@ -1775,6 +1789,7 @@ func handleMetrics(w http.ResponseWriter, r *http.Request) {
 	evalCacheMu.RUnlock()
 
 	telemetry := loadTelemetryFromRedis()
+	sentinelToday := loadSentinelTodayFromRedis()
 
 	w.Header().Set("Content-Type", "application/json")
 	json.NewEncoder(w).Encode(map[string]interface{}{
@@ -1805,9 +1820,29 @@ func handleMetrics(w http.ResponseWriter, r *http.Request) {
 		"neg_cache_size":   negSize,
 		"total_ms_fetches": skuFetches + linkFetches,
 		"telemetry":        telemetry,
+		"sentinel_today":   sentinelToday,
 	})
 }
 
+// loadSentinelTodayFromRedis returns today's Sentinel-rejection count and an
+// approximate distinct-source count (from the HyperLogLog seeded in
+// handleTelemetry). Returns nil when Redis is unavailable.
+func loadSentinelTodayFromRedis() map[string]interface{} {
+	if rdb == nil {
+		return nil
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
+	defer cancel()
+	day := time.Now().UTC().Format("2006-01-02")
+	count, _ := rdb.Get(ctx, "msdl:telemetry:sentinel_count:"+day).Int64()
+	distinct, _ := rdb.PFCount(ctx, "msdl:telemetry:sentinel_hll:"+day).Result()
+	return map[string]interface{}{
+		"date":                 day,
+		"errors_today":         count,
+		"distinct_sources_est": distinct,
+	}
+}
+
 func warmEvalCache() {
 	for slug, product := range evalProductMap {
 		go func(s string, p EvalProduct) {