From b252ba1c30de753cfb3f6f722eeda96d6138c91e Mon Sep 17 00:00:00 2001 From: Shekhar Vaidya Date: Mon, 13 Jul 2026 12:37:01 +0530 Subject: [PATCH 1/2] feat(backend): approximate distinct-source count for Sentinel rejections CLI telemetry showed a growing count of Sentinel rejections but no way to tell whether they come from one user retrying repeatedly or many distinct users independently getting blocked. Adds a per-day Redis HyperLogLog (msdl:telemetry:sentinel_hll:) seeded from the reporting IP whenever a CLI error contains "Sentinel", plus a parallel daily counter. /metrics now exposes sentinel_today: {date, errors_today, distinct_sources_est}. No raw IPs are stored -- HLL is a one-way cardinality estimator, and both keys expire after 48h. Co-Authored-By: Claude Sonnet 4.6 --- backend/main.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/backend/main.go b/backend/main.go index 3fa9785..e4a7b7c 100644 --- a/backend/main.go +++ b/backend/main.go @@ -1675,6 +1675,20 @@ func handleTelemetry(w http.ResponseWriter, r *http.Request) { rdb.HIncrBy(ctx, "msdl:telemetry:results", result, 1) if !p.Success && p.Error != "" && len(p.Error) <= 200 { rdb.HIncrBy(ctx, "msdl:telemetry:errors", p.Error, 1) + if strings.Contains(p.Error, "Sentinel") { + // Track today's Sentinel-rejection count and an approximate + // distinct-source count (via HyperLogLog on the reporting + // IP) so /metrics can show whether rejections are + // concentrated in a few sources or spread across many, + // without ever storing raw IPs. + day := time.Now().UTC().Format("2006-01-02") + countKey := "msdl:telemetry:sentinel_count:" + day + hllKey := "msdl:telemetry:sentinel_hll:" + day + rdb.Incr(ctx, countKey) + rdb.Expire(ctx, countKey, 48*time.Hour) + rdb.PFAdd(ctx, hllKey, ip) + rdb.Expire(ctx, hllKey, 48*time.Hour) + } } } @@ -1775,6 +1789,7 @@ func handleMetrics(w http.ResponseWriter, r *http.Request) { evalCacheMu.RUnlock() telemetry := loadTelemetryFromRedis() + sentinelToday := loadSentinelTodayFromRedis() w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(map[string]interface{}{ @@ -1805,9 +1820,29 @@ func handleMetrics(w http.ResponseWriter, r *http.Request) { "neg_cache_size": negSize, "total_ms_fetches": skuFetches + linkFetches, "telemetry": telemetry, + "sentinel_today": sentinelToday, }) } +// loadSentinelTodayFromRedis returns today's Sentinel-rejection count and an +// approximate distinct-source count (from the HyperLogLog seeded in +// handleTelemetry). Returns nil when Redis is unavailable. +func loadSentinelTodayFromRedis() map[string]interface{} { + if rdb == nil { + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + day := time.Now().UTC().Format("2006-01-02") + count, _ := rdb.Get(ctx, "msdl:telemetry:sentinel_count:"+day).Int64() + distinct, _ := rdb.PFCount(ctx, "msdl:telemetry:sentinel_hll:"+day).Result() + return map[string]interface{}{ + "date": day, + "errors_today": count, + "distinct_sources_est": distinct, + } +} + func warmEvalCache() { for slug, product := range evalProductMap { go func(s string, p EvalProduct) { -- 2.45.3 From 327c5e195dfc9d77ee89e1fee0ad5b8812e92588 Mon Sep 17 00:00:00 2001 From: Shekhar Vaidya Date: Mon, 13 Jul 2026 12:43:31 +0530 Subject: [PATCH 2/2] simplify(backend): drop date-scoping for Sentinel stats, use flat all-time fields Matches the shape actually asked for and how every other telemetry counter in this file already works (all-time cumulative via HIncrBy, no expiry, no per-day key rotation): /metrics now exposes flat sentinel_errors and sentinel_distinct_sources_est fields instead of a nested, date-scoped sentinel_today object. Co-Authored-By: Claude Sonnet 4.6 --- backend/main.go | 44 +++++++++++++++++--------------------------- 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/backend/main.go b/backend/main.go index e4a7b7c..9df52bb 100644 --- a/backend/main.go +++ b/backend/main.go @@ -937,7 +937,6 @@ func fetchDownloadLinksFromMS(productID, skuID string) ([]byte, error) { jar = &simpleCookieJar{} } - proxyQ := url.Values{} proxyQ.Set("profile", PROFILE) proxyQ.Set("productEditionId", "undefined") @@ -1676,18 +1675,13 @@ func handleTelemetry(w http.ResponseWriter, r *http.Request) { if !p.Success && p.Error != "" && len(p.Error) <= 200 { rdb.HIncrBy(ctx, "msdl:telemetry:errors", p.Error, 1) if strings.Contains(p.Error, "Sentinel") { - // Track today's Sentinel-rejection count and an approximate + // All-time Sentinel-rejection count, plus an approximate // distinct-source count (via HyperLogLog on the reporting // IP) so /metrics can show whether rejections are // concentrated in a few sources or spread across many, // without ever storing raw IPs. - day := time.Now().UTC().Format("2006-01-02") - countKey := "msdl:telemetry:sentinel_count:" + day - hllKey := "msdl:telemetry:sentinel_hll:" + day - rdb.Incr(ctx, countKey) - rdb.Expire(ctx, countKey, 48*time.Hour) - rdb.PFAdd(ctx, hllKey, ip) - rdb.Expire(ctx, hllKey, 48*time.Hour) + rdb.Incr(ctx, "msdl:telemetry:sentinel_count") + rdb.PFAdd(ctx, "msdl:telemetry:sentinel_hll", ip) } } } @@ -1789,7 +1783,7 @@ func handleMetrics(w http.ResponseWriter, r *http.Request) { evalCacheMu.RUnlock() telemetry := loadTelemetryFromRedis() - sentinelToday := loadSentinelTodayFromRedis() + sentinelErrors, sentinelDistinctSources := loadSentinelStatsFromRedis() w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(map[string]interface{}{ @@ -1817,30 +1811,26 @@ func handleMetrics(w http.ResponseWriter, r *http.Request) { "hit_rate": hitRate(evalHits, evalReqs), "cache_size": evalSize, }, - "neg_cache_size": negSize, - "total_ms_fetches": skuFetches + linkFetches, - "telemetry": telemetry, - "sentinel_today": sentinelToday, + "neg_cache_size": negSize, + "total_ms_fetches": skuFetches + linkFetches, + "telemetry": telemetry, + "sentinel_errors": sentinelErrors, + "sentinel_distinct_sources_est": sentinelDistinctSources, }) } -// loadSentinelTodayFromRedis returns today's Sentinel-rejection count and an -// approximate distinct-source count (from the HyperLogLog seeded in -// handleTelemetry). Returns nil when Redis is unavailable. -func loadSentinelTodayFromRedis() map[string]interface{} { +// loadSentinelStatsFromRedis returns the all-time Sentinel-rejection count +// and an approximate distinct-source count (from the HyperLogLog seeded in +// handleTelemetry). Returns (0, 0) when Redis is unavailable. +func loadSentinelStatsFromRedis() (int64, int64) { if rdb == nil { - return nil + return 0, 0 } ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) defer cancel() - day := time.Now().UTC().Format("2006-01-02") - count, _ := rdb.Get(ctx, "msdl:telemetry:sentinel_count:"+day).Int64() - distinct, _ := rdb.PFCount(ctx, "msdl:telemetry:sentinel_hll:"+day).Result() - return map[string]interface{}{ - "date": day, - "errors_today": count, - "distinct_sources_est": distinct, - } + count, _ := rdb.Get(ctx, "msdl:telemetry:sentinel_count").Int64() + distinct, _ := rdb.PFCount(ctx, "msdl:telemetry:sentinel_hll").Result() + return count, distinct } func warmEvalCache() { -- 2.45.3