feat: Phase 3 crowdsourced cache contribution

- Redis L2 persistent cache: initRedis() from REDIS_URL env, seed
  in-memory linkCache on startup, write-through on every fresh fetch
  and background refresh (silent fallback to memory-only if unavailable)
- POST /contribute endpoint: validates product catalog, expiry > 1h,
  Microsoft CDN host allow-list, per-IP token bucket (~5/min),
  CONTRIBUTE_SECRET header; writes to in-memory + Redis; logs all
  accepts and rejections
- CLI --no-contribute flag (also MSDL_NO_CONTRIBUTE=1 env var):
  background goroutine posts raw MS JSON to /contribute after every
  consumer fetch; best-effort 5s timeout; opt-out message to stderr

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

backend/go.mod

@@ -3,6 +3,9 @@ module msdl-backend-go
 go 1.25.0
 
 require (
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/redis/go-redis/v9 v9.20.1 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
 	golang.org/x/sync v0.20.0 // indirect
 )

backend/go.sum

@@ -1,4 +1,10 @@
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/redis/go-redis/v9 v9.20.1 h1:sfCU6A8P3dXbKyWes02uxA2baehGux9dZHfEKtsTB1w=
+github.com/redis/go-redis/v9 v9.20.1/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA=
+go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
+go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
 golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=

backend/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"html"
@@ -20,6 +21,7 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/redis/go-redis/v9"
 	"golang.org/x/sync/singleflight"
 )
 
@@ -131,6 +133,10 @@ var (
 
 var sfGroup singleflight.Group
 
+// --- Redis L2 cache client ---
+
+var rdb *redis.Client
+
 // --- Metrics counters (atomic — no mutex needed) ---
 
 var (
@@ -285,6 +291,198 @@ type rateLimitError struct{ message string }
 
 func (e *rateLimitError) Error() string { return e.message }
 
+// --- Redis L2 cache helpers ---
+
+func redisKey(productID, skuID string) string {
+	return "msdl:link:" + productID + ":" + skuID
+}
+
+func initRedis() {
+	redisURL := os.Getenv("REDIS_URL")
+	if redisURL == "" {
+		log.Println("redis: REDIS_URL not set — memory-only mode")
+		return
+	}
+	opts, err := redis.ParseURL(redisURL)
+	if err != nil {
+		log.Printf("redis: invalid REDIS_URL: %v — memory-only mode\n", err)
+		return
+	}
+	client := redis.NewClient(opts)
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+	if err := client.Ping(ctx).Err(); err != nil {
+		log.Printf("redis: ping failed: %v — memory-only mode\n", err)
+		return
+	}
+	rdb = client
+	log.Println("redis: connected")
+}
+
+func redisWriteLink(productID, skuID string, raw []byte, expiresAt time.Time) {
+	if rdb == nil {
+		return
+	}
+	ttl := time.Until(expiresAt)
+	if ttl <= 0 {
+		return
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	if err := rdb.Set(ctx, redisKey(productID, skuID), raw, ttl).Err(); err != nil {
+		log.Printf("redis: write failed for %s:%s: %v\n", productID, skuID, err)
+	}
+}
+
+func redisSeedLinkCache() {
+	if rdb == nil {
+		return
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	var cursor uint64
+	var seeded int
+	for {
+		keys, nextCursor, err := rdb.Scan(ctx, cursor, "msdl:link:*", 100).Result()
+		if err != nil {
+			log.Printf("redis: seed scan error: %v\n", err)
+			return
+		}
+		for _, key := range keys {
+			raw, err := rdb.Get(ctx, key).Bytes()
+			if err != nil {
+				continue
+			}
+			ttl, err := rdb.TTL(ctx, key).Result()
+			if err != nil || ttl <= 0 {
+				continue
+			}
+			// Key format: "msdl:link:<productID>:<skuID>"
+			parts := strings.SplitN(strings.TrimPrefix(key, "msdl:link:"), ":", 2)
+			if len(parts) != 2 {
+				continue
+			}
+			cacheKey := parts[0] + ":" + parts[1]
+			expiresAt := time.Now().Add(ttl)
+			linkCacheMu.Lock()
+			linkCache[cacheKey] = linkCacheEntry{RawJSON: raw, ExpiresAt: expiresAt, FetchedAt: time.Now()}
+			linkCacheMu.Unlock()
+			seeded++
+		}
+		cursor = nextCursor
+		if cursor == 0 {
+			break
+		}
+	}
+	if seeded > 0 {
+		log.Printf("redis: seeded %d link cache entries from Redis\n", seeded)
+	}
+}
+
+// --- /contribute validation helpers ---
+
+// validContributeProducts is the canonical product ID allow-list for POST /contribute.
+var validContributeProducts = map[string]bool{
+	"52": true, "2378": true, "2618": true,
+	"3113": true, "3114": true, "3115": true,
+	"3131": true, "3132": true, "3133": true,
+	"3262": true, "3263": true, "3264": true,
+	"3265": true, "3266": true, "3267": true,
+	"3321": true, "3324": true,
+}
+
+// allowedCDNSuffixes is the CDN host allow-list for contributed download URLs.
+// Worst-case abuse: attacker submits a valid Microsoft-signed link — harmless.
+var allowedCDNSuffixes = []string{
+	".download.prss.microsoft.com",
+	".dl.delivery.mp.microsoft.com",
+	".delivery.mp.microsoft.com",
+}
+
+func isAllowedCDNHost(rawURL string) bool {
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		return false
+	}
+	host := strings.ToLower(u.Hostname())
+	for _, suffix := range allowedCDNSuffixes {
+		if strings.HasSuffix(host, suffix) {
+			return true
+		}
+	}
+	return false
+}
+
+// --- Per-IP rate limiter (token bucket, used by /contribute) ---
+
+type tokenBucket struct {
+	tokens     float64
+	maxTokens  float64
+	refillRate float64 // tokens per second
+	lastRefill time.Time
+}
+
+func (b *tokenBucket) allow() bool {
+	now := time.Now()
+	elapsed := now.Sub(b.lastRefill).Seconds()
+	b.tokens += elapsed * b.refillRate
+	if b.tokens > b.maxTokens {
+		b.tokens = b.maxTokens
+	}
+	b.lastRefill = now
+	if b.tokens >= 1 {
+		b.tokens--
+		return true
+	}
+	return false
+}
+
+type ipRateLimiter struct {
+	mu      sync.Mutex
+	buckets map[string]*tokenBucket
+	rate    float64
+	burst   float64
+}
+
+func newIPRateLimiter(ratePerMin, burst float64) *ipRateLimiter {
+	rl := &ipRateLimiter{
+		buckets: make(map[string]*tokenBucket),
+		rate:    ratePerMin / 60.0,
+		burst:   burst,
+	}
+	go func() {
+		ticker := time.NewTicker(10 * time.Minute)
+		for range ticker.C {
+			rl.mu.Lock()
+			for ip, b := range rl.buckets {
+				if b.tokens >= b.maxTokens {
+					delete(rl.buckets, ip)
+				}
+			}
+			rl.mu.Unlock()
+		}
+	}()
+	return rl
+}
+
+func (rl *ipRateLimiter) allow(ip string) bool {
+	rl.mu.Lock()
+	defer rl.mu.Unlock()
+	b, ok := rl.buckets[ip]
+	if !ok {
+		b = &tokenBucket{
+			tokens:     rl.burst,
+			maxTokens:  rl.burst,
+			refillRate: rl.rate,
+			lastRefill: time.Now(),
+		}
+		rl.buckets[ip] = b
+	}
+	return b.allow()
+}
+
+var contributeRL = newIPRateLimiter(5, 5) // 5 requests/min, burst of 5
+
 // --- Evalcenter helpers ---
 
 func detectArch(rawURL string) string {
@@ -745,7 +943,7 @@ func enableCORS(next http.Handler) http.Handler {
 		if origin == "https://msdl.tech-latest.com" || strings.HasPrefix(origin, "http://localhost:") {
 			w.Header().Set("Access-Control-Allow-Origin", origin)
 		}
-		w.Header().Set("Access-Control-Allow-Methods", "GET, OPTIONS")
+		w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
 		w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
 		w.Header().Set("Access-Control-Expose-Headers", "X-MSDL-Link-Status, X-MSDL-Link-Expires")
 		if r.Method == "OPTIONS" {
@@ -888,6 +1086,29 @@ func handleProxy(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// 1b. Redis L2 cache — check after in-memory miss, before calling Microsoft
+	if !forceRefresh && !hasCached && rdb != nil {
+		rctx, rcancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
+		redisRaw, redisErr := rdb.Get(rctx, redisKey(productID, skuID)).Bytes()
+		rcancel()
+		if redisErr == nil && len(redisRaw) > 0 {
+			exp := parseLinkExpiry(redisRaw)
+			if time.Now().Before(exp) {
+				linkCacheMu.Lock()
+				linkCache[cacheKey] = linkCacheEntry{RawJSON: redisRaw, ExpiresAt: exp, FetchedAt: time.Now()}
+				linkCacheMu.Unlock()
+				atomic.AddInt64(&mLinkCacheHits, 1)
+				log.Printf("/proxy: product_id=%s sku_id=%s -> redis L2 hit\n", productID, skuID)
+				w.Header().Set("X-MSDL-Link-Status", "cached")
+				if rawExp := extractRawExpiry(redisRaw); rawExp != "" {
+					w.Header().Set("X-MSDL-Link-Expires", rawExp)
+				}
+				w.Write(redisRaw)
+				return
+			}
+		}
+	}
+
 	// 2. Check negative cache — but only if no stale data and not a force refresh
 	if !forceRefresh && !hasCached {
 		negCacheMu.RLock()
@@ -929,6 +1150,7 @@ func handleProxy(w http.ResponseWriter, r *http.Request) {
 					linkCacheMu.Lock()
 					linkCache[cacheKey] = linkCacheEntry{RawJSON: raw, ExpiresAt: exp, FetchedAt: time.Now()}
 					linkCacheMu.Unlock()
+					redisWriteLink(productID, skuID, raw, exp)
 					log.Printf("/proxy: background refresh succeeded for %s:%s, cached until %s\n", productID, skuID, exp.Format(time.RFC3339))
 					return nil, nil
 				})
@@ -969,6 +1191,7 @@ func handleProxy(w http.ResponseWriter, r *http.Request) {
 		FetchedAt: time.Now(),
 	}
 	linkCacheMu.Unlock()
+	redisWriteLink(productID, skuID, raw, expiresAt)
 
 	atomic.AddInt64(&mLinkFetches, 1)
 	if forceRefresh {
@@ -983,6 +1206,118 @@ func handleProxy(w http.ResponseWriter, r *http.Request) {
 	w.Write(raw)
 }
 
+// --- /contribute endpoint ---
+
+func handleContribute(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+
+	// Check CONTRIBUTE_SECRET header
+	if secret := os.Getenv("CONTRIBUTE_SECRET"); secret != "" {
+		if r.Header.Get("X-Contribute-Secret") != secret {
+			respondJSONError(w, http.StatusUnauthorized, "unauthorized")
+			return
+		}
+	}
+
+	// Per-IP rate limit (~5/min)
+	ip := r.RemoteAddr
+	if idx := strings.LastIndex(ip, ":"); idx != -1 {
+		ip = ip[:idx]
+	}
+	if fwd := r.Header.Get("X-Forwarded-For"); fwd != "" {
+		ip = strings.TrimSpace(strings.SplitN(fwd, ",", 2)[0])
+	}
+	if !contributeRL.allow(ip) {
+		respondJSONError(w, http.StatusTooManyRequests, "rate limit exceeded")
+		return
+	}
+
+	// Parse body
+	body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) // 1 MB cap
+	if err != nil {
+		respondJSONError(w, http.StatusBadRequest, "failed to read body")
+		return
+	}
+	var req struct {
+		ProductID string          `json:"product_id"`
+		SkuID     string          `json:"sku_id"`
+		RawJSON   json.RawMessage `json:"raw_json"`
+	}
+	if err := json.Unmarshal(body, &req); err != nil {
+		respondJSONError(w, http.StatusBadRequest, "invalid JSON")
+		return
+	}
+	productID := req.ProductID
+	skuID := req.SkuID
+
+	// Validation 1: product in catalog
+	if !validContributeProducts[productID] {
+		log.Printf("contribute: product_id=%s sku_id=%s -> rejected (unknown product)\n", productID, skuID)
+		respondJSONError(w, http.StatusUnprocessableEntity, "unknown product_id")
+		return
+	}
+
+	// Validation 2: sku_id format
+	if !validSkuID.MatchString(skuID) {
+		log.Printf("contribute: product_id=%s sku_id=%s -> rejected (invalid sku_id format)\n", productID, skuID)
+		respondJSONError(w, http.StatusUnprocessableEntity, "invalid sku_id")
+		return
+	}
+
+	raw := []byte(req.RawJSON)
+	if len(raw) == 0 {
+		respondJSONError(w, http.StatusBadRequest, "raw_json is required")
+		return
+	}
+
+	// Validation 3: expiry must be > 1 hour out
+	expiresAt := parseLinkExpiry(raw)
+	if time.Until(expiresAt) < time.Hour {
+		log.Printf("contribute: product_id=%s sku_id=%s -> rejected (expiry < 1h)\n", productID, skuID)
+		respondJSONError(w, http.StatusUnprocessableEntity, "link expires in less than 1 hour")
+		return
+	}
+
+	// Validation 4: all download URLs must be from allowed Microsoft CDN hosts
+	var data map[string]interface{}
+	if err := json.Unmarshal(raw, &data); err != nil {
+		respondJSONError(w, http.StatusUnprocessableEntity, "raw_json is not valid JSON")
+		return
+	}
+	opts, _ := data["ProductDownloadOptions"].([]interface{})
+	if len(opts) == 0 {
+		log.Printf("contribute: product_id=%s sku_id=%s -> rejected (no ProductDownloadOptions)\n", productID, skuID)
+		respondJSONError(w, http.StatusUnprocessableEntity, "raw_json has no ProductDownloadOptions")
+		return
+	}
+	for _, opt := range opts {
+		m, ok := opt.(map[string]interface{})
+		if !ok {
+			continue
+		}
+		uri, _ := m["Uri"].(string)
+		if uri == "" || !isAllowedCDNHost(uri) {
+			log.Printf("contribute: product_id=%s sku_id=%s -> rejected (disallowed host: %s)\n", productID, skuID, uri)
+			respondJSONError(w, http.StatusUnprocessableEntity, "download URL host not in allow-list")
+			return
+		}
+	}
+
+	// Write to in-memory cache + Redis
+	cacheKey := productID + ":" + skuID
+	linkCacheMu.Lock()
+	linkCache[cacheKey] = linkCacheEntry{RawJSON: raw, ExpiresAt: expiresAt, FetchedAt: time.Now()}
+	linkCacheMu.Unlock()
+	redisWriteLink(productID, skuID, raw, expiresAt)
+
+	log.Printf("contribute: product_id=%s sku_id=%s -> accepted, cached until %s\n", productID, skuID, expiresAt.Format(time.RFC3339))
+	w.WriteHeader(http.StatusNoContent)
+}
+
 // --- /evallinks endpoint ---
 
 func handleEvalLinks(w http.ResponseWriter, r *http.Request) {
@@ -1185,6 +1520,8 @@ func warmEvalCache() {
 }
 
 func main() {
+	initRedis()
+	go redisSeedLinkCache()
 	go cleanupSessions()
 	go cleanupCaches()
 	go warmEvalCache()
@@ -1193,6 +1530,7 @@ func main() {
 
 	mux.HandleFunc("/skuinfo", handleSkuInfo)
 	mux.HandleFunc("/proxy", handleProxy)
+	mux.HandleFunc("/contribute", handleContribute)
 	mux.HandleFunc("/evallinks", handleEvalLinks)
 	mux.HandleFunc("/metrics", handleMetrics)
 	mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {

cli/main.go

@@ -1,12 +1,49 @@
 package main
 
 import (
+	"bytes"
+	"context"
+	"encoding/json"
 	"flag"
 	"fmt"
+	"net/http"
 	"os"
 	"strings"
+	"sync"
+	"time"
 )
 
+// contributeURL is the MSDL backend endpoint that warms the shared link cache.
+// Override with MSDL_API_URL env var, e.g. export MSDL_API_URL=http://localhost:3002
+const contributeSecret = "msdl-contribute-2026"
+
+func contributeURL() string {
+	if u := os.Getenv("MSDL_API_URL"); u != "" {
+		return strings.TrimRight(u, "/") + "/contribute"
+	}
+	return "https://api.msdl.tech-latest.com/contribute"
+}
+
+func postContribute(productID, skuID string, rawJSON []byte) {
+	payload := struct {
+		ProductID string          `json:"product_id"`
+		SkuID     string          `json:"sku_id"`
+		RawJSON   json.RawMessage `json:"raw_json"`
+	}{ProductID: productID, SkuID: skuID, RawJSON: rawJSON}
+	body, _ := json.Marshal(payload)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	req, _ := http.NewRequestWithContext(ctx, http.MethodPost, contributeURL(), bytes.NewReader(body))
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("X-Contribute-Secret", contributeSecret)
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return
+	}
+	resp.Body.Close()
+}
+
 func main() {
 	if err := run(os.Args[1:]); err != nil {
 		fmt.Fprintln(os.Stderr, "error:", err)
@@ -35,11 +72,13 @@ Flags:`)
 	langFlag := fs.String("lang", "", `language name, e.g. "English (United States)"`)
 	evalMode := fs.Bool("eval", false, "fetch evaluation ISOs")
 	listMode := fs.Bool("list", false, "list all products and exit")
+	noContributeFlag := fs.Bool("no-contribute", false, "skip sharing the link with the msdl.tech cache")
 
 	if err := fs.Parse(args); err != nil {
 		return err
 	}
 	query := strings.Join(fs.Args(), " ")
+	noContribute := *noContributeFlag || os.Getenv("MSDL_NO_CONTRIBUTE") == "1"
 
 	if *listMode {
 		fmt.Fprintln(os.Stderr, "Consumer products:")
@@ -56,10 +95,10 @@ Flags:`)
 	if *evalMode {
 		return runEval(query)
 	}
-	return runConsumer(*productID, query, *langFlag)
+	return runConsumer(*productID, query, *langFlag, noContribute)
 }
 
-func runConsumer(productID, query, langName string) error {
+func runConsumer(productID, query, langName string, noContribute bool) error {
 	var product Product
 
 	if productID != "" {
@@ -115,14 +154,35 @@ func runConsumer(productID, query, langName string) error {
 	}
 
 	fmt.Fprintf(os.Stderr, "Fetching download link...\n")
-	links, err := fetchDownloadLinks(client, sessionID, product.ID, lang.ID)
+	links, rawJSON, err := fetchDownloadLinks(client, sessionID, product.ID, lang.ID)
 	if err != nil {
 		return fmt.Errorf("fetching download links: %w", err)
 	}
 
+	// Fire contribute in background before printing so the goroutine starts immediately.
+	var wg sync.WaitGroup
+	if !noContribute {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			postContribute(product.ID, lang.ID, rawJSON)
+		}()
+	}
+
 	for _, link := range links {
 		fmt.Println(link.URI)
 	}
+
+	if !noContribute {
+		fmt.Fprintln(os.Stderr, "Shared with msdl.tech cache to help other users (--no-contribute to opt out)")
+		// Wait up to 5s for the contribution to complete before process exit.
+		done := make(chan struct{})
+		go func() { wg.Wait(); close(done) }()
+		select {
+		case <-done:
+		case <-time.After(5 * time.Second):
+		}
+	}
 	return nil
 }
 

cli/microsoft.go

@@ -266,7 +266,8 @@ func fetchLanguages(client *http.Client, sessionID, productID string) ([]Languag
 	return parseSkuInfo(raw)
 }
 
-func fetchDownloadLinks(client *http.Client, sessionID, productID, skuID string) ([]DownloadLink, error) {
+// fetchDownloadLinks returns parsed links and the raw Microsoft JSON (for cache contribution).
+func fetchDownloadLinks(client *http.Client, sessionID, productID, skuID string) ([]DownloadLink, []byte, error) {
 	wq := url.Values{}
 	wq.Set("profile", msProfile)
 	wq.Set("productEditionId", productID)
@@ -289,9 +290,13 @@ func fetchDownloadLinks(client *http.Client, sessionID, productID, skuID string)
 		"https://www.microsoft.com/software-download-connector/api/GetProductDownloadLinksBySku?"+q.Encode(),
 		productID)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	return parseDownloadLinks(raw)
+	links, err := parseDownloadLinks(raw)
+	if err != nil {
+		return nil, nil, err
+	}
+	return links, raw, nil
 }
 
 func extractFwlinks(pageHTML string) []string {