Compare commits
54 commits
cli/v0.3.2
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fba4502055 | ||
|
|
d562bf8d0e | ||
|
|
f6659fcc42 | ||
|
|
34e2b35486 | ||
|
|
f27e2931b4 | ||
|
|
33f3a46777 | ||
|
|
cf318eb15c | ||
|
|
78be837a4a | ||
|
|
9e22cd8bf8 | ||
|
|
6ace2ceb19 | ||
|
|
6ab849be5e | ||
|
|
5a806eea87 | ||
|
|
73eee730c9 | ||
|
|
b516804541 | ||
|
|
47698d8129 | ||
|
|
61fdcffaaa | ||
|
|
a831cddb8e | ||
|
|
8019db0314 | ||
|
|
da1e894e6b | ||
|
|
c53eae3f0e | ||
|
|
c30c2d2b3b | ||
|
|
da8787e93a | ||
|
|
f67fe84348 | ||
|
|
3ceb179ae6 | ||
|
|
11e7f7274f | ||
|
|
4dce41483d | ||
|
|
2fae096b15 | ||
|
|
5eec0351e7 | ||
|
|
151f70d67f | ||
|
|
eddf7802bb | ||
|
|
c14c18e955 | ||
|
|
e660e95d49 | ||
|
|
f514d972a3 | ||
|
|
388f115db6 | ||
|
|
a314cdcb5e | ||
|
|
1432cb3158 | ||
|
|
327c5e195d | ||
|
|
b252ba1c30 | ||
|
|
3994a82d6d | ||
|
|
b1ee80f9b6 | ||
|
|
6e374e6a45 | ||
|
|
1d5fc9c0f5 | ||
|
|
e1151ff0cf | ||
|
|
4bb0ba9f2a | ||
|
|
98f61b1831 | ||
|
|
e2e41e8ecf | ||
|
|
c4fb7a251b | ||
|
|
a3f75c1ce5 | ||
|
|
75295943b6 | ||
|
|
61a296b92b | ||
|
|
48a6cb5eb4 | ||
|
|
8b430fab5d | ||
|
|
e5d9dc3d61 | ||
|
|
ef64e06b99 |
45 changed files with 1821 additions and 3216 deletions
3
.gitattributes
vendored
Normal file
3
.gitattributes
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
aur/msdl-bin/PKGBUILD text eol=lf
|
||||
aur/msdl-bin/.SRCINFO text eol=lf
|
||||
backend/install.sh text eol=lf
|
||||
1
.github/FUNDING.yml
vendored
Normal file
1
.github/FUNDING.yml
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
github: starkSV
|
||||
47
.github/workflows/cli-release.yml
vendored
47
.github/workflows/cli-release.yml
vendored
|
|
@ -26,6 +26,7 @@ jobs:
|
|||
VERSION="${VERSION#cli/v}"
|
||||
mkdir -p ../dist
|
||||
GOOS=linux GOARCH=amd64 go build -ldflags="-s -w -X main.Version=${VERSION}" -o ../dist/msdl-linux-amd64 .
|
||||
GOOS=linux GOARCH=arm64 go build -ldflags="-s -w -X main.Version=${VERSION}" -o ../dist/msdl-linux-arm64 .
|
||||
GOOS=darwin GOARCH=amd64 go build -ldflags="-s -w -X main.Version=${VERSION}" -o ../dist/msdl-darwin-amd64 .
|
||||
GOOS=darwin GOARCH=arm64 go build -ldflags="-s -w -X main.Version=${VERSION}" -o ../dist/msdl-darwin-arm64 .
|
||||
GOOS=windows GOARCH=amd64 go build -ldflags="-s -w -X main.Version=${VERSION}" -o ../dist/msdl-windows-amd64.exe .
|
||||
|
|
@ -35,7 +36,10 @@ jobs:
|
|||
run: |
|
||||
PREV_TAG=$(git tag --sort=-version:refname | grep 'cli/v' | sed -n '2p')
|
||||
if [ -n "$PREV_TAG" ]; then
|
||||
CHANGELOG=$(git log --pretty=format:"- %s" "$PREV_TAG".."${{ github.ref_name }}" -- cli/ backend/ frontend/ | grep -v '^- chore(winget)' | head -30)
|
||||
CHANGELOG=$(git log --pretty=format:"- %s" "$PREV_TAG".."${{ github.ref_name }}" -- cli/ backend/ frontend/ .github/workflows/cli-release.yml | grep -v '^- chore(winget)' | grep -v '^- chore(release): bump' | head -30)
|
||||
if [ -z "$CHANGELOG" ]; then
|
||||
CHANGELOG="- Maintenance release (see commit history for details)"
|
||||
fi
|
||||
else
|
||||
CHANGELOG="- Initial release"
|
||||
fi
|
||||
|
|
@ -63,12 +67,36 @@ jobs:
|
|||
|
||||
**Windows (direct download):** Download `msdl-windows-amd64.exe` below, rename to `msdl.exe`, place in a folder on your PATH.
|
||||
|
||||
**Linux/macOS:**
|
||||
**macOS/Linux (Homebrew):**
|
||||
```bash
|
||||
brew tap starkSV/msdl
|
||||
brew install msdl-cli
|
||||
```
|
||||
(formula is `msdl-cli`, not `msdl` -- homebrew/core has an unrelated package named `msdl`; the installed command is still just `msdl`)
|
||||
|
||||
**Arch Linux (AUR):**
|
||||
```bash
|
||||
yay -S msdl-bin
|
||||
# or: paru -S msdl-bin
|
||||
```
|
||||
[aur.archlinux.org/packages/msdl-bin](https://aur.archlinux.org/packages/msdl-bin)
|
||||
|
||||
**macOS/Linux (no Homebrew/AUR):**
|
||||
```bash
|
||||
curl -fsSL https://api.msdl.tech-latest.com/install.sh | bash
|
||||
```
|
||||
Auto-detects OS/arch (including Termux on Android) and installs the latest release.
|
||||
|
||||
**Linux/macOS (direct download):**
|
||||
```bash
|
||||
# Linux x86_64
|
||||
curl -L https://github.com/starkSV/windows-iso-downloader/releases/download/${{ github.ref_name }}/msdl-linux-amd64 -o msdl
|
||||
chmod +x msdl && sudo mv msdl /usr/local/bin/
|
||||
|
||||
# Linux ARM64 (also works in Termux on Android)
|
||||
curl -L https://github.com/starkSV/windows-iso-downloader/releases/download/${{ github.ref_name }}/msdl-linux-arm64 -o msdl
|
||||
chmod +x msdl && mv msdl $PREFIX/bin/ 2>/dev/null || sudo mv msdl /usr/local/bin/
|
||||
|
||||
# macOS Apple Silicon
|
||||
curl -L https://github.com/starkSV/windows-iso-downloader/releases/download/${{ github.ref_name }}/msdl-darwin-arm64 -o msdl
|
||||
chmod +x msdl && sudo mv msdl /usr/local/bin/
|
||||
|
|
@ -82,13 +110,20 @@ jobs:
|
|||
```
|
||||
msdl # fully interactive (includes Server/Enterprise)
|
||||
msdl "windows 11 25h2" # filter products, pick language
|
||||
msdl --id 3262 --lang "English (United States)" # direct, no prompts
|
||||
msdl --id 3262 --lang "English" # direct, no prompts
|
||||
msdl --eval server-2025 # evaluation ISOs directly
|
||||
msdl --list # list all products
|
||||
```
|
||||
|
||||
submit-winget:
|
||||
# wingetcreate is a native Windows tool (needs .NET 6 + VC++ Redistributable) --
|
||||
# it is NOT published as a NuGet/dotnet-tool package, so it can only run on a
|
||||
# windows-latest runner, not ubuntu-latest like the release job above.
|
||||
needs: release
|
||||
runs-on: windows-latest
|
||||
steps:
|
||||
- name: Submit winget update
|
||||
if: success()
|
||||
shell: bash
|
||||
env:
|
||||
WINGET_TOKEN: ${{ secrets.WINGET_TOKEN }}
|
||||
run: |
|
||||
|
|
@ -96,10 +131,10 @@ jobs:
|
|||
echo "WINGET_TOKEN not set, skipping winget update"
|
||||
exit 0
|
||||
fi
|
||||
dotnet tool install --global wingetcreate
|
||||
curl -L -o wingetcreate.exe https://github.com/microsoft/winget-create/releases/latest/download/wingetcreate.exe
|
||||
VERSION="${{ github.ref_name }}"
|
||||
VERSION="${VERSION#cli/v}"
|
||||
wingetcreate update starkSV.msdl \
|
||||
./wingetcreate.exe update starkSV.msdl \
|
||||
--version "$VERSION" \
|
||||
--urls "https://github.com/starkSV/windows-iso-downloader/releases/download/${{ github.ref_name }}/msdl-windows-amd64.exe" \
|
||||
--submit \
|
||||
|
|
|
|||
62
CLAUDE.md
62
CLAUDE.md
|
|
@ -37,8 +37,64 @@ negative cache (60s), dynamic TTL, stale-on-failure, jitter. Verified in product
|
|||
- [x] **Recently viewed** — localStorage only. Shows on homepage. Consumer + eval pages both tracked. Expired state shown when link expiry known.
|
||||
- ~~**File size**~~ — not feasible. Microsoft CDN does not return file size in the API response.
|
||||
|
||||
### Sentinel WAF resilience (ongoing)
|
||||
|
||||
Confirmed via three weekly `/metrics` + docker-log checkpoints (2026-07-14 → 07-21 → 07-31):
|
||||
the backend's own direct link-fetch (`/proxy` → Microsoft) is **100% blocked**, permanently —
|
||||
`link.ms_fetches: 0` every time, and zero even-attempted-and-failed fetches in the raw logs
|
||||
(the lockdown gate short-circuits before reaching that code path). The site stays alive
|
||||
entirely on cached/stale entries plus CLI-contributed links (286 accepted, 0 rejected,
|
||||
confirmed 2026-07-31). `/skuinfo` and `/evallinks` are unaffected — only the download-link
|
||||
endpoint is targeted.
|
||||
|
||||
The CLI's own residential-IP requests also see a stable ~20% Sentinel rejection rate across
|
||||
all three checkpoints — not improving, not worsening. Microsoft's own API error response
|
||||
literally names the system (`"Sentinel marked this request as rejected."`, `Type: 9` in
|
||||
`Errors[]`), confirming it's a real, named product, not our guess. The clean structured-JSON
|
||||
deny (not an HTML/JS challenge page) suggests a signature/reputation gate rather than full
|
||||
interactive bot-management — TLS ClientHello fingerprinting is a plausible contributing
|
||||
signal, since Go's stdlib `crypto/tls` doesn't look like any real browser, independent of IP.
|
||||
|
||||
- [x] **Back off Sentinel retries** — bumped `lockdownTTL` 90min → 5h (`backend/main.go`).
|
||||
181 retries over 17 days, 0 successes; retrying that often was pure noise.
|
||||
(fix/sentinel-lockdown-and-proxy-validation, merged)
|
||||
- [x] **Validate `product_id` in `/proxy`** — was passing unknown IDs straight through to a
|
||||
real Microsoft session attempt (found via a stray `product_id=2861`, never a real
|
||||
product, in the logs). Now rejected with 404 before any outbound call. (same PR, merged)
|
||||
- [x] **CLI TLS/HTTP2 fingerprint hardening** — swapped the CLI's transport from stdlib
|
||||
`net/http` to `github.com/bogdanfinn/tls-client` (wraps `utls` with a maintained Chrome
|
||||
profile). Shipped in `cli/v0.3.7` (merged, released 2026-07-31). **Checked in on
|
||||
2026-08-17, 17 days post-release: inconclusive-to-negative.** Aggregate CLI
|
||||
Sentinel-rejection rate is still ~21%, statistically unchanged from every pre-fix
|
||||
checkpoint. Real caveat: mixed-version population (`0.3.6`: 1156 actions vs `0.3.7`: 493
|
||||
in that window) means a `0.3.7`-only improvement could still be masked — `/metrics`
|
||||
doesn't break the error down by CLI version, so this can't be fully isolated yet. Revisit
|
||||
once `0.3.7`+ dominates usage share; if the aggregate still hasn't moved by then, treat
|
||||
the fingerprint theory as disproven.
|
||||
- [ ] **`/needs-warming` community page** — proposed, not built. Surfaces products currently
|
||||
failing web users (active Sentinel/rate-limit lockdown, no cached/stale link available)
|
||||
with a one-click CLI command to fix it. See
|
||||
`docs/superpowers/specs/2026-07-13-needs-warming-design.md`.
|
||||
- [ ] **Per-language SHA256 checksums on product pages** — Microsoft's own download pages
|
||||
(e.g. `/software-download/windows11`) publish a static hash table (one SHA256 per
|
||||
locale) for the *current* ISO build, confirmed live via direct network inspection
|
||||
2026-08-18. It's not in any API response MSDL/CLI call (`GetProductDownloadLinksBySku`
|
||||
never includes a hash, confirmed against Microsoft's own live page too) — it's static
|
||||
HTML on the Windows-version download page, tied to the product/build, not the session.
|
||||
Since it only changes when a product ID is replaced (same trigger as adding a new
|
||||
catalog entry), no scraper/refresh job needed: copy the hash table by hand at the same
|
||||
time a new product ID is added, per the existing "Adding a new consumer Windows release"
|
||||
steps in CONTRIBUTING.md. Open questions before building: where to store it
|
||||
(`products.json` field vs. separate file), which languages to cover (all 38 vs. just
|
||||
the popular ones), and where to surface it (collapsible "Verify your download" section
|
||||
on the product page, mirroring Microsoft's own UX; maybe CLI output too).
|
||||
|
||||
### Known bugs / open issues
|
||||
|
||||
- [ ] **`_redirects` Cloudflare Pages** — SPA fallback rule (`/* /index.html 200`) is flagged
|
||||
as an infinite loop and ignored by Cloudflare Pages, potentially causing 404s on
|
||||
direct navigation to non-home routes. Needs investigation and fix.
|
||||
- [x] **`_redirects` Cloudflare Pages** — investigated 2026-07-21. The rule was indeed ignored
|
||||
(Cloudflare's deploy log flags it as a false-positive infinite loop), but this turned out
|
||||
to be a non-issue: Cloudflare Pages' own default fallback already serves `index.html` for
|
||||
any unmatched path, so the explicit rule was redundant. Verified live: direct navigation to
|
||||
`/about`, a dynamic route (`/product/3262`), and a genuinely invalid path all return HTTP 200
|
||||
with the correct content (including React Router's own 404 page for the invalid one). Removed
|
||||
`frontend/public/_redirects` entirely to stop the recurring deploy warning.
|
||||
|
|
|
|||
55
CONTRIBUTING.md
Normal file
55
CONTRIBUTING.md
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
# Contributing to MSDL
|
||||
|
||||
Thanks for considering a contribution. This project is a small, community-run tool — bug reports, product additions, and fixes are all welcome.
|
||||
|
||||
## Reporting bugs
|
||||
|
||||
Open a [GitHub issue](https://github.com/starkSV/windows-iso-downloader/issues). Include:
|
||||
|
||||
- Whether you hit it via the web app, the CLI, or both
|
||||
- CLI version (`msdl --help` prints it) or which page on msdl.tech-latest.com
|
||||
- The exact error message, if any
|
||||
- Product/language you were trying to fetch
|
||||
|
||||
## Development setup
|
||||
|
||||
See the [README's "Running Locally" section](./README.md#running-locally) for backend and frontend setup. For the CLI:
|
||||
|
||||
```bash
|
||||
cd cli
|
||||
go run . --help
|
||||
go test ./...
|
||||
```
|
||||
|
||||
## Adding a new consumer Windows release
|
||||
|
||||
1. Find the product ID at `www.microsoft.com/software-download-connector/api/`
|
||||
2. Add it to `frontend/public/data/products.json` (name, archs, badge, related, active)
|
||||
3. Add it to `cli/catalog.go`'s `consumerProducts` slice
|
||||
4. Update the product table in `README.md`
|
||||
|
||||
## Adding a new evaluation edition
|
||||
|
||||
1. Find the fwlink URL at `microsoft.com/en-us/evalcenter/download-*`
|
||||
2. Add the slug and fwlink to the `evalProducts` map in `backend/main.go`
|
||||
3. Add the same slug to `cli/catalog.go`'s `evalProducts` slice
|
||||
4. Add the product config to `frontend/src/data/evalProducts.ts`
|
||||
5. Update the eval table in `README.md`
|
||||
|
||||
## Making changes
|
||||
|
||||
- Small, one-line config fixes can go straight to `main`.
|
||||
- Anything touching behavior (new feature, bug fix, refactor) should go through a feature branch and PR.
|
||||
- **Test locally before opening a PR** — build and run the affected component (`go build`, `go test ./...`, `npm run dev`) rather than relying on review to catch it.
|
||||
- Commit messages follow [Conventional Commits](https://www.conventionalcommits.org/): `feat:`, `fix:`, `docs:`, `chore:`.
|
||||
- Keep PRs focused — one fix or feature per PR, not a bundle of unrelated changes.
|
||||
|
||||
## CLI releases
|
||||
|
||||
CLI releases are cut by the maintainer via git tag (`cli/vX.Y.Z`), which triggers the GitHub Actions build for all platforms. Contributors don't need to worry about tagging or versioning — just get the fix or feature merged to `main`.
|
||||
|
||||
Package manager updates (winget, the [Homebrew tap](https://github.com/starkSV/homebrew-msdl), and the [`msdl-bin` AUR package](./aur/msdl-bin)) are submitted manually by the maintainer after each release, not automated in CI.
|
||||
|
||||
## Code style
|
||||
|
||||
No linter is enforced yet — match the surrounding code's style (Go: stdlib-first, minimal dependencies; frontend: existing Tailwind/component patterns). Don't add abstractions, comments, or error handling beyond what the change actually needs.
|
||||
|
|
@ -1,284 +0,0 @@
|
|||
# MSDL Resilience & CLI Implementation Plan
|
||||
|
||||
> **Status:** Proposed — 2026-06-17
|
||||
> **Context:** Microsoft's Azure Sentinel WAF is blocking data-center IPs (Hetzner, Cloudflare Workers) on the `GetProductDownloadLinksBySku` endpoint at the ASN level. Two senior engineers independently concluded that server-side calls from hosting IPs are not sustainable, and that moving the Microsoft call onto the end-user's machine (the Rufus/Fido model) is the only durable fix.
|
||||
|
||||
---
|
||||
|
||||
## 1. Problem Statement
|
||||
|
||||
MSDL fetches signed Windows ISO download URLs from Microsoft's internal download API. That endpoint is now protected by Azure Sentinel WAF, which blocks by **ASN reputation** — data-center ranges are cleanly distinguishable from residential/commercial ISPs, so Microsoft can throttle hosting traffic with zero impact on real browser users.
|
||||
|
||||
**Observed escalation:**
|
||||
- 2026-06-15: ~2 Sentinel rejections/day, intermittent
|
||||
- 2026-06-16: ~15 rejections clustered in the evening, consistently hitting Win11 25H2 (products 3262, 3321)
|
||||
- Windows 10 (2618) and eval builds still succeed; newer/high-traffic products are targeted first
|
||||
|
||||
**Current blast radius:** The caching layer (95% hit rate) absorbs most of it. The failure mode is: cache expires → backend refresh is WAF-blocked → stale-on-failure serves the old URL → eventually the URL passes its real `se` expiry → **user gets a dead Microsoft CDN link.**
|
||||
|
||||
**Confirmed not fixable server-side:** IP rotation buys weeks; CF Workers already blocked; residential proxies cost money and add a volatile dependency unjustifiable for a free tool. This is Microsoft policy, not a glitch.
|
||||
|
||||
---
|
||||
|
||||
## 2. Architecture Decision
|
||||
|
||||
**Invert the network footprint.** Today: `backend (Hetzner IP) → Microsoft → users`. The hard call originates from a blocked ASN.
|
||||
|
||||
New model: **`users (residential IP) → Microsoft → backend cache → all users`**
|
||||
|
||||
Three coordinated changes:
|
||||
|
||||
1. **CLI calls Microsoft directly** from the user's own machine. Their residential/commercial IP is one Microsoft cannot block without breaking downloads for real customers. This is exactly why Rufus/Fido work.
|
||||
2. **Crowdsourced cache warming.** When a CLI user fetches a fresh link, the CLI optionally contributes that URL back to the MSDL backend. Microsoft's signed URLs are *not* user-specific (the `se` param is just an expiry timestamp — MSDL already serves one cached URL to many users), so a link fetched by one user is valid for everyone. The backend's Microsoft API calls trend toward zero; Sentinel becomes irrelevant.
|
||||
3. **Web graceful degradation.** When the cache is dead and the backend is blocked, the web stops silently serving dead links — it flags expiry honestly and offers both the official Microsoft fallback and the web-to-CLI handoff.
|
||||
|
||||
**What stays unchanged:** The Go backend keeps serving the React frontend. Eval builds (separate Eval Center endpoint, unaffected) remain a first-class, reliable offering. The two-layer caching system is retained and becomes the distribution layer for crowdsourced links.
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────┐
|
||||
│ User visits msdl.tech-latest.com │
|
||||
└───────────────────┬───────────────────────┘
|
||||
│
|
||||
Cached link valid & fresh?
|
||||
──────────┬──────────────┬──────────
|
||||
(Yes) │ │ (No / expired / WAF-blocked)
|
||||
▼ ▼
|
||||
Serve signed link Graceful degradation:
|
||||
1. Show "likely expired" flag
|
||||
2. Official Microsoft fallback (guided)
|
||||
3. Web-to-CLI handoff command
|
||||
│
|
||||
▼
|
||||
User runs: msdl --id 3262 --lang "..."
|
||||
│
|
||||
┌────────────────────────┼────────────────────────┐
|
||||
▼ ▼ ▼
|
||||
CLI fetches from MS Prints URL to user POST /contribute
|
||||
(their residential IP) (immediate result) (warms shared cache)
|
||||
│
|
||||
▼
|
||||
Next web visitor: cache hit
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Phasing Overview
|
||||
|
||||
| Phase | Deliverable | Why this order |
|
||||
|-------|-------------|----------------|
|
||||
| **1** | Web graceful degradation (expired flag + official fallback) | Immediate — stops users hitting silent dead links today. Pure frontend + small backend signal. No dependency on the CLI. |
|
||||
| **2** | CLI rework: direct-to-Microsoft | The core durable fix. Replaces the backend-routed CLI plan. |
|
||||
| **3** | Crowdsourced cache contribution (`/contribute` + CLI flag) | Depends on Phase 2 CLI existing. Turns every CLI run into cache warming. |
|
||||
| **4** | Web-to-CLI handoff UX | Depends on Phases 2–3. Wires the website's expired state to the CLI install/command. |
|
||||
| **5** | Distribution & winget | Ships the CLI to users (covered in detail in the existing TDD plan). |
|
||||
|
||||
> The granular, TDD-level, step-by-step build instructions for the CLI binary itself (module scaffold, catalog, picker, flags, release workflow, winget manifests) already live in
|
||||
> [`docs/superpowers/plans/2026-06-15-msdl-cli.md`](docs/superpowers/plans/2026-06-15-msdl-cli.md).
|
||||
> **That plan must be amended for Phase 2** (it currently routes through `api.msdl.tech-latest.com`; it must instead port the Microsoft session flow into the binary). This document is the strategic master; that document is the implementation detail.
|
||||
|
||||
---
|
||||
|
||||
## Phase 1 — Web Graceful Degradation (Immediate)
|
||||
|
||||
**Goal:** No user is ever silently handed a dead Microsoft CDN link. When the link is at-risk or unavailable, say so and give a guaranteed path forward.
|
||||
|
||||
### 1.1 Backend: expose link freshness in the proxy response
|
||||
|
||||
**Files:** `backend/main.go` (`handleProxy`, ~line 817; `linkCacheEntry` line 105)
|
||||
|
||||
The frontend already parses `se` for the countdown, but it can't distinguish "fresh from Microsoft" from "stale-on-failure served past a failed refresh." Surface that explicitly.
|
||||
|
||||
- Add a response header on `/proxy` responses indicating provenance and confidence:
|
||||
- `X-MSDL-Link-Status: fresh` — fetched live from Microsoft this request
|
||||
- `X-MSDL-Link-Status: cached` — served from a valid (not-yet-soft-expired) cache entry
|
||||
- `X-MSDL-Link-Status: stale` — served via stale-on-failure after a WAF block; **may be expired**
|
||||
- The `stale` branch is the existing `serving stale` path (the `fetch failed (...), serving stale` log line). Set the header there.
|
||||
- Also emit `X-MSDL-Link-Expires: <RFC3339>` (the `se`-derived expiry) so the frontend has authoritative metadata for the countdown without re-parsing the URL itself. *(Engg-2 suggestion.)*
|
||||
- Ensure `enableCORS` (line 708) exposes the headers: add `Access-Control-Expose-Headers: X-MSDL-Link-Status, X-MSDL-Link-Expires`.
|
||||
|
||||
### 1.2 Frontend: "likely expired" flag
|
||||
|
||||
**Files:** `frontend/src/pages/ProductDetailPage.tsx`
|
||||
|
||||
- Read `X-MSDL-Link-Status` from the `/proxy` fetch response.
|
||||
- When `stale`, render an amber inline warning above the download button:
|
||||
> ⚠️ This link is cached and may have expired. Microsoft is currently limiting automated requests. Use the options below to get a guaranteed fresh download.
|
||||
- When `fresh`/`cached`, keep the existing expiry countdown unchanged.
|
||||
|
||||
### 1.3 Frontend: official Microsoft fallback (guided)
|
||||
|
||||
**Files:** `frontend/src/pages/ProductDetailPage.tsx`, new `frontend/src/components/OfficialFallback.tsx`
|
||||
|
||||
- Below the download card, a collapsible "Get it directly from Microsoft" section that:
|
||||
- Links to the official page (`https://www.microsoft.com/software-download/windows11`, or the matching page per product family).
|
||||
- Lists the exact dropdown selections the user must make (edition + the language matching their chosen SKU).
|
||||
- **Data-driven, not hardcoded** *(Engg-2 suggestion):* derive the official-page URL and edition label from the product catalog (`frontend/public/data/products.json`) keyed by product family, and use the SKU's own language string for the language step — so adding a product doesn't require touching the fallback component.
|
||||
- This keeps MSDL useful as an **information directory** even when automation is fully blocked.
|
||||
|
||||
### 1.4 Acceptance
|
||||
|
||||
- Trigger a stale serve (point the backend at a WAF-blocked product or mock the stale path); confirm the amber warning + official fallback both render.
|
||||
- Confirm `fresh`/`cached` responses show no warning and the countdown still works.
|
||||
- `git commit -m "feat: graceful degradation for WAF-blocked links (expiry flag + official fallback)"`
|
||||
|
||||
---
|
||||
|
||||
## Phase 2 — CLI Direct-to-Microsoft (Core Durable Fix)
|
||||
|
||||
**Goal:** A standalone `msdl` binary that performs the **entire** Microsoft flow locally — session setup, SKU lookup, link fetch — so the request originates from the user's residential IP.
|
||||
|
||||
### 2.1 Amend the existing CLI plan
|
||||
|
||||
The detailed plan at [`docs/superpowers/plans/2026-06-15-msdl-cli.md`](docs/superpowers/plans/2026-06-15-msdl-cli.md) currently has `cli/api.go` calling `api.msdl.tech-latest.com`. **Replace that approach:** the CLI must own the Microsoft logic. Keep from that plan: module scaffold (Task 1), product catalog (Task 2), interactive picker (Task 4), main flow/flags (Task 5), release workflow + winget (Tasks 6–7). **Rewrite the API layer (Task 3).**
|
||||
|
||||
### 2.2 Port the Microsoft session flow into the CLI
|
||||
|
||||
**Source (backend):** these functions are the reference implementation to port —
|
||||
- `setupSession() (string, *simpleCookieJar, error)` — `backend/main.go:394` (vlscppe → ov-df fingerprinting chain + cookie accumulation)
|
||||
- `simpleCookieJar` — custom `http.CookieJar` with domain scoping disabled
|
||||
- `fetchSkuInfoFromMS(productID string) ([]byte, int, error)` — `backend/main.go:470`
|
||||
- `fetchDownloadLinksFromMS(productID, skuID string) ([]byte, error)` — `backend/main.go:566`
|
||||
|
||||
**Target (CLI):** create `cli/microsoft.go` with the ported equivalents. Strip server concerns (no shared session map — the CLI creates a fresh session per invocation, which is fine and more browser-like). Reuse the catalog and picker from the existing plan.
|
||||
|
||||
> **Shared package vs. copy** *(Engg-2 raised `pkg/microsoft/`):* the CLI is a separate Go module (`cli/go.mod`), so sharing a package across the backend and CLI modules requires a Go workspace or `replace` directives — added friction for ~250 lines of stable code. **Decision: copy/port for v1** (the logic changes rarely; divergence risk is low). Revisit a shared module only if the fingerprinting flow starts changing frequently and the two copies drift.
|
||||
|
||||
**New `cli/microsoft.go` responsibilities:**
|
||||
- `newSession() (*http.Client, string)` — build the `simpleCookieJar`-backed client, run the fingerprinting chain, return the client + session ID.
|
||||
- `fetchLanguages(client, sessionID, productID) ([]Language, error)` — GET SKU info, parse `Skus`.
|
||||
- `fetchDownloadLinks(client, sessionID, productID, skuID) ([]DownloadLink, error)` — link fetch (with SKU-info warmup), parse `ProductDownloadOptions`.
|
||||
- `fetchEvalLinks(evalURL) ([]EvalLink, error)` — scrape the Eval Center page directly and resolve fwlinks.
|
||||
|
||||
**Eval builds run directly too.** For consistency and future-proofing, the CLI scrapes the Microsoft Eval Center itself rather than proxying evals through the backend. The Eval Center isn't WAF-blocked today, but "today" is exactly the assumption that just broke for the consumer endpoint — keeping the CLI fully backend-independent means it never inherits a future Eval Center block.
|
||||
|
||||
### 2.3 TLS fingerprint hardening (note, not blocking)
|
||||
|
||||
Go's default `crypto/tls` ClientHello produces a JA3 fingerprint unlike any real browser. On user residential IPs this is a secondary signal (IP diversity is the primary protection), so ship with stdlib first. **If Sentinel starts fingerprinting the CLI,** swap `net/http`'s transport for [`utls`](https://github.com/refraction-networking/utls) with a Chrome ClientHello profile. Track as a follow-up, not a launch blocker.
|
||||
|
||||
### 2.4 Acceptance
|
||||
|
||||
- On a residential connection: `msdl --id 3262 --lang "English (United States)"` prints a valid signed URL with no backend involved.
|
||||
- `msdl --id 3321` (a product the backend currently gets WAF-blocked on) succeeds from a home IP — this is the proof the architecture works.
|
||||
- Eval builds: `msdl --eval server-2025` scrapes the Eval Center directly and prints the resolved `.iso` URLs — no backend involved.
|
||||
|
||||
---
|
||||
|
||||
## Phase 3 — Crowdsourced Cache Contribution
|
||||
|
||||
**Goal:** Every CLI link fetch optionally warms the shared backend cache, so web visitors get hits without the backend ever calling Microsoft.
|
||||
|
||||
### 3.0 Redis/Valkey cache persistence (prerequisite to 3.1)
|
||||
|
||||
**Why:** Without persistence, a backend restart wipes the link cache. If the backend is also WAF-blocked it cannot rebuild from Microsoft — crowdsourced contributions would be the only source of fresh links, so losing them on restart defeats Phase 3 entirely.
|
||||
|
||||
**Stack:** Redis/Valkey on Coolify (same Hetzner instance). TTL-native, Go has a first-class client (`github.com/redis/go-redis/v9`), zero added ops cost. Preferred over PostgreSQL (overkill for key-value TTL cache) and CF KV (requires routing through the Worker from Go, awkward).
|
||||
|
||||
**Architecture:** Two-layer cache — Redis as L2 persistent store; the existing in-memory `linkCache` map stays as L1 for hot reads.
|
||||
|
||||
**Files:** `backend/main.go`, `backend/go.mod`
|
||||
|
||||
- Add `REDIS_URL` env var (e.g. `redis://localhost:6379`). If unset, skip Redis silently — backend runs in memory-only mode for local dev.
|
||||
- Key format: `msdl:link:<productID>:<skuID>` (namespaced).
|
||||
- On startup: seed in-memory cache from Redis (so a restart recovers immediately).
|
||||
- On link cache write (fresh fetch or contribution): `SET key value EX <ttl_seconds>` using `time.Until(entry.ExpiresAt)`.
|
||||
- On in-memory cache miss in `handleProxy`: check Redis before calling Microsoft.
|
||||
- If Redis is unavailable at runtime: log a warning and fall through to in-memory-only (keeps the existing failure behaviour, no hard dependency).
|
||||
|
||||
**Deploy order:** Provision a Valkey service on Coolify and set `REDIS_URL` before Phase 3 goes to production. Add `github.com/redis/go-redis/v9` to `backend/go.mod`.
|
||||
|
||||
---
|
||||
|
||||
### 3.1 Backend: `/contribute` endpoint
|
||||
|
||||
**Files:** `backend/main.go` (register in `main()` ~line 1147; cache write mirrors `handleProxy` line 922)
|
||||
|
||||
- `POST /contribute` accepting JSON: `{ "product_id": "3262", "sku_id": "19675", "raw_json": <the exact GetProductDownloadLinksBySku response bytes> }`.
|
||||
- **Validation (reject untrusted contributions):**
|
||||
1. `product_id` must be in the known catalog (reuse the allow-list logic the proxy uses).
|
||||
2. Parse `raw_json` with the existing `parseLinkExpiry(rawJSON []byte) time.Time` (line 208). Reject if expiry is in the past or < 1 hour out.
|
||||
3. Extract the URL(s) from `ProductDownloadOptions`; reject unless the host matches the Microsoft CDN pattern (e.g. `*.download.prss.microsoft.com` / `software-static.download.prss.microsoft.com`).
|
||||
4. **No `HEAD` validation by default** *(both engineers):* parsing + host-allow-list + expiry are the real defense, and an extra request to Microsoft's CDN on every contribution adds traffic Microsoft may be sensitive to. Rely on parsing first; only add a `HEAD` check later if junk slips through.
|
||||
- On success, write to the shared cache exactly as `handleProxy` does:
|
||||
```go
|
||||
cacheKey := productID + ":" + skuID
|
||||
expiresAt := parseLinkExpiry(raw)
|
||||
linkCacheMu.Lock()
|
||||
linkCache[cacheKey] = linkCacheEntry{RawJSON: raw, ExpiresAt: expiresAt, FetchedAt: time.Now()}
|
||||
linkCacheMu.Unlock()
|
||||
```
|
||||
- Respond `204 No Content` on accept, `400`/`422` on validation failure.
|
||||
- **Logging** *(Engg-2):* log accepts as `contribute: product_id=%s sku_id=%s -> accepted, cached until %s` AND rejections clearly as `contribute: product_id=%s sku_id=%s -> rejected (<reason>)` so contribution patterns/abuse are visible in ops logs from day one.
|
||||
- **Abuse protection:** since this writes to a cache served to all users —
|
||||
- **Per-IP rate limit** on `/contribute`: ~5/min *(Engg-2's concrete number)*. Build it as small middleware (a per-IP token bucket in a `map[string]…` with mutex) — there's no existing rate-limit middleware to reuse.
|
||||
- **`CONTRIBUTE_SECRET`** env var, baked into released CLI builds, checked as a request header. Cheap obfuscation that blocks casual abuse; not real security.
|
||||
- The host allow-list + expiry validation remain the real defense: the worst a bad actor can do is submit a *valid Microsoft link*, which is harmless.
|
||||
|
||||
### 3.2 CLI: contribution flag
|
||||
|
||||
**Files:** `cli/main.go`, `cli/microsoft.go`
|
||||
|
||||
- After a successful link fetch, POST the raw response to `https://api.msdl.tech-latest.com/contribute` in the background. **Truly non-blocking** *(Engg-2):* fire it in a goroutine with a `context.WithTimeout(ctx, 5*time.Second)`, send the `CONTRIBUTE_SECRET` as a header, ignore all errors. The user already has their URL printed before this runs — and `main` must wait on it only briefly (or the process may exit first; spawn it before printing the result and `wg.Wait()` with the same short timeout, or simply accept best-effort).
|
||||
- **Opt-out, transparent by default:** print to stderr `✓ Shared with msdl.tech cache to help other users (disable with --no-contribute)`. Add `--no-contribute` flag. Honor a `MSDL_NO_CONTRIBUTE=1` env var too.
|
||||
|
||||
### 3.3 Acceptance
|
||||
|
||||
- Run the CLI for a product whose backend cache is empty/expired; confirm `/contribute` logs an accept and a subsequent web request to `/proxy` for that product is a **cache hit** with `X-MSDL-Link-Status: cached`.
|
||||
- Submit a junk/non-Microsoft URL to `/contribute`; confirm rejection.
|
||||
- Submit a near-expired link; confirm rejection.
|
||||
|
||||
---
|
||||
|
||||
## Phase 4 — Web-to-CLI Handoff
|
||||
|
||||
**Goal:** When the web cache is dead and the backend is WAF-blocked, the site offers the CLI as the guaranteed path — and that CLI run heals the cache for the next visitor (Phase 3).
|
||||
|
||||
### 4.1 Frontend handoff UI
|
||||
|
||||
**Files:** `frontend/src/pages/ProductDetailPage.tsx`, `frontend/src/components/CliHandoff.tsx`
|
||||
|
||||
- In the `stale` state (from Phase 1.1), in addition to the official fallback, show:
|
||||
> **Get a fresh link instantly with the MSDL CLI** — runs from your computer, always works.
|
||||
- Install one-liner per platform (winget for Windows, curl|sh for Linux/macOS — final commands from the release in Phase 5).
|
||||
- The exact command pre-filled for this product: `msdl --id 3262 --lang "English (United States)"`
|
||||
- One-line explanation: "Runs the download lookup from your own connection, so Microsoft's rate limits don't apply. It also shares the fresh link back to help other visitors."
|
||||
- Copy-to-clipboard button (reuse the existing CLI command tab component pattern).
|
||||
|
||||
### 4.2 Acceptance
|
||||
|
||||
- Force a stale state; confirm the handoff block shows the correct install command + pre-filled product command, and copy works.
|
||||
|
||||
---
|
||||
|
||||
## Phase 5 — Distribution
|
||||
|
||||
Covered in full by [`docs/superpowers/plans/2026-06-15-msdl-cli.md`](docs/superpowers/plans/2026-06-15-msdl-cli.md) Tasks 6–7:
|
||||
- GitHub Actions release workflow (4-platform binaries, `cli/v*` tag trigger)
|
||||
- Winget publishing (`starkSV.msdl`) with `wingetcreate` auto-PR on future releases
|
||||
- `curl | sh` install script for Linux/macOS
|
||||
|
||||
No changes needed here beyond ensuring the `--no-contribute` flag and contribute endpoint URL are baked into release builds.
|
||||
|
||||
---
|
||||
|
||||
## 4. Cleanup / Catalog Note (do ASAP — independent of the above)
|
||||
|
||||
- **Product 48 (Windows 8.1 Single Language)** returns `ERROR [502]: no download links found for this SKU` on every attempt — this is a Microsoft-side removal, not a WAF block. **Both reviewers flagged this as do-ASAP.** Remove it from `frontend/public/data/products.json` now (and from `cli/catalog.go` when the CLI is built). This is a one-line config change → straight to main.
|
||||
|
||||
---
|
||||
|
||||
## 5. Risk & Open Questions
|
||||
|
||||
- **Contribution trust model:** the CDN-host allow-list + expiry validation make malicious contributions essentially harmless (worst case: a valid Microsoft link). Revisit if abuse appears.
|
||||
- **Eval Center longevity:** the CLI scrapes Eval Center directly (decided). If Microsoft changes that page's markup, `fwlinkRe`/`isoLangRe` in `cli/microsoft.go` (and the backend) need updating — low risk, easy fix.
|
||||
- **TLS fingerprinting (`utls`):** only if Sentinel begins fingerprinting the CLI on residential IPs. Not a launch blocker.
|
||||
- **Backend's own Microsoft calls:** once crowdsourced contributions are flowing, consider whether the backend should still attempt live fetches at all, or become contribution-only + stale-serving. Likely keep live fetch as a fallback for cold products with no contributor yet.
|
||||
|
||||
---
|
||||
|
||||
## 6. Sequencing Recommendation
|
||||
|
||||
1. **Phase 1 now** — independent, immediate user benefit, low risk.
|
||||
2. **Catalog cleanup (§4)** — trivial, do alongside Phase 1.
|
||||
3. **Phase 2** — amend & execute the CLI TDD plan with the direct-to-Microsoft rewrite.
|
||||
4. **Phase 3** — once the CLI fetches links, add contribution.
|
||||
5. **Phase 4 + 5** — wire the web handoff and ship distribution together.
|
||||
84
PROGRESS.md
84
PROGRESS.md
|
|
@ -2,6 +2,89 @@
|
|||
|
||||
## Shipped
|
||||
|
||||
### Sentinel WAF investigation & resilience hardening — backend merged, CLI fix pending
|
||||
|
||||
Three weekly `/metrics` + raw docker-log checkpoints (2026-07-14 → 07-21 → 07-31) turned the
|
||||
"Microsoft blocks our server IP" assumption from a one-off observation into a confirmed,
|
||||
stable fact — and revealed the crowdsourcing architecture below is now doing all the real work.
|
||||
|
||||
**What the data actually showed:**
|
||||
- `link.ms_fetches: 0` at every checkpoint — the backend's own direct link-fetch to Microsoft
|
||||
has not succeeded once, ever, across 17+ days.
|
||||
- Raw docker logs showed not just zero successes but zero *attempted-and-failed* fetches
|
||||
either — the existing Sentinel lockdown gate short-circuits the request before it even
|
||||
reaches Microsoft, serving cache/stale silently instead. 181 blocked session attempts over
|
||||
the same window, all 181 unsuccessful.
|
||||
- The site stayed online anyway: 286 CLI contributions accepted, 0 rejected, in that same
|
||||
window — confirming the crowdsourced cache from the CLI (see architecture entry below) is
|
||||
the thing actually keeping links fresh now, not the backend's own fetch.
|
||||
- `/skuinfo` (125 successful Microsoft fetches) and `/evallinks` (126 cache hits, 0 failures)
|
||||
are both unaffected — Sentinel targets the download-link endpoint specifically.
|
||||
- The CLI's own residential-IP requests held a stable ~20% Sentinel-rejection rate across all
|
||||
three checkpoints too — not improving, not worsening, despite running from IPs that
|
||||
shouldn't trip ASN-based blocking. Traced the exact error text
|
||||
(`"Sentinel marked this request as rejected."`) to Microsoft's own API response
|
||||
(`Errors[0].Value` with `Type: 9`) — confirming "Sentinel" is Microsoft's real name for this
|
||||
system, not our guess, and that it returns a clean structured JSON deny rather than an
|
||||
HTML/JS challenge page (suggesting a signature/reputation gate rather than full interactive
|
||||
bot management).
|
||||
|
||||
**Backend fixes (`fix/sentinel-lockdown-and-proxy-validation`, merged):**
|
||||
- `lockdownTTL` bumped 90min → 5h. 181 retries, 0 successes — retrying that often was pure
|
||||
noise against Microsoft (and our own IP's reputation) for zero return.
|
||||
- `/proxy` now validates `product_id` against the existing catalog allow-list before
|
||||
attempting a Microsoft session. Found via a stray `product_id=2861` (never a real product)
|
||||
in the logs — the backend was burning a full outbound attempt and a Sentinel-block hit on
|
||||
IDs that could never succeed anyway.
|
||||
|
||||
**CLI fix (`feat/cli-tls-fingerprint-hardening`, not yet merged):** swapped the CLI's HTTP
|
||||
transport from stdlib `net/http` to `github.com/bogdanfinn/tls-client` (wraps `utls` with a
|
||||
maintained Chrome TLS+HTTP2 fingerprint), on the theory that Go's default TLS handshake is
|
||||
itself a detectable non-browser signal, independent of IP. Verified functionally correct
|
||||
end-to-end (real link fetched and contributed), but that does *not* prove the theory — the
|
||||
old client already succeeded ~80% of the time. Real validation is watching the
|
||||
Sentinel-rejection-rate telemetry over a comparable multi-day window post-merge.
|
||||
|
||||
---
|
||||
|
||||
### CLI + resilience architecture (formerly `IMPLEMENTATION_PLAN.md`, Phases 1–5) — merged
|
||||
|
||||
Microsoft's Azure Sentinel WAF started blocking the backend's data-center IP (Hetzner) on
|
||||
the link endpoint. Rather than fight it server-side, inverted the network footprint: the
|
||||
CLI now talks to Microsoft directly from the user's own residential IP, and feeds fresh
|
||||
links back into the shared backend cache.
|
||||
|
||||
- **`msdl` CLI (`cli/`)** — standalone Go binary, ports the full Microsoft session flow
|
||||
(`cli/microsoft.go`: vlscppe permit → ov-df fingerprint → SKU lookup → signed link fetch)
|
||||
so requests originate from the user's machine, not a blockable ASN. Also scrapes the
|
||||
Eval Center directly for eval builds — no backend dependency either way.
|
||||
- **Crowdsourced cache contribution** — after a successful CLI fetch, the raw Microsoft
|
||||
response is POSTed to the backend's `POST /contribute` (validated against a product
|
||||
allow-list, CDN-host allow-list, and expiry check; rate-limited per IP; gated by
|
||||
`CONTRIBUTE_SECRET`). Opt out with `--no-contribute` or `MSDL_NO_CONTRIBUTE=1`.
|
||||
- **Redis/Valkey L2 cache** — `REDIS_URL` env var; in-memory cache is seeded from Redis on
|
||||
startup and written through on every fresh fetch, so a backend restart no longer loses
|
||||
crowdsourced links. Falls back to memory-only mode silently if unset/unreachable.
|
||||
Added `github.com/redis/go-redis/v9`.
|
||||
- **Web graceful degradation** — `/proxy` responses carry `X-MSDL-Link-Status`
|
||||
(`fresh` / `cached` / `stale`) and `X-MSDL-Link-Expires` headers. A stale (WAF-blocked
|
||||
refresh) response now shows an explicit warning plus an `OfficialFallback` component
|
||||
(data-driven from `products.json`, links to Microsoft's own download page) instead of
|
||||
silently handing out a possibly-dead link.
|
||||
- **CLI handoff UI** — `CliHandoff` component on the product page offers the install
|
||||
one-liner + pre-filled `msdl` command as a guaranteed-fresh alternative when the cached
|
||||
link is stale.
|
||||
- **CLI telemetry** — `GET /cli/version` (update check) and `POST /telemetry` (anonymous
|
||||
action/platform/version counts, no personal data) endpoints; opt out with
|
||||
`--no-telemetry` / `MSDL_NO_TELEMETRY=1`. Counts surfaced in `/metrics`.
|
||||
- **Catalog cleanup** — removed product `48` (Windows 8.1 Single Language), which
|
||||
Microsoft had already pulled server-side (`ERROR [502]: no download links found`).
|
||||
- **Distribution** — GitHub Actions release workflow (4-platform binaries on `cli/vX.Y.Z`
|
||||
tags), winget manifest (`starkSV.msdl`), Homebrew tap (`msdl-cli` formula), AUR package
|
||||
(`msdl-bin`), and a `curl | bash` installer for Linux/macOS without Homebrew.
|
||||
|
||||
---
|
||||
|
||||
### eval ISOs — PR #13 `feat/evalcenter-isos`
|
||||
**Windows Server 2016–2025 + Windows 11 Enterprise evaluation editions**
|
||||
|
||||
|
|
@ -82,4 +165,3 @@ CF Worker ensures Hetzner IP is never exposed to Microsoft — rate-limit block
|
|||
| Item | Notes |
|
||||
|---|---|
|
||||
| Per-IP / per-product rate limiter | Revisit after 1 month of production traffic data |
|
||||
| `_redirects` Cloudflare Pages bug | SPA fallback rule flagged as infinite loop, may cause 404s on direct nav |
|
||||
|
|
|
|||
72
README.md
72
README.md
|
|
@ -58,16 +58,40 @@ msdl --list
|
|||
|
||||
### Install
|
||||
|
||||
Download the latest binary from [GitHub Releases](https://github.com/starkSV/windows-iso-downloader/releases/latest) and rename it:
|
||||
**Windows (winget):**
|
||||
```
|
||||
winget install starkSV.msdl
|
||||
```
|
||||
|
||||
**macOS / Linux (Homebrew):**
|
||||
```bash
|
||||
brew tap starkSV/msdl
|
||||
brew install msdl-cli
|
||||
```
|
||||
(the formula is named `msdl-cli`, not `msdl` — `homebrew/core` already has an unrelated package called `msdl`; the installed command is still just `msdl`)
|
||||
|
||||
**Arch Linux (AUR):**
|
||||
```bash
|
||||
yay -S msdl-bin
|
||||
# or: paru -S msdl-bin
|
||||
```
|
||||
Package: [aur.archlinux.org/packages/msdl-bin](https://aur.archlinux.org/packages/msdl-bin). Source tracked here at [`aur/msdl-bin`](./aur/msdl-bin).
|
||||
|
||||
**macOS / Linux (no Homebrew/AUR):**
|
||||
```bash
|
||||
curl -fsSL https://api.msdl.tech-latest.com/install.sh | bash
|
||||
```
|
||||
Auto-detects OS/arch (including Termux on Android) and installs the latest release.
|
||||
|
||||
**Direct download:** Grab the latest binary from [GitHub Releases](https://github.com/starkSV/windows-iso-downloader/releases/latest) and rename it:
|
||||
|
||||
| Platform | File | Rename to |
|
||||
|---|---|---|
|
||||
| Windows | `msdl-windows-amd64.exe` | `msdl.exe` |
|
||||
| macOS (Apple Silicon) | `msdl-darwin-arm64` | `msdl` |
|
||||
| macOS (Intel) | `msdl-darwin-amd64` | `msdl` |
|
||||
| Linux | `msdl-linux-amd64` | `msdl` |
|
||||
|
||||
winget support is in progress: `winget install starkSV.msdl` (pending package review).
|
||||
| Linux (x86_64) | `msdl-linux-amd64` | `msdl` |
|
||||
| Linux (ARM64, incl. Termux on Android) | `msdl-linux-arm64` | `msdl` |
|
||||
|
||||
### Crowdsourced cache
|
||||
|
||||
|
|
@ -115,8 +139,6 @@ Browser → Backend → (CF Worker) → Microsoft API → Signed CDN URL
|
|||
5. Cache result → return to browser
|
||||
```
|
||||
|
||||
The flow mirrors [Fido.ps1](https://github.com/pbatard/Fido) by Pete Batard — the same script bundled with Rufus.
|
||||
|
||||
Outbound requests to Microsoft are optionally routed through a Cloudflare Worker (`cloudflare-worker/worker.js`). This distributes requests across Cloudflare's global edge IPs instead of a single server IP, preventing Microsoft's rate-limit block (error 715-123130) under high traffic. The Worker is opt-in via environment variables — omit them to go direct to Microsoft.
|
||||
|
||||
### Caching layer
|
||||
|
|
@ -230,17 +252,17 @@ Valid slugs: `server-2025`, `server-2022`, `server-2019`, `server-2016`, `win11-
|
|||
|
||||
### `POST /contribute`
|
||||
|
||||
Accepts a crowdsourced link from the CLI tool to warm the cache. Requires `Authorization: Bearer <CONTRIBUTE_SECRET>` header.
|
||||
Accepts a crowdsourced link from the CLI tool to warm the cache. Requires an `X-Contribute-Secret: <CONTRIBUTE_SECRET>` header.
|
||||
|
||||
```json
|
||||
{
|
||||
"product_id": "3262",
|
||||
"sku_id": "0x0409",
|
||||
"response": { /* raw Microsoft JSON */ }
|
||||
"raw_json": { /* raw Microsoft JSON */ }
|
||||
}
|
||||
```
|
||||
|
||||
Returns `200 OK` on acceptance, `400` on validation failure (expired link, unknown product), `429` on rate limit (~5 req/min per IP).
|
||||
Returns `204 No Content` on acceptance; `422` on validation failure (unknown product, invalid SKU, expiry under 1 hour, disallowed download host); `400` on a malformed request body; `401` if the secret header doesn't match; `429` on rate limit (~5 req/min per IP).
|
||||
|
||||
### `GET /metrics?secret=<secret>`
|
||||
|
||||
|
|
@ -252,10 +274,22 @@ Returns real-time cache statistics for the running instance. Auth via `?secret=`
|
|||
"link": { "requests": 38, "cache_hits": 35, "ms_fetches": 3, "neg_hits": 0, "stale": 0, "hit_rate": "92.1%", "cache_size": 6 },
|
||||
"eval": { "requests": 12, "cache_hits": 12, "stale": 0, "hit_rate": "100.0%", "cache_size": 5 },
|
||||
"neg_cache_size": 0,
|
||||
"total_ms_fetches": 6
|
||||
"total_ms_fetches": 6,
|
||||
"sentinel_errors": 0,
|
||||
"sentinel_distinct_sources_est": 0,
|
||||
"telemetry": {
|
||||
"actions": { "fetch": 364, "eval": 40, "interactive": 14 },
|
||||
"platforms": { "windows": 176, "darwin": 70, "linux": 132 },
|
||||
"products": { "3262": 109, "2618": 95 },
|
||||
"results": { "success": 197, "failed": 221 },
|
||||
"versions": { "0.3.6": 41 },
|
||||
"errors": { "fetching download links: Sentinel marked this request as rejected.": 90 }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`sentinel_errors` / `sentinel_distinct_sources_est` track how often Microsoft's Sentinel WAF has blocked the backend's own outbound Microsoft calls. `telemetry` aggregates anonymous CLI usage events (see [Usage telemetry](#usage-telemetry) below) — omitted entirely if no CLI events have been recorded yet.
|
||||
|
||||
---
|
||||
|
||||
## Supported Products
|
||||
|
|
@ -266,14 +300,13 @@ Returns real-time cache statistics for the running instance. Auth via `?secret=`
|
|||
|---|---|---|
|
||||
| Windows 11 25H2 | 3262 | x64 |
|
||||
| Windows 11 25H2 | 3265 | ARM64 |
|
||||
| Windows 11 25H2 (updated) | 3321 | x64 |
|
||||
| Windows 11 25H2 (updated) | 3324 | ARM64 |
|
||||
| Windows 11 25H2 (V2) | 3321 | x64 |
|
||||
| Windows 11 25H2 (V2) | 3324 | ARM64 |
|
||||
| Windows 11 24H2 | 3113 | x64 |
|
||||
| Windows 11 24H2 | 3131 | ARM64 |
|
||||
| Windows 10 22H2 | 2618 | x64 / x86 |
|
||||
| Windows 10 22H2 Home China | 2378 | x64 |
|
||||
| Windows 8.1 | 52 | x64 / x86 |
|
||||
| Windows 8.1 Single Language | 48 | x64 / x86 |
|
||||
|
||||
### Evaluation editions (Server & Enterprise)
|
||||
|
||||
|
|
@ -343,18 +376,7 @@ Recommended setup:
|
|||
|
||||
## Contributing
|
||||
|
||||
Pull requests welcome. To add a new consumer Windows release:
|
||||
|
||||
1. Find the product ID on `www.microsoft.com/software-download-connector/api/`
|
||||
2. Add it to `frontend/public/data/products.json` (name, archs, badge, related, active)
|
||||
3. Update the product table in this README
|
||||
|
||||
To add a new evaluation edition:
|
||||
|
||||
1. Find the fwlink URL on `microsoft.com/en-us/evalcenter/download-*`
|
||||
2. Add the slug and fwlink to the `evalProducts` map in `backend/main.go`
|
||||
3. Add the product config to `frontend/src/data/evalProducts.ts`
|
||||
4. Update the eval table in this README
|
||||
Pull requests welcome — see [CONTRIBUTING.md](./CONTRIBUTING.md) for dev setup, how to add products/eval editions, and PR conventions.
|
||||
|
||||
---
|
||||
|
||||
|
|
|
|||
16
aur/msdl-bin/.SRCINFO
Normal file
16
aur/msdl-bin/.SRCINFO
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
pkgbase = msdl-bin
|
||||
pkgdesc = Download Windows ISO files directly from Microsoft's servers
|
||||
pkgver = 0.3.7
|
||||
pkgrel = 1
|
||||
url = https://msdl.tech-latest.com
|
||||
arch = x86_64
|
||||
arch = aarch64
|
||||
license = MIT
|
||||
provides = msdl
|
||||
conflicts = msdl
|
||||
source_x86_64 = msdl-bin-0.3.7-x86_64::https://github.com/starkSV/windows-iso-downloader/releases/download/cli%2Fv0.3.7/msdl-linux-amd64
|
||||
sha256sums_x86_64 = 4714f47044814733ca2e20ae8f64327ad0b71b912d6fe6f677f18c90b3a5f1c0
|
||||
source_aarch64 = msdl-bin-0.3.7-aarch64::https://github.com/starkSV/windows-iso-downloader/releases/download/cli%2Fv0.3.7/msdl-linux-arm64
|
||||
sha256sums_aarch64 = 9059678043de3995f67ddc3083badf674b572d9bba8e8118e511f57d1f80d3eb
|
||||
|
||||
pkgname = msdl-bin
|
||||
20
aur/msdl-bin/PKGBUILD
Normal file
20
aur/msdl-bin/PKGBUILD
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
# Maintainer: starkSV <shekharvaidya2@gmail.com>
|
||||
pkgname=msdl-bin
|
||||
pkgver=0.3.7
|
||||
pkgrel=1
|
||||
pkgdesc="Download Windows ISO files directly from Microsoft's servers"
|
||||
arch=('x86_64' 'aarch64')
|
||||
url="https://msdl.tech-latest.com"
|
||||
license=('MIT')
|
||||
provides=('msdl')
|
||||
conflicts=('msdl')
|
||||
|
||||
source_x86_64=("$pkgname-$pkgver-x86_64::https://github.com/starkSV/windows-iso-downloader/releases/download/cli%2Fv${pkgver}/msdl-linux-amd64")
|
||||
sha256sums_x86_64=('4714f47044814733ca2e20ae8f64327ad0b71b912d6fe6f677f18c90b3a5f1c0')
|
||||
|
||||
source_aarch64=("$pkgname-$pkgver-aarch64::https://github.com/starkSV/windows-iso-downloader/releases/download/cli%2Fv${pkgver}/msdl-linux-arm64")
|
||||
sha256sums_aarch64=('9059678043de3995f67ddc3083badf674b572d9bba8e8118e511f57d1f80d3eb')
|
||||
|
||||
package() {
|
||||
install -Dm755 "$srcdir/$pkgname-$pkgver-$CARCH" "$pkgdir/usr/bin/msdl"
|
||||
}
|
||||
31
aur/msdl-bin/README.md
Normal file
31
aur/msdl-bin/README.md
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
# msdl-bin (AUR)
|
||||
|
||||
> **✅ Registration reopened as of 2026-07-31:** Arch Linux disabled new AUR
|
||||
> account registrations on 2026-06-15 after a malware campaign compromised
|
||||
> 1,500+ AUR packages across several waves; registration is open again.
|
||||
> This package (`PKGBUILD` + `.SRCINFO` below) is ready to publish — see
|
||||
> "Publishing" below once an account + SSH key are set up at
|
||||
> [aur.archlinux.org](https://aur.archlinux.org/register/).
|
||||
|
||||
`PKGBUILD` and `.SRCINFO` for the [msdl-bin](https://aur.archlinux.org/packages/msdl-bin) AUR package, tracked here so version bumps have the same history/review as the winget manifests.
|
||||
|
||||
`.SRCINFO` is generated with the real `makepkg` tool (via an ephemeral `archlinux` Docker container — no VM or native Arch install needed) whenever `PKGBUILD` changes:
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$(pwd)":/pkg -w /pkg archlinux:latest bash -c "
|
||||
useradd -m builder && chown -R builder:builder /pkg &&
|
||||
pacman -Sy --noconfirm --needed base-devel &&
|
||||
su builder -c 'makepkg --printsrcinfo' > .SRCINFO
|
||||
"
|
||||
```
|
||||
|
||||
Publishing (requires an AUR account with an SSH key registered at aur.archlinux.org — this step can't be automated on someone else's behalf):
|
||||
|
||||
```bash
|
||||
git clone ssh://aur@aur.archlinux.org/msdl-bin.git
|
||||
cp PKGBUILD .SRCINFO msdl-bin/
|
||||
cd msdl-bin
|
||||
git add PKGBUILD .SRCINFO
|
||||
git commit -m "Update to <version>"
|
||||
git push
|
||||
```
|
||||
48
backend/install.sh
Normal file
48
backend/install.sh
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
#!/usr/bin/env bash
|
||||
# Installs the msdl CLI. Usage: curl -fsSL https://api.msdl.tech-latest.com/install.sh | bash
|
||||
set -euo pipefail
|
||||
|
||||
REPO="starkSV/windows-iso-downloader"
|
||||
|
||||
os="$(uname -s)"
|
||||
arch="$(uname -m)"
|
||||
|
||||
case "$os" in
|
||||
Linux) platform="linux" ;;
|
||||
Darwin) platform="darwin" ;;
|
||||
*)
|
||||
echo "msdl: unsupported OS: $os (Windows users: winget install starkSV.msdl)" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
case "$arch" in
|
||||
x86_64|amd64) goarch="amd64" ;;
|
||||
aarch64|arm64) goarch="arm64" ;;
|
||||
*)
|
||||
echo "msdl: unsupported architecture: $arch" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
asset="msdl-${platform}-${goarch}"
|
||||
url="https://github.com/${REPO}/releases/latest/download/${asset}"
|
||||
|
||||
tmp="$(mktemp)"
|
||||
echo "Downloading ${asset}..."
|
||||
curl -fsSL "$url" -o "$tmp"
|
||||
chmod +x "$tmp"
|
||||
|
||||
if [ -n "${PREFIX:-}" ] && [ -d "${PREFIX}/bin" ]; then
|
||||
# Termux (or similar $PREFIX-based environment) -- no sudo available or needed
|
||||
mv "$tmp" "${PREFIX}/bin/msdl"
|
||||
echo "Installed to ${PREFIX}/bin/msdl"
|
||||
elif [ -w /usr/local/bin ]; then
|
||||
mv "$tmp" /usr/local/bin/msdl
|
||||
echo "Installed to /usr/local/bin/msdl"
|
||||
else
|
||||
sudo mv "$tmp" /usr/local/bin/msdl
|
||||
echo "Installed to /usr/local/bin/msdl"
|
||||
fi
|
||||
|
||||
echo "Run 'msdl --help' to get started."
|
||||
|
|
@ -3,6 +3,7 @@ package main
|
|||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"html"
|
||||
|
|
@ -27,6 +28,9 @@ import (
|
|||
"golang.org/x/sync/singleflight"
|
||||
)
|
||||
|
||||
//go:embed install.sh
|
||||
var installScript []byte
|
||||
|
||||
// Package-level rand source — avoids global mutex contention under concurrent load.
|
||||
// Go 1.20+ auto-seeds the global rand, but a local source is faster at high QPS.
|
||||
var rng = rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
|
|
@ -39,7 +43,7 @@ const (
|
|||
CUSTOMER_ID = "560dc9f3-1aa5-4a2f-b63c-9e18f8d0e175"
|
||||
PORT = ":3002"
|
||||
|
||||
latestCLIVersion = "0.3.1"
|
||||
latestCLIVersion = "0.3.7"
|
||||
)
|
||||
|
||||
// --- Session cache (short-lived, used to chain /skuinfo → /proxy) ---
|
||||
|
|
@ -128,7 +132,7 @@ func truncateBody(s string) string {
|
|||
return s
|
||||
}
|
||||
|
||||
// --- Negative cache (60s normal / 90min Sentinel lockdown) ---
|
||||
// --- Negative cache (60s normal / 5h Sentinel lockdown) ---
|
||||
|
||||
type negCacheEntry struct {
|
||||
Message string
|
||||
|
|
@ -141,7 +145,10 @@ var (
|
|||
negCache = make(map[string]negCacheEntry)
|
||||
negCacheMu sync.RWMutex
|
||||
negCacheTTL = 60 * time.Second
|
||||
lockdownTTL = 90 * time.Minute
|
||||
// Bumped from 90min on 2026-07-31: 181 retry attempts over 17 days, 0 successes.
|
||||
// Retries yield zero value right now, so back off harder instead of hammering
|
||||
// Microsoft (and our own IP's reputation) every 90 minutes for nothing.
|
||||
lockdownTTL = 5 * time.Hour
|
||||
)
|
||||
|
||||
// --- Singleflight (one in-flight Microsoft fetch per unique key) ---
|
||||
|
|
@ -475,7 +482,8 @@ var validContributeProducts = map[string]bool{
|
|||
"3131": true, "3132": true, "3133": true,
|
||||
"3262": true, "3263": true, "3264": true,
|
||||
"3265": true, "3266": true, "3267": true,
|
||||
"3321": true, "3324": true,
|
||||
"3321": true, "3322": true, "3323": true,
|
||||
"3324": true, "3325": true, "3326": true,
|
||||
}
|
||||
|
||||
// allowedCDNSuffixes is the CDN host allow-list for contributed download URLs.
|
||||
|
|
@ -937,7 +945,6 @@ func fetchDownloadLinksFromMS(productID, skuID string) ([]byte, error) {
|
|||
jar = &simpleCookieJar{}
|
||||
}
|
||||
|
||||
|
||||
proxyQ := url.Values{}
|
||||
proxyQ.Set("profile", PROFILE)
|
||||
proxyQ.Set("productEditionId", "undefined")
|
||||
|
|
@ -1180,6 +1187,10 @@ func handleProxy(w http.ResponseWriter, r *http.Request) {
|
|||
respondJSONError(w, http.StatusBadRequest, "sku_id contains invalid characters")
|
||||
return
|
||||
}
|
||||
if !validContributeProducts[productID] {
|
||||
respondJSONError(w, http.StatusNotFound, "unknown product_id")
|
||||
return
|
||||
}
|
||||
|
||||
atomic.AddInt64(&mLinkRequests, 1)
|
||||
cacheKey := productID + ":" + skuID
|
||||
|
|
@ -1199,10 +1210,20 @@ func handleProxy(w http.ResponseWriter, r *http.Request) {
|
|||
staleEntry, hasStale := linkCache[cacheKey]
|
||||
linkCacheMu.RUnlock()
|
||||
if hasStale {
|
||||
atomic.AddInt64(&mLinkStale, 1)
|
||||
log.Printf("/proxy: product_id=%s sku_id=%s -> lockdown active, serving stale (expires %s)\n",
|
||||
productID, skuID, lockdownEntry.ExpiresAt.Format(time.RFC3339))
|
||||
w.Header().Set("X-MSDL-Link-Status", "stale")
|
||||
if time.Now().Before(staleEntry.ExpiresAt) {
|
||||
// Entry is still within its normal TTL (e.g. just contributed by
|
||||
// a CLI run) — genuinely fresh, not actually stale, just being
|
||||
// served via the lockdown gate before the usual cache check.
|
||||
atomic.AddInt64(&mLinkCacheHits, 1)
|
||||
log.Printf("/proxy: product_id=%s sku_id=%s -> lockdown active, serving cached (expires %s)\n",
|
||||
productID, skuID, staleEntry.ExpiresAt.Format(time.RFC3339))
|
||||
w.Header().Set("X-MSDL-Link-Status", "cached")
|
||||
} else {
|
||||
atomic.AddInt64(&mLinkStale, 1)
|
||||
log.Printf("/proxy: product_id=%s sku_id=%s -> lockdown active, serving stale (expires %s)\n",
|
||||
productID, skuID, lockdownEntry.ExpiresAt.Format(time.RFC3339))
|
||||
w.Header().Set("X-MSDL-Link-Status", "stale")
|
||||
}
|
||||
if exp := extractRawExpiry(staleEntry.RawJSON); exp != "" {
|
||||
w.Header().Set("X-MSDL-Link-Expires", exp)
|
||||
}
|
||||
|
|
@ -1287,7 +1308,7 @@ func handleProxy(w http.ResponseWriter, r *http.Request) {
|
|||
atomic.AddInt64(&mLinkStale, 1)
|
||||
|
||||
if isSentinel {
|
||||
// Sentinel lockdown: set 90-min neg cache, add lockdown header, skip
|
||||
// Sentinel lockdown: set lockdownTTL neg cache, add lockdown header, skip
|
||||
// background refresh — retrying extends the Sentinel block timer.
|
||||
negCacheMu.Lock()
|
||||
negCache[negKey] = negCacheEntry{
|
||||
|
|
@ -1297,7 +1318,7 @@ func handleProxy(w http.ResponseWriter, r *http.Request) {
|
|||
IsSentinel: true,
|
||||
}
|
||||
negCacheMu.Unlock()
|
||||
log.Printf("/proxy: product_id=%s sku_id=%s -> Sentinel lockdown (90min), serving stale\n", productID, skuID)
|
||||
log.Printf("/proxy: product_id=%s sku_id=%s -> Sentinel lockdown (%s), serving stale\n", productID, skuID, lockdownTTL)
|
||||
w.Header().Set("X-MSDL-Lockdown", "1")
|
||||
w.Header().Set("X-MSDL-Link-Status", "stale")
|
||||
if exp := extractRawExpiry(cached.RawJSON); exp != "" {
|
||||
|
|
@ -1351,7 +1372,7 @@ func handleProxy(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
negCacheMu.Unlock()
|
||||
if isSentinel {
|
||||
log.Printf("/proxy: product_id=%s sku_id=%s -> Sentinel lockdown (90min), no stale\n", productID, skuID)
|
||||
log.Printf("/proxy: product_id=%s sku_id=%s -> Sentinel lockdown (%s), no stale\n", productID, skuID, lockdownTTL)
|
||||
w.Header().Set("X-MSDL-Lockdown", "1")
|
||||
w.WriteHeader(code)
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
|
|
@ -1665,6 +1686,15 @@ func handleTelemetry(w http.ResponseWriter, r *http.Request) {
|
|||
rdb.HIncrBy(ctx, "msdl:telemetry:results", result, 1)
|
||||
if !p.Success && p.Error != "" && len(p.Error) <= 200 {
|
||||
rdb.HIncrBy(ctx, "msdl:telemetry:errors", p.Error, 1)
|
||||
if strings.Contains(p.Error, "Sentinel") {
|
||||
// All-time Sentinel-rejection count, plus an approximate
|
||||
// distinct-source count (via HyperLogLog on the reporting
|
||||
// IP) so /metrics can show whether rejections are
|
||||
// concentrated in a few sources or spread across many,
|
||||
// without ever storing raw IPs.
|
||||
rdb.Incr(ctx, "msdl:telemetry:sentinel_count")
|
||||
rdb.PFAdd(ctx, "msdl:telemetry:sentinel_hll", ip)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1683,6 +1713,20 @@ func handleCLIVersion(w http.ResponseWriter, r *http.Request) {
|
|||
json.NewEncoder(w).Encode(map[string]string{"latest": latestCLIVersion})
|
||||
}
|
||||
|
||||
// --- /install.sh endpoint ---
|
||||
|
||||
// handleInstallScript serves the Linux/macOS install script embedded at
|
||||
// build time from install.sh. Auto-detects OS/arch and always resolves the
|
||||
// latest GitHub release, so it needs no per-release maintenance.
|
||||
func handleInstallScript(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/x-sh; charset=utf-8")
|
||||
w.Write(installScript)
|
||||
}
|
||||
|
||||
// --- /metrics endpoint ---
|
||||
|
||||
// loadTelemetryFromRedis reads all telemetry counters from Redis.
|
||||
|
|
@ -1765,6 +1809,7 @@ func handleMetrics(w http.ResponseWriter, r *http.Request) {
|
|||
evalCacheMu.RUnlock()
|
||||
|
||||
telemetry := loadTelemetryFromRedis()
|
||||
sentinelErrors, sentinelDistinctSources := loadSentinelStatsFromRedis()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
|
|
@ -1792,12 +1837,28 @@ func handleMetrics(w http.ResponseWriter, r *http.Request) {
|
|||
"hit_rate": hitRate(evalHits, evalReqs),
|
||||
"cache_size": evalSize,
|
||||
},
|
||||
"neg_cache_size": negSize,
|
||||
"total_ms_fetches": skuFetches + linkFetches,
|
||||
"telemetry": telemetry,
|
||||
"neg_cache_size": negSize,
|
||||
"total_ms_fetches": skuFetches + linkFetches,
|
||||
"telemetry": telemetry,
|
||||
"sentinel_errors": sentinelErrors,
|
||||
"sentinel_distinct_sources_est": sentinelDistinctSources,
|
||||
})
|
||||
}
|
||||
|
||||
// loadSentinelStatsFromRedis returns the all-time Sentinel-rejection count
|
||||
// and an approximate distinct-source count (from the HyperLogLog seeded in
|
||||
// handleTelemetry). Returns (0, 0) when Redis is unavailable.
|
||||
func loadSentinelStatsFromRedis() (int64, int64) {
|
||||
if rdb == nil {
|
||||
return 0, 0
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
|
||||
defer cancel()
|
||||
count, _ := rdb.Get(ctx, "msdl:telemetry:sentinel_count").Int64()
|
||||
distinct, _ := rdb.PFCount(ctx, "msdl:telemetry:sentinel_hll").Result()
|
||||
return count, distinct
|
||||
}
|
||||
|
||||
func warmEvalCache() {
|
||||
for slug, product := range evalProductMap {
|
||||
go func(s string, p EvalProduct) {
|
||||
|
|
@ -1830,6 +1891,7 @@ func main() {
|
|||
mux.HandleFunc("/metrics", handleMetrics)
|
||||
mux.HandleFunc("/telemetry", handleTelemetry)
|
||||
mux.HandleFunc("/cli/version", handleCLIVersion)
|
||||
mux.HandleFunc("/install.sh", handleInstallScript)
|
||||
mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(`{"status":"ok"}`))
|
||||
|
|
|
|||
|
|
@ -29,8 +29,12 @@ var consumerProducts = []Product{
|
|||
{"3265", "Windows 11 Arm64 25H2 (26200.6584)"},
|
||||
{"3266", "Windows 11 Arm64 25H2 Home China (26200.6584)"},
|
||||
{"3267", "Windows 11 Arm64 25H2 Pro China (26200.6584)"},
|
||||
{"3321", "Windows 11 25H2 (Updated Oct)"},
|
||||
{"3324", "Windows 11 Arm64 25H2 (Updated Oct)"},
|
||||
{"3321", "Windows 11 25H2 (V2)"},
|
||||
{"3322", "Windows 11 25H2 Home China (V2)"},
|
||||
{"3323", "Windows 11 25H2 Pro China (V2)"},
|
||||
{"3324", "Windows 11 Arm64 25H2 (V2)"},
|
||||
{"3325", "Windows 11 Arm64 25H2 Home China (V2)"},
|
||||
{"3326", "Windows 11 Arm64 25H2 Pro China (V2)"},
|
||||
}
|
||||
|
||||
var evalProducts = []EvalProduct{
|
||||
|
|
@ -60,7 +64,7 @@ func searchProducts(query string) []Product {
|
|||
name := strings.ToLower(p.Name)
|
||||
match := true
|
||||
for _, w := range words {
|
||||
if !strings.Contains(name, w) {
|
||||
if !containsWordStart(name, w) {
|
||||
match = false
|
||||
break
|
||||
}
|
||||
|
|
@ -72,6 +76,31 @@ func searchProducts(query string) []Product {
|
|||
return results
|
||||
}
|
||||
|
||||
// containsWordStart reports whether s contains substr starting at a word
|
||||
// boundary: the start of s, or right after a non-alphanumeric character.
|
||||
// Plain strings.Contains would let a query word match a digit fragment
|
||||
// buried inside an unrelated number -- e.g. "10" matching inside a build
|
||||
// number like "26100.1742" -- which made "windows 10" incorrectly return
|
||||
// Windows 11 24H2 results.
|
||||
func containsWordStart(s, substr string) bool {
|
||||
from := 0
|
||||
for {
|
||||
i := strings.Index(s[from:], substr)
|
||||
if i < 0 {
|
||||
return false
|
||||
}
|
||||
pos := from + i
|
||||
if pos == 0 || !isAlphanumeric(s[pos-1]) {
|
||||
return true
|
||||
}
|
||||
from = pos + 1
|
||||
}
|
||||
}
|
||||
|
||||
func isAlphanumeric(b byte) bool {
|
||||
return (b >= 'a' && b <= 'z') || (b >= '0' && b <= '9')
|
||||
}
|
||||
|
||||
func findEvalProduct(slug string) (EvalProduct, bool) {
|
||||
for _, p := range evalProducts {
|
||||
if p.Slug == slug {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
package main
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFindProductByID_found(t *testing.T) {
|
||||
p, ok := findProductByID("3262")
|
||||
|
|
@ -40,6 +43,35 @@ func TestSearchProducts_caseInsensitive(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestSearchProducts_noDigitFragmentCollision(t *testing.T) {
|
||||
// "10" must not match inside the "26100.1742" build number of Windows 11
|
||||
// 24H2/25H2 products -- regression test for the reported "windows 10"
|
||||
// bug that returned Windows 11 results.
|
||||
results := searchProducts("windows 10")
|
||||
for _, p := range results {
|
||||
if strings.Contains(p.Name, "11") {
|
||||
t.Errorf("query %q matched %q, a Windows 11 product", "windows 10", p.Name)
|
||||
}
|
||||
}
|
||||
if len(results) == 0 {
|
||||
t.Fatal("expected windows 10 to match the actual Windows 10 products")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchProducts_prefixSubstringStillWorks(t *testing.T) {
|
||||
// "arm" should still fuzzy-match "ARM64" -- the word-boundary fix must
|
||||
// only reject matches that start mid-digit-run, not all substrings.
|
||||
results := searchProducts("arm")
|
||||
if len(results) == 0 {
|
||||
t.Fatal("expected arm to match ARM64 products")
|
||||
}
|
||||
for _, p := range results {
|
||||
if !strings.Contains(strings.ToLower(p.Name), "arm") {
|
||||
t.Errorf("unexpected match %q for query %q", p.Name, "arm")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindEvalProduct_found(t *testing.T) {
|
||||
p, ok := findEvalProduct("server-2025")
|
||||
if !ok {
|
||||
|
|
|
|||
23
cli/go.mod
23
cli/go.mod
|
|
@ -1,3 +1,24 @@
|
|||
module github.com/starkSV/msdl-cli
|
||||
|
||||
go 1.21
|
||||
go 1.24.1
|
||||
|
||||
require (
|
||||
github.com/bogdanfinn/fhttp v0.6.8
|
||||
github.com/bogdanfinn/tls-client v1.15.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||
github.com/bdandy/go-errors v1.2.2 // indirect
|
||||
github.com/bdandy/go-socks4 v1.2.3 // indirect
|
||||
github.com/bogdanfinn/quic-go-utls v1.0.9-utls // indirect
|
||||
github.com/bogdanfinn/utls v1.7.7-barnius // indirect
|
||||
github.com/bogdanfinn/websocket v1.5.5-barnius // indirect
|
||||
github.com/klauspost/compress v1.18.2 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 // indirect
|
||||
golang.org/x/crypto v0.46.0 // indirect
|
||||
golang.org/x/net v0.48.0 // indirect
|
||||
golang.org/x/sys v0.39.0 // indirect
|
||||
golang.org/x/text v0.32.0 // indirect
|
||||
)
|
||||
|
|
|
|||
48
cli/go.sum
Normal file
48
cli/go.sum
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
|
||||
github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/bdandy/go-errors v1.2.2 h1:WdFv/oukjTJCLa79UfkGmwX7ZxONAihKu4V0mLIs11Q=
|
||||
github.com/bdandy/go-errors v1.2.2/go.mod h1:NkYHl4Fey9oRRdbB1CoC6e84tuqQHiqrOcZpqFEkBxM=
|
||||
github.com/bdandy/go-socks4 v1.2.3 h1:Q6Y2heY1GRjCtHbmlKfnwrKVU/k81LS8mRGLRlmDlic=
|
||||
github.com/bdandy/go-socks4 v1.2.3/go.mod h1:98kiVFgpdogR8aIGLWLvjDVZ8XcKPsSI/ypGrO+bqHI=
|
||||
github.com/bogdanfinn/fhttp v0.6.8 h1:LiQyHOY3i0QoxxNB7nq27/nGNNbtPj0fuBPozhR7Ws4=
|
||||
github.com/bogdanfinn/fhttp v0.6.8/go.mod h1:A+EKDzMx2hb4IUbMx4TlkoHnaJEiLl8r/1Ss1Y+5e5M=
|
||||
github.com/bogdanfinn/quic-go-utls v1.0.9-utls h1:tV6eDEiRbRCcepALSzxR94JUVD3N3ACIiRLgyc2Ep8s=
|
||||
github.com/bogdanfinn/quic-go-utls v1.0.9-utls/go.mod h1:aHph9B9H9yPOt5xnhWKSOum27DJAqpiHzwX+gjvaXcg=
|
||||
github.com/bogdanfinn/tls-client v1.15.1 h1:KiFAlED55DJ8Fcocn+/1nX6PrDFcttIHAf/GDkV6KN8=
|
||||
github.com/bogdanfinn/tls-client v1.15.1/go.mod h1:LsU6mXVn8MOFDwTkyRfI7V1BZM1p0wf2ZfZsICW/1fM=
|
||||
github.com/bogdanfinn/utls v1.7.7-barnius h1:OuJ497cc7F3yKNVHRsYPQdGggmk5x6+V5ZlrCR7fOLU=
|
||||
github.com/bogdanfinn/utls v1.7.7-barnius/go.mod h1:aAK1VZQlpKZClF1WEQeq6kyclbkPq4hz6xTbB5xSlmg=
|
||||
github.com/bogdanfinn/websocket v1.5.5-barnius h1:bY+qnxpai1qe7Jmjx+Sds/cmOSpuuLoR8x61rWltjOI=
|
||||
github.com/bogdanfinn/websocket v1.5.5-barnius/go.mod h1:gvvEw6pTKHb7yOiFvIfAFTStQWyrm25BMVCTj5wRSsI=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
|
||||
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
|
||||
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 h1:YqAladjX7xpA6BM04leXMWAEjS0mTZ5kUU9KRBriQJc=
|
||||
github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5/go.mod h1:2JjD2zLQYH5HO74y5+aE3remJQvl6q4Sn6aWA2wD1Ng=
|
||||
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
|
||||
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
||||
go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko=
|
||||
go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o=
|
||||
golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
|
||||
golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
|
||||
golang.org/x/net v0.0.0-20211104170005-ce137452f963/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
|
||||
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
|
||||
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
|
||||
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
114
cli/homepage.go
Normal file
114
cli/homepage.go
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// homepageFeaturedIDs are the consumer products shown on the bare `msdl`
|
||||
// landing screen, picked from real usage telemetry rather than the full
|
||||
// catalog. The full catalog is still one step away via "list".
|
||||
var homepageFeaturedIDs = []string{"3262", "2618", "3321", "3113", "3265"}
|
||||
|
||||
// homepageEvalSlugs are the eval products shown on the landing screen.
|
||||
var homepageEvalSlugs = []string{"server-2025", "win11-ent"}
|
||||
|
||||
type homepageKind int
|
||||
|
||||
const (
|
||||
homepageKindProduct homepageKind = iota
|
||||
homepageKindEval
|
||||
homepageKindList
|
||||
homepageKindSearch
|
||||
)
|
||||
|
||||
type homepageChoice struct {
|
||||
kind homepageKind
|
||||
productID string
|
||||
evalSlug string
|
||||
query string
|
||||
}
|
||||
|
||||
const homepageRule = "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
// showHomepage prints the landing screen for a bare `msdl` invocation and
|
||||
// reads the user's choice: a number, "list" for the full catalog, or free
|
||||
// text to search by name.
|
||||
func showHomepage(latestVersion string) (homepageChoice, error) {
|
||||
fmt.Fprintln(os.Stderr, homepageRule)
|
||||
fmt.Fprintln(os.Stderr, "msdl — Windows ISO Downloader")
|
||||
fmt.Fprintln(os.Stderr, "Direct links from Microsoft's CDN. No browser, no Media Creation Tool.")
|
||||
fmt.Fprintln(os.Stderr, homepageRule)
|
||||
fmt.Fprintln(os.Stderr)
|
||||
|
||||
fmt.Fprintln(os.Stderr, "Popular:")
|
||||
featured := make([]Product, 0, len(homepageFeaturedIDs))
|
||||
for _, id := range homepageFeaturedIDs {
|
||||
if p, ok := findProductByID(id); ok {
|
||||
featured = append(featured, p)
|
||||
}
|
||||
}
|
||||
for i, p := range featured {
|
||||
fmt.Fprintf(os.Stderr, "%d. %s\n", i+1, p.Name)
|
||||
}
|
||||
fmt.Fprintln(os.Stderr)
|
||||
|
||||
fmt.Fprintln(os.Stderr, "Evaluation / Enterprise:")
|
||||
evalShortlist := make([]EvalProduct, 0, len(homepageEvalSlugs))
|
||||
for _, slug := range homepageEvalSlugs {
|
||||
if ep, ok := findEvalProduct(slug); ok {
|
||||
evalShortlist = append(evalShortlist, ep)
|
||||
}
|
||||
}
|
||||
for i, ep := range evalShortlist {
|
||||
fmt.Fprintf(os.Stderr, "%d. %s\n", len(featured)+i+1, ep.Name)
|
||||
}
|
||||
fmt.Fprintln(os.Stderr)
|
||||
|
||||
fmt.Fprintln(os.Stderr, `Commands: --list (full catalog) · --help`)
|
||||
fmt.Fprintln(os.Stderr)
|
||||
fmt.Fprintln(os.Stderr, homepageRule)
|
||||
if latestVersion != "" && latestVersion != Version && Version != "dev" {
|
||||
fmt.Fprintf(os.Stderr, "Version %s (Update Available %s) - a TechLatest Open-source Contribution\n", Version, latestVersion)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "Version %s - a TechLatest Open-source Contribution\n", Version)
|
||||
}
|
||||
fmt.Fprintln(os.Stderr)
|
||||
|
||||
max := len(featured) + len(evalShortlist)
|
||||
fmt.Fprintf(os.Stderr, `Choice [1-%d], search by name, or "list": `, max)
|
||||
|
||||
scanner := bufio.NewScanner(os.Stdin)
|
||||
if !scanner.Scan() {
|
||||
if err := scanner.Err(); err != nil {
|
||||
return homepageChoice{}, fmt.Errorf("reading input: %w", err)
|
||||
}
|
||||
return homepageChoice{}, fmt.Errorf("no input")
|
||||
}
|
||||
return parseHomepageInput(scanner.Text(), featured, evalShortlist)
|
||||
}
|
||||
|
||||
// parseHomepageInput interprets the raw input line from the homepage prompt.
|
||||
func parseHomepageInput(input string, featured []Product, evalShortlist []EvalProduct) (homepageChoice, error) {
|
||||
text := strings.TrimSpace(input)
|
||||
if text == "" {
|
||||
return homepageChoice{}, fmt.Errorf(`enter a choice, search term, or "list"`)
|
||||
}
|
||||
if strings.EqualFold(text, "list") {
|
||||
return homepageChoice{kind: homepageKindList}, nil
|
||||
}
|
||||
if n, err := strconv.Atoi(text); err == nil {
|
||||
max := len(featured) + len(evalShortlist)
|
||||
if n < 1 || n > max {
|
||||
return homepageChoice{}, fmt.Errorf(`enter a number between 1 and %d, a search term, or "list"`, max)
|
||||
}
|
||||
if n <= len(featured) {
|
||||
return homepageChoice{kind: homepageKindProduct, productID: featured[n-1].ID}, nil
|
||||
}
|
||||
return homepageChoice{kind: homepageKindEval, evalSlug: evalShortlist[n-len(featured)-1].Slug}, nil
|
||||
}
|
||||
return homepageChoice{kind: homepageKindSearch, query: text}, nil
|
||||
}
|
||||
166
cli/homepage_test.go
Normal file
166
cli/homepage_test.go
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func testFeatured() []Product {
|
||||
return []Product{
|
||||
{ID: "3262", Name: "Windows 11 25H2 (26200.6584)"},
|
||||
{ID: "2618", Name: "Windows 10 22H2 v1 (19045.2965)"},
|
||||
}
|
||||
}
|
||||
|
||||
func testEvalShortlist() []EvalProduct {
|
||||
return []EvalProduct{
|
||||
{Slug: "server-2025", Name: "Windows Server 2025"},
|
||||
{Slug: "win11-ent", Name: "Windows 11 Enterprise"},
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHomepageInput_product(t *testing.T) {
|
||||
choice, err := parseHomepageInput("1", testFeatured(), testEvalShortlist())
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if choice.kind != homepageKindProduct || choice.productID != "3262" {
|
||||
t.Errorf("got %+v, want product 3262", choice)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHomepageInput_eval(t *testing.T) {
|
||||
choice, err := parseHomepageInput("3", testFeatured(), testEvalShortlist())
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if choice.kind != homepageKindEval || choice.evalSlug != "server-2025" {
|
||||
t.Errorf("got %+v, want eval server-2025", choice)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHomepageInput_evalSecond(t *testing.T) {
|
||||
choice, err := parseHomepageInput("4", testFeatured(), testEvalShortlist())
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if choice.kind != homepageKindEval || choice.evalSlug != "win11-ent" {
|
||||
t.Errorf("got %+v, want eval win11-ent", choice)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHomepageInput_outOfRange(t *testing.T) {
|
||||
_, err := parseHomepageInput("5", testFeatured(), testEvalShortlist())
|
||||
if err == nil {
|
||||
t.Fatal("expected error for out-of-range choice")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHomepageInput_listCaseInsensitive(t *testing.T) {
|
||||
for _, s := range []string{"list", "List", "LIST"} {
|
||||
choice, err := parseHomepageInput(s, testFeatured(), testEvalShortlist())
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error for %q: %v", s, err)
|
||||
}
|
||||
if choice.kind != homepageKindList {
|
||||
t.Errorf("input %q: got kind %v, want list", s, choice.kind)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHomepageInput_searchFallback(t *testing.T) {
|
||||
choice, err := parseHomepageInput("windows 10", testFeatured(), testEvalShortlist())
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if choice.kind != homepageKindSearch || choice.query != "windows 10" {
|
||||
t.Errorf("got %+v, want search query \"windows 10\"", choice)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHomepageInput_emptyInput(t *testing.T) {
|
||||
_, err := parseHomepageInput(" ", testFeatured(), testEvalShortlist())
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty input")
|
||||
}
|
||||
}
|
||||
|
||||
// captureHomepage runs showHomepage with stdin/stderr redirected to pipes so
|
||||
// the rendered output and the parsed choice can both be inspected without a
|
||||
// real terminal.
|
||||
func captureHomepage(t *testing.T, stdin string, latestVersion string) (homepageChoice, string) {
|
||||
t.Helper()
|
||||
oldStdin, oldStderr := os.Stdin, os.Stderr
|
||||
defer func() { os.Stdin, os.Stderr = oldStdin, oldStderr }()
|
||||
|
||||
inR, inW, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
os.Stdin = inR
|
||||
inW.WriteString(stdin)
|
||||
inW.Close()
|
||||
|
||||
outR, outW, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
os.Stderr = outW
|
||||
|
||||
choice, hpErr := showHomepage(latestVersion)
|
||||
outW.Close()
|
||||
|
||||
var buf bytes.Buffer
|
||||
io.Copy(&buf, outR)
|
||||
if hpErr != nil {
|
||||
t.Fatalf("unexpected error: %v", hpErr)
|
||||
}
|
||||
return choice, buf.String()
|
||||
}
|
||||
|
||||
func TestShowHomepage_rendersFeaturedAndEval(t *testing.T) {
|
||||
choice, output := captureHomepage(t, "1\n", "")
|
||||
if choice.kind != homepageKindProduct || choice.productID != "3262" {
|
||||
t.Errorf("got %+v, want product 3262", choice)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"1. Windows 11 25H2 (26200.6584)",
|
||||
"2. Windows 10 22H2 v1 (19045.2965)",
|
||||
"6. Windows Server 2025",
|
||||
"7. Windows 11 Enterprise",
|
||||
`Commands: --list (full catalog) · --help`,
|
||||
`Choice [1-7], search by name, or "list":`,
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Errorf("output missing %q, got:\n%s", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestShowHomepage_updateAvailableFooter(t *testing.T) {
|
||||
oldVersion := Version
|
||||
Version = "0.3.3"
|
||||
defer func() { Version = oldVersion }()
|
||||
|
||||
_, output := captureHomepage(t, "1\n", "0.3.4")
|
||||
if !strings.Contains(output, "Version 0.3.3 (Update Available 0.3.4)") {
|
||||
t.Errorf("output missing update-available footer, got:\n%s", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShowHomepage_noUpdateFooter(t *testing.T) {
|
||||
oldVersion := Version
|
||||
Version = "0.3.4"
|
||||
defer func() { Version = oldVersion }()
|
||||
|
||||
_, output := captureHomepage(t, "1\n", "0.3.4")
|
||||
if strings.Contains(output, "Update Available") {
|
||||
t.Errorf("footer should not mention an update when already current, got:\n%s", output)
|
||||
}
|
||||
if !strings.Contains(output, "Version 0.3.4 - a TechLatest Open-source Contribution") {
|
||||
t.Errorf("output missing plain version footer, got:\n%s", output)
|
||||
}
|
||||
}
|
||||
100
cli/main.go
100
cli/main.go
|
|
@ -46,9 +46,9 @@ func contributeURL() string {
|
|||
return apiBaseURL() + "/contribute"
|
||||
}
|
||||
|
||||
// printUpdateNotice checks /cli/version and prints a notice if a newer version
|
||||
// is available. Blocks up to 500ms; silently no-ops on timeout or any error.
|
||||
func printUpdateNotice() {
|
||||
// fetchLatestVersion fetches /cli/version and returns the latest version
|
||||
// string, or "" on any error or after a 500ms timeout.
|
||||
func fetchLatestVersion() string {
|
||||
ch := make(chan string, 1)
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
|
||||
|
|
@ -72,11 +72,18 @@ func printUpdateNotice() {
|
|||
}()
|
||||
select {
|
||||
case latest := <-ch:
|
||||
if latest != "" && latest != Version && Version != "dev" {
|
||||
fmt.Fprintf(os.Stderr, "\n A new version of msdl is available: v%s\n", latest)
|
||||
fmt.Fprintf(os.Stderr, " Download: https://github.com/starkSV/windows-iso-downloader/releases/latest\n\n")
|
||||
}
|
||||
return latest
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// printUpdateNotice prints a notice if latest is newer than the running
|
||||
// binary's version.
|
||||
func printUpdateNotice(latest string) {
|
||||
if latest != "" && latest != Version && Version != "dev" {
|
||||
fmt.Fprintf(os.Stderr, "\n A new version of msdl is available: v%s\n", latest)
|
||||
fmt.Fprintf(os.Stderr, " Download: https://github.com/starkSV/windows-iso-downloader/releases/latest\n\n")
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -91,16 +98,23 @@ type cliTelemetryPayload struct {
|
|||
}
|
||||
|
||||
// truncateError caps an error string for telemetry (avoids leaking huge bodies,
|
||||
// keeps Redis field values small).
|
||||
// keeps Redis field values small). Keeps both head and tail: the head usually
|
||||
// names what failed (e.g. "language X not available for Y"), while wrapped
|
||||
// errors put the root cause last (e.g. "fetching X: <root cause>") — a long
|
||||
// middle section (URLs, option lists) is the least useful part to keep.
|
||||
func truncateError(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
s := err.Error()
|
||||
if len(s) > 150 {
|
||||
s = s[:150]
|
||||
const max = 150
|
||||
if len(s) <= max {
|
||||
return s
|
||||
}
|
||||
return s
|
||||
const headLen = 60
|
||||
const sep = " ... "
|
||||
tailLen := max - headLen - len(sep)
|
||||
return s[:headLen] + sep + s[len(s)-tailLen:]
|
||||
}
|
||||
|
||||
// sendTelemetry posts a single telemetry event. Fire-and-forget: all errors
|
||||
|
|
@ -160,7 +174,9 @@ func run(args []string) error {
|
|||
Usage:
|
||||
msdl [search terms] interactive: filter + pick product, pick language
|
||||
msdl --id 3262 skip product picker
|
||||
msdl --id 3262 --lang "English (United States)" no prompts, print URL directly
|
||||
msdl --id 3262 --lang "English" no prompts, print URL directly
|
||||
(language names vary by product — omit --lang
|
||||
or check the error's "available" list to see them)
|
||||
msdl --eval [slug] evaluation ISOs (server-2025, win11-ent, ...)
|
||||
msdl --list list all products and exit
|
||||
|
||||
|
|
@ -176,7 +192,7 @@ More info: https://msdl.tech-latest.com/cli`)
|
|||
}
|
||||
|
||||
productID := fs.String("id", "", "consumer product ID (skips product picker)")
|
||||
langFlag := fs.String("lang", "", `language name, e.g. "English (United States)"`)
|
||||
langFlag := fs.String("lang", "", `language name, e.g. "English"`)
|
||||
evalMode := fs.Bool("eval", false, "fetch evaluation ISOs")
|
||||
listMode := fs.Bool("list", false, "list all products and exit")
|
||||
noContributeFlag := fs.Bool("no-contribute", false, "skip sharing the link with the msdl.tech cache")
|
||||
|
|
@ -190,10 +206,17 @@ More info: https://msdl.tech-latest.com/cli`)
|
|||
query := strings.Join(fs.Args(), " ")
|
||||
noContribute := *noContributeFlag || os.Getenv("MSDL_NO_CONTRIBUTE") == "1"
|
||||
noTelemetry := os.Getenv("MSDL_NO_TELEMETRY") == "1"
|
||||
isHomepage := *productID == "" && !*evalMode && !*listMode && query == "" && isTerminal()
|
||||
|
||||
// Update check — blocks up to 500ms, then continues regardless
|
||||
// Update check — blocks up to 500ms, then continues regardless. The
|
||||
// homepage screen folds this into its own footer instead of printing a
|
||||
// separate notice.
|
||||
latest := ""
|
||||
if !noTelemetry {
|
||||
printUpdateNotice()
|
||||
latest = fetchLatestVersion()
|
||||
if !isHomepage {
|
||||
printUpdateNotice(latest)
|
||||
}
|
||||
}
|
||||
|
||||
if *listMode {
|
||||
|
|
@ -230,20 +253,37 @@ More info: https://msdl.tech-latest.com/cli`)
|
|||
telAction = "fetch"
|
||||
telProductID = *productID
|
||||
err = runConsumer(*productID, query, *langFlag, noContribute)
|
||||
case query == "" && isTerminal():
|
||||
// Pure interactive: combined consumer + eval picker
|
||||
isEval, product, ep, pickErr := pickCombined()
|
||||
if pickErr != nil {
|
||||
return pickErr
|
||||
case isHomepage:
|
||||
choice, hpErr := showHomepage(latest)
|
||||
if hpErr != nil {
|
||||
return hpErr
|
||||
}
|
||||
if isEval {
|
||||
telAction = "eval"
|
||||
telEvalSlug = ep.Slug
|
||||
err = runEval(ep.Slug)
|
||||
} else {
|
||||
switch choice.kind {
|
||||
case homepageKindProduct:
|
||||
telAction = "fetch"
|
||||
telProductID = product.ID
|
||||
err = runConsumer(product.ID, "", *langFlag, noContribute)
|
||||
telProductID = choice.productID
|
||||
err = runConsumer(choice.productID, "", *langFlag, noContribute)
|
||||
case homepageKindEval:
|
||||
telAction = "eval"
|
||||
telEvalSlug = choice.evalSlug
|
||||
err = runEval(choice.evalSlug)
|
||||
case homepageKindSearch:
|
||||
telAction = "fetch"
|
||||
err = runConsumer("", choice.query, *langFlag, noContribute)
|
||||
case homepageKindList:
|
||||
isEval, product, ep, pickErr := pickCombined()
|
||||
if pickErr != nil {
|
||||
return pickErr
|
||||
}
|
||||
if isEval {
|
||||
telAction = "eval"
|
||||
telEvalSlug = ep.Slug
|
||||
err = runEval(ep.Slug)
|
||||
} else {
|
||||
telAction = "fetch"
|
||||
telProductID = product.ID
|
||||
err = runConsumer(product.ID, "", *langFlag, noContribute)
|
||||
}
|
||||
}
|
||||
default:
|
||||
err = runConsumer(*productID, query, *langFlag, noContribute)
|
||||
|
|
@ -310,7 +350,11 @@ func runConsumer(productID, query, langName string, noContribute bool) error {
|
|||
}
|
||||
}
|
||||
if lang.ID == "" {
|
||||
return fmt.Errorf("language %q not available for %s", langName, product.Name)
|
||||
names := make([]string, len(langs))
|
||||
for i, l := range langs {
|
||||
names[i] = l.Language
|
||||
}
|
||||
return fmt.Errorf("language %q not available for %s — available: %s", langName, product.Name, strings.Join(names, ", "))
|
||||
}
|
||||
} else {
|
||||
lang, err = pickLanguage(langs)
|
||||
|
|
|
|||
|
|
@ -7,17 +7,25 @@ import (
|
|||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
http "github.com/bogdanfinn/fhttp"
|
||||
tls_client "github.com/bogdanfinn/tls-client"
|
||||
"github.com/bogdanfinn/tls-client/profiles"
|
||||
)
|
||||
|
||||
// msTLSProfile is the browser TLS/HTTP2 fingerprint the CLI presents to Microsoft.
|
||||
// Kept in step with msUA's claimed Chrome version -- a mismatched UA vs. TLS
|
||||
// fingerprint is itself a detectable signal, so if one changes, change both.
|
||||
var msTLSProfile = profiles.Chrome_133
|
||||
|
||||
const (
|
||||
msUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
msUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36"
|
||||
msProfile = "606624d44113"
|
||||
msLocale = "en-US"
|
||||
msOrgID = "y6jn8c31"
|
||||
|
|
@ -47,37 +55,6 @@ type EvalLink struct {
|
|||
URL string
|
||||
}
|
||||
|
||||
type simpleCookieJar struct {
|
||||
mu sync.Mutex
|
||||
cookies []*http.Cookie
|
||||
}
|
||||
|
||||
func (j *simpleCookieJar) SetCookies(_ *url.URL, cookies []*http.Cookie) {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
for _, c := range cookies {
|
||||
found := false
|
||||
for i, existing := range j.cookies {
|
||||
if existing.Name == c.Name {
|
||||
j.cookies[i] = c
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
j.cookies = append(j.cookies, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (j *simpleCookieJar) Cookies(_ *url.URL) []*http.Cookie {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
out := make([]*http.Cookie, len(j.cookies))
|
||||
copy(out, j.cookies)
|
||||
return out
|
||||
}
|
||||
|
||||
func newSessionID() string {
|
||||
b := make([]byte, 16)
|
||||
rand.Read(b)
|
||||
|
|
@ -116,10 +93,16 @@ func mapDownloadType(n int) string {
|
|||
}
|
||||
}
|
||||
|
||||
func newSession() (*http.Client, string) {
|
||||
func newSession() (tls_client.HttpClient, string) {
|
||||
sessionID := newSessionID()
|
||||
jar := &simpleCookieJar{}
|
||||
client := &http.Client{Timeout: 15 * time.Second, Jar: jar}
|
||||
client, err := tls_client.NewHttpClient(tls_client.NewNoopLogger(),
|
||||
tls_client.WithTimeoutSeconds(15),
|
||||
tls_client.WithClientProfile(msTLSProfile),
|
||||
tls_client.WithCookieJar(tls_client.NewCookieJar()),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, sessionID
|
||||
}
|
||||
|
||||
q1 := url.Values{}
|
||||
q1.Set("org_id", msOrgID)
|
||||
|
|
@ -158,7 +141,7 @@ func newSession() (*http.Client, string) {
|
|||
return client, sessionID
|
||||
}
|
||||
|
||||
func msGet(client *http.Client, reqURL, productID string) ([]byte, error) {
|
||||
func msGet(client tls_client.HttpClient, reqURL, productID string) ([]byte, error) {
|
||||
req, _ := http.NewRequest("GET", reqURL, nil)
|
||||
req.Header.Set("User-Agent", msUA)
|
||||
req.Header.Set("Referer", referer(productID))
|
||||
|
|
@ -249,7 +232,7 @@ func parseDownloadLinks(raw []byte) ([]DownloadLink, error) {
|
|||
return links, nil
|
||||
}
|
||||
|
||||
func fetchLanguages(client *http.Client, sessionID, productID string) ([]Language, error) {
|
||||
func fetchLanguages(client tls_client.HttpClient, sessionID, productID string) ([]Language, error) {
|
||||
q := url.Values{}
|
||||
q.Set("profile", msProfile)
|
||||
q.Set("productEditionId", productID)
|
||||
|
|
@ -267,7 +250,7 @@ func fetchLanguages(client *http.Client, sessionID, productID string) ([]Languag
|
|||
}
|
||||
|
||||
// fetchDownloadLinks returns parsed links and the raw Microsoft JSON (for cache contribution).
|
||||
func fetchDownloadLinks(client *http.Client, sessionID, productID, skuID string) ([]DownloadLink, []byte, error) {
|
||||
func fetchDownloadLinks(client tls_client.HttpClient, sessionID, productID, skuID string) ([]DownloadLink, []byte, error) {
|
||||
wq := url.Values{}
|
||||
wq.Set("profile", msProfile)
|
||||
wq.Set("productEditionId", productID)
|
||||
|
|
@ -335,7 +318,13 @@ func detectLang(rawURL string) string {
|
|||
}
|
||||
|
||||
func fetchEvalLinks(evalURL string) ([]EvalLink, error) {
|
||||
client := &http.Client{Timeout: 20 * time.Second}
|
||||
client, err := tls_client.NewHttpClient(tls_client.NewNoopLogger(),
|
||||
tls_client.WithTimeoutSeconds(20),
|
||||
tls_client.WithClientProfile(msTLSProfile),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating http client: %w", err)
|
||||
}
|
||||
req, _ := http.NewRequest("GET", evalURL, nil)
|
||||
req.Header.Set("User-Agent", msUA)
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml")
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
|
@ -1,200 +0,0 @@
|
|||
# CLI Telemetry, Update Check & Help — Design Spec
|
||||
|
||||
**Date:** 2026-06-21
|
||||
**Status:** Approved
|
||||
**Scope:** CLI (cli/) + Backend (backend/main.go)
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Three related additions to the msdl CLI and backend:
|
||||
|
||||
1. **Telemetry** — anonymous usage counters sent on every CLI run, stored durably in Redis
|
||||
2. **Update check** — CLI checks our backend for the latest version on every run, notifies if behind
|
||||
3. **Help** — `msdl --help` / `-h` prints usage with all flags and examples
|
||||
|
||||
No opt-in required. `MSDL_NO_TELEMETRY=1` skips both telemetry and update check silently.
|
||||
Disclosed in README and `/cli` page with one line each.
|
||||
|
||||
---
|
||||
|
||||
## Backend Changes
|
||||
|
||||
### 1. New constant: `latestCLIVersion`
|
||||
|
||||
```go
|
||||
const latestCLIVersion = "0.3.0"
|
||||
```
|
||||
|
||||
Bumped manually in `backend/main.go` whenever a new CLI release is cut.
|
||||
|
||||
### 2. `GET /cli/version` — public, no auth
|
||||
|
||||
Returns the latest CLI version. The CLI calls this on every run.
|
||||
|
||||
```json
|
||||
{ "latest": "0.3.0" }
|
||||
```
|
||||
|
||||
No rate limiting needed — response is trivial and cacheable by the client.
|
||||
|
||||
### 3. `POST /telemetry` — public, no auth
|
||||
|
||||
Accepts one event per CLI run. Body:
|
||||
|
||||
```json
|
||||
{
|
||||
"action": "fetch" | "eval" | "list" | "interactive",
|
||||
"product_id": "2618",
|
||||
"eval_slug": "server-2025",
|
||||
"platform": "windows" | "darwin" | "linux",
|
||||
"version": "0.3.0",
|
||||
"success": true
|
||||
}
|
||||
```
|
||||
|
||||
- `product_id` present only for `fetch` action
|
||||
- `eval_slug` present only for `eval` action
|
||||
- All other fields always present
|
||||
- Returns `200 OK` with `{}` — CLI ignores the response
|
||||
- Per-IP rate limit: ~10 req/min (same token bucket pattern as `/contribute`)
|
||||
- Increments Redis counters via `HINCRBY` directly (no in-memory layer)
|
||||
- If Redis unavailable: silently drop (telemetry is best-effort)
|
||||
|
||||
### 4. Redis telemetry keys
|
||||
|
||||
```
|
||||
msdl:telemetry:actions hash { fetch, eval, list, interactive }
|
||||
msdl:telemetry:platforms hash { windows, darwin, linux }
|
||||
msdl:telemetry:versions hash { "0.2.0", "0.3.0", ... }
|
||||
msdl:telemetry:products hash { "2618", "3262", "3113", ... }
|
||||
msdl:telemetry:results hash { success, failed }
|
||||
```
|
||||
|
||||
### 5. Existing metrics — Redis persistence
|
||||
|
||||
Current in-memory `atomic.Int64` counters are preserved for fast per-request increments. Added durability:
|
||||
|
||||
- **On startup**: seed in-memory counters from Redis (`HGETALL msdl:metrics:*`)
|
||||
- **Every 5 minutes**: flush in-memory counters to Redis (`HSET msdl:metrics:*`)
|
||||
- **On graceful shutdown**: final flush before exit
|
||||
|
||||
Redis keys for existing metrics:
|
||||
|
||||
```
|
||||
msdl:metrics:sku hash { requests, hits, fetches, neg_hits }
|
||||
msdl:metrics:link hash { requests, hits, fetches, neg_hits, stale }
|
||||
msdl:metrics:eval hash { requests, hits, stale }
|
||||
```
|
||||
|
||||
### 6. `/metrics` response — updated
|
||||
|
||||
Adds telemetry section to existing cache stats:
|
||||
|
||||
```json
|
||||
{
|
||||
"sku": { ... },
|
||||
"link": { ... },
|
||||
"eval": { ... },
|
||||
"telemetry": {
|
||||
"actions": { "fetch": 142, "eval": 31, "list": 8, "interactive": 67 },
|
||||
"platforms": { "windows": 180, "darwin": 42, "linux": 18 },
|
||||
"versions": { "0.3.0": 145, "0.2.0": 95 },
|
||||
"products": { "3262": 134, "2618": 89, "3113": 67 },
|
||||
"results": { "success": 390, "failed": 48 }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## CLI Changes
|
||||
|
||||
### 1. Version constant
|
||||
|
||||
Injected at build time via ldflags:
|
||||
|
||||
```
|
||||
-ldflags "-X main.Version=0.3.0"
|
||||
```
|
||||
|
||||
Added to the existing GitHub Actions release workflow (`cli-release.yml`). Fallback: `const Version = "dev"` in source.
|
||||
|
||||
### 2. Update check + telemetry — concurrent goroutines
|
||||
|
||||
On every run, immediately after flag parsing, two goroutines are launched:
|
||||
|
||||
```
|
||||
main()
|
||||
├── go updateCheck() — GET /cli/version, 500ms timeout, print notice if behind
|
||||
└── go sendTelemetry() — POST /telemetry, fire-and-forget, result ignored
|
||||
```
|
||||
|
||||
Both are non-blocking. The update check result is printed before any other output if it arrives within 500ms. If the timeout fires, it's silently skipped for that run.
|
||||
|
||||
**Update notice format** (printed before picker or output):
|
||||
```
|
||||
A new version of msdl is available: v0.3.0
|
||||
Download: https://github.com/starkSV/windows-iso-downloader/releases/latest
|
||||
```
|
||||
|
||||
Both goroutines are skipped entirely when `MSDL_NO_TELEMETRY=1` is set.
|
||||
|
||||
### 3. Telemetry payload construction
|
||||
|
||||
| Scenario | action | product_id | eval_slug | success |
|
||||
|---|---|---|---|---|
|
||||
| `msdl` (interactive) | `interactive` | set after pick | — | true/false |
|
||||
| `msdl --id X --lang Y` | `fetch` | X | — | true/false |
|
||||
| `msdl --eval slug` | `eval` | — | slug | true/false |
|
||||
| `msdl --list` | `list` | — | — | true |
|
||||
|
||||
`platform` is set from `runtime.GOOS` at runtime (not build time — same result, simpler).
|
||||
`version` is set from the `Version` constant.
|
||||
|
||||
Telemetry is sent **after** the main action completes so `success` reflects the actual outcome.
|
||||
|
||||
### 4. `--help` / `-h`
|
||||
|
||||
Prints a formatted usage block and exits 0. Shown automatically by Go's `flag` package or via a custom print if the existing CLI uses manual arg parsing.
|
||||
|
||||
```
|
||||
msdl — Windows ISO downloader
|
||||
|
||||
Usage:
|
||||
msdl Interactive mode — pick product and language
|
||||
msdl --id <id> --lang <lang> Fetch link directly (skip picker)
|
||||
msdl --eval <slug> Evaluation / Server ISO
|
||||
msdl --list List all available products
|
||||
|
||||
Flags:
|
||||
--id <id> Product ID (e.g. 3262 for Windows 11 25H2)
|
||||
--lang <language> Language name (e.g. "English")
|
||||
--eval <slug> Eval ISO slug (server-2025, server-2022, win11-ent, ...)
|
||||
--list List all products and exit
|
||||
--no-contribute Skip contributing link back to msdl web cache
|
||||
-h, --help Show this help
|
||||
|
||||
Environment:
|
||||
MSDL_NO_TELEMETRY=1 Disable anonymous usage reporting and update checks
|
||||
MSDL_NO_CONTRIBUTE=1 Disable cache contribution
|
||||
|
||||
More info: https://msdl.tech-latest.com/cli
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Documentation Updates
|
||||
|
||||
- **README** — one line under CLI section: "msdl sends anonymous usage counts (action, platform, version) to help us understand which products are popular. Set `MSDL_NO_TELEMETRY=1` to opt out."
|
||||
- **`/cli` page** — same one-liner in the contribute section, next to the `--no-contribute` note.
|
||||
|
||||
---
|
||||
|
||||
## What's Not In Scope
|
||||
|
||||
- No per-user tracking, no IP storage, no session IDs
|
||||
- No telemetry dashboard UI (raw numbers via `/metrics` is sufficient)
|
||||
- No forced update or auto-update — notification only
|
||||
- No telemetry for the web frontend
|
||||
184
docs/superpowers/specs/2026-07-13-needs-warming-design.md
Normal file
184
docs/superpowers/specs/2026-07-13-needs-warming-design.md
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
# Spec: "Needs Warming" Community Help Page
|
||||
|
||||
**Status:** Proposed
|
||||
**Date:** 2026-07-13
|
||||
**Author:** Shekhar + Grok (original idea), refined against live codebase by Claude
|
||||
**Related:** Crowdsourced cache warming (`/contribute`), Sentinel lockdown resilience, CLI contribution flow
|
||||
|
||||
---
|
||||
|
||||
## 1. Goal
|
||||
|
||||
Give visible, honest signal for products currently failing web users because Microsoft's Sentinel WAF (or a rate limit) has blocked the backend and no cached link is available to fall back on. Point people at the CLI as the fix, and let a successful fetch/contribution clear the item automatically.
|
||||
|
||||
This is not a new mechanism bolted onto the project — it's a public window into resilience state (`negCache` / `linkCache`) that already exists and already drives the per-product "blocked, use CLI" banner. This just makes that signal visible in aggregate, with a stronger call to action.
|
||||
|
||||
**Change from the original draft:** a dedicated page, not a homepage section. Original draft (written by Grok from the GitHub repo alone, without runtime visibility into cache state) proposed a parallel tracking system; this version derives the list from state that already exists, which is both less code and immune to drift between two copies of "is this broken."
|
||||
|
||||
---
|
||||
|
||||
## 2. What Actually Needs Tracking (verified against `backend/main.go`)
|
||||
|
||||
`handleProxy` (main.go:1189-1404) has exactly three outcomes when a product/SKU is under an active Sentinel or rate-limit block:
|
||||
|
||||
| Outcome | Code path | User experience | Needs warming? |
|
||||
|---|---|---|---|
|
||||
| `serving cached` | negCache active, `linkCache` entry still within its own TTL (e.g. just contributed) | Gets a working link | **No** |
|
||||
| `serving stale` | negCache active, `linkCache` entry expired but still returned | Gets a (possibly near-expired) link | **No** |
|
||||
| `no stale available` | negCache active, nothing in `linkCache` at all | Gets a 429, no link | **Yes** — this is the actual failure |
|
||||
|
||||
So "needs warming" = **an active `negCache` entry with no corresponding valid-or-stale `linkCache` entry** — computed on read, not tracked as separate state. No new manager struct, no `Record()`/`Remove()` calls to keep in sync, no new Redis schema.
|
||||
|
||||
**Known gap, scoped out of MVP:** a fetch that fails with a non-rate-limit error (network blip, malformed response) and has no stale cache to fall back on returns an error directly (main.go:1377) without ever touching `negCache`. In practice this hasn't been the dominant failure mode this month — Sentinel/rate-limit blocks have been — so the MVP doesn't cover it. If it turns out to matter, closing the gap is a two-line addition (write a `negCacheEntry` with `IsSentinel: false` at that call site) rather than a redesign.
|
||||
|
||||
---
|
||||
|
||||
## 3. Data Needed That Doesn't Exist Yet
|
||||
|
||||
Two small additions, both minimal:
|
||||
|
||||
1. **A request counter per locked product/SKU.** `negCache`/`linkCache` don't track "how many times did this get hit while broken" — add a small `map[string]int` (or extend `negCacheEntry` with a counter field) incremented at the existing `"no stale available"` log call sites.
|
||||
2. **Consumer product display names on the backend.** `validContributeProducts` (main.go:476) is `map[string]bool` — just an ID allow-list, no names. Eval products already have names (`{Name, EvalURL}` map at main.go:207-211). Converting `validContributeProducts` to `map[string]string` (ID → Name) serves both its existing validation role and this new lookup need, with no behavior change to `/contribute`.
|
||||
|
||||
Language name hydration for consumer products reuses the existing `skuCache` (already populated per-product from the normal SKU-fetch flow) — look up the SKU by ID within that product's cached SKU list to get `Language`/`LocalizedLanguage`. No new structure needed; if the product isn't in `skuCache` yet, just omit the language field rather than blocking the response.
|
||||
|
||||
---
|
||||
|
||||
## 4. New Endpoint
|
||||
|
||||
### `GET /needs-warming`
|
||||
|
||||
Computed on each request by iterating `negCache`, filtering to entries where `IsSentinel` (or rate-limited) is still active **and** no valid `linkCache`/eval-cache entry exists for that key. No background job, no persistence — it's a live view.
|
||||
|
||||
**Query params:** `limit` (default 10, max 50)
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"items": [
|
||||
{
|
||||
"product_id": "3262",
|
||||
"sku_id": "0x0409",
|
||||
"product_name": "Windows 11 25H2",
|
||||
"language": "English (United States)",
|
||||
"is_eval": false,
|
||||
"reason": "waf_blocked",
|
||||
"last_seen": "2026-07-13T17:42:00Z",
|
||||
"request_count": 47,
|
||||
"cli_command": "msdl --id 3262 --lang \"English (United States)\""
|
||||
}
|
||||
],
|
||||
"total": 3
|
||||
}
|
||||
```
|
||||
|
||||
- Public, unauthenticated, rate-limited (30 req/min per IP — matches the existing pattern used by `/contribute` at 5/min and `/telemetry` at 10/min)
|
||||
- Sorted by `request_count` descending (surfaces the highest-impact items first, more useful than recency alone)
|
||||
- Eval products included (`is_eval: true`, `cli_command` uses `msdl --eval <slug>` instead of `--id`)
|
||||
|
||||
---
|
||||
|
||||
## 5. Backend Implementation Outline
|
||||
|
||||
No new package-level state beyond the two additions in §3. The handler:
|
||||
|
||||
```go
|
||||
func handleNeedsWarming(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
if !needsWarmingRL.allow(clientIP(r)) {
|
||||
respondJSONError(w, http.StatusTooManyRequests, "rate limit exceeded")
|
||||
return
|
||||
}
|
||||
limit := parseLimitParam(r, 10, 50)
|
||||
|
||||
negCacheMu.RLock()
|
||||
defer negCacheMu.RUnlock()
|
||||
linkCacheMu.RLock()
|
||||
defer linkCacheMu.RUnlock()
|
||||
|
||||
var items []needsWarmingItem
|
||||
for key, neg := range negCache {
|
||||
if time.Now().After(neg.ExpiresAt) {
|
||||
continue // lockdown/rate-limit window itself already expired
|
||||
}
|
||||
if _, hasValidLink := resolveValidLink(key, linkCache); hasValidLink {
|
||||
continue // already resolved via cached/stale link -- not a failure state
|
||||
}
|
||||
items = append(items, buildNeedsWarmingItem(key, neg))
|
||||
}
|
||||
sort.Slice(items, func(i, j int) bool { return items[i].RequestCount > items[j].RequestCount })
|
||||
if len(items) > limit {
|
||||
items = items[:limit]
|
||||
}
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{"items": items, "total": len(items)})
|
||||
}
|
||||
```
|
||||
|
||||
Integration points:
|
||||
- Increment the new request counter at the existing `"lockdown active, no stale available"` and first-time-Sentinel/rate-limit-with-no-stale log sites (main.go:1225, ~1367)
|
||||
- No changes needed to `/contribute` or the fresh-fetch success path — an item stops appearing automatically once `linkCache` has a valid entry, which those paths already populate
|
||||
|
||||
---
|
||||
|
||||
## 6. Frontend: Dedicated Page
|
||||
|
||||
**Route:** `/needs-warming` (or `/community` — bikeshed-able, not load-bearing)
|
||||
|
||||
Linked from the footer and from the per-product "blocked" banner (`CliHandoff.tsx`'s `highlight` variant) — "See what else needs help →" — so people already primed to help via the CLI can see the fuller picture instead of just their one product.
|
||||
|
||||
**Empty state:** hide the page's list section entirely (or show a short "Everything's healthy right now" line) — no "0 items" placeholder box. Most visits will likely find this empty, which is a good sign, not a gap to fill with UI.
|
||||
|
||||
Each card:
|
||||
- Product name + language (or eval product name)
|
||||
- Request count as a small badge (social proof — "47 people hit this")
|
||||
- One-click "Copy CLI command" button (same interaction pattern as the existing `CliHandoff` component — reuse its copy-button styling)
|
||||
- No per-item "reason" jargon exposed in the UI (WAF/rate-limit distinction is backend detail); just "temporarily unavailable"
|
||||
|
||||
Tone: matches the project's existing calm, transparent voice (README already explains the caching layer and Sentinel lockdown openly) — helpful, not alarmist.
|
||||
|
||||
---
|
||||
|
||||
## 7. CLI Impact
|
||||
|
||||
None required for MVP. Optional future addition: `msdl --needs-warming` prints the same list from the terminal and offers to fetch+contribute the top item directly — natural fit for the homepage-screen work already shipped (`cli/homepage.go`), but a separate phase.
|
||||
|
||||
---
|
||||
|
||||
## 8. Privacy & Safety
|
||||
|
||||
- Only ever stores/exposes `product_id`, `sku_id`/slug, reason, counters, timestamps — no IPs, no user identifiers (matches the existing telemetry/contribute philosophy already documented in README)
|
||||
- Rate-limited public endpoint, same pattern as existing public endpoints
|
||||
- Nothing new to abuse: the list is derived from state Microsoft's own WAF already put the backend into, not something a client can inject
|
||||
|
||||
---
|
||||
|
||||
## 9. Implementation Phases
|
||||
|
||||
| Phase | Description | Effort |
|
||||
|---|---|---|
|
||||
| MVP | Request counter + product-name map conversion + `/needs-warming` handler + dedicated page | Small — most of the state already exists |
|
||||
| 2 | Link from per-product blocked banner; polish empty/loading states | Small |
|
||||
| 3 | Close the non-rate-limit fetch-error gap (§2) if it turns out to matter in practice | Small, only if needed |
|
||||
| 4 | `msdl --needs-warming` CLI command | Small |
|
||||
|
||||
(Dropped the original draft's "Redis-backed persistence" phase — this is inherently live/transient state mirroring `negCache`'s own in-memory, reset-on-restart behavior; persisting it would add a Redis key namespace for no real benefit.)
|
||||
|
||||
---
|
||||
|
||||
## 10. Open Questions Resolved
|
||||
|
||||
1. **Eval products included?** Yes.
|
||||
2. **Max age before auto-removal?** No separate TTL — inherits the underlying `negCache` entry's own expiry (60s rate-limit / 90min Sentinel), since the list is computed live rather than tracked separately.
|
||||
3. **Empty state?** Hide the list, don't show a "0 items" box.
|
||||
4. **Tone?** Helpful/neutral, matching existing project voice.
|
||||
|
||||
---
|
||||
|
||||
## 11. Success Metrics (Future)
|
||||
|
||||
- Contributions triggered from this page specifically (could tag `/contribute` calls with a `source=needs-warming` query param to measure this)
|
||||
- Reduction in time-to-resolution for locked-down products
|
||||
- CLI download/usage bump correlated with items appearing on the page
|
||||
|
|
@ -1 +0,0 @@
|
|||
/* /index.html 200
|
||||
|
|
@ -12,7 +12,7 @@
|
|||
},
|
||||
"2378": {
|
||||
"name": "Windows 10 22H2 Home China (19045.2006)",
|
||||
"badge": "EOL SOON",
|
||||
"badge": "EOL",
|
||||
"archs": [
|
||||
"x64"
|
||||
],
|
||||
|
|
@ -22,7 +22,7 @@
|
|||
},
|
||||
"2618": {
|
||||
"name": "Windows 10 22H2 v1 (19045.2965)",
|
||||
"badge": "EOL SOON",
|
||||
"badge": "EOL",
|
||||
"archs": [
|
||||
"x64",
|
||||
"x86"
|
||||
|
|
@ -169,7 +169,7 @@
|
|||
]
|
||||
},
|
||||
"3321": {
|
||||
"name": "Windows 11 25H2 (Updated Oct)",
|
||||
"name": "Windows 11 25H2 (V2)",
|
||||
"badge": "LATEST",
|
||||
"archs": [
|
||||
"x64"
|
||||
|
|
@ -180,8 +180,30 @@
|
|||
"3113"
|
||||
]
|
||||
},
|
||||
"3322": {
|
||||
"name": "Windows 11 25H2 Home China (V2)",
|
||||
"badge": "LATEST",
|
||||
"archs": [
|
||||
"x64"
|
||||
],
|
||||
"related": [
|
||||
"3321",
|
||||
"3323"
|
||||
]
|
||||
},
|
||||
"3323": {
|
||||
"name": "Windows 11 25H2 Pro China (V2)",
|
||||
"badge": "LATEST",
|
||||
"archs": [
|
||||
"x64"
|
||||
],
|
||||
"related": [
|
||||
"3321",
|
||||
"3322"
|
||||
]
|
||||
},
|
||||
"3324": {
|
||||
"name": "Windows 11 Arm64 25H2 (Updated Oct)",
|
||||
"name": "Windows 11 Arm64 25H2 (V2)",
|
||||
"badge": "LATEST",
|
||||
"archs": [
|
||||
"ARM64"
|
||||
|
|
@ -191,5 +213,27 @@
|
|||
"3262",
|
||||
"3265"
|
||||
]
|
||||
},
|
||||
"3325": {
|
||||
"name": "Windows 11 Arm64 25H2 Home China (V2)",
|
||||
"badge": "LATEST",
|
||||
"archs": [
|
||||
"ARM64"
|
||||
],
|
||||
"related": [
|
||||
"3324",
|
||||
"3326"
|
||||
]
|
||||
},
|
||||
"3326": {
|
||||
"name": "Windows 11 Arm64 25H2 Pro China (V2)",
|
||||
"badge": "LATEST",
|
||||
"archs": [
|
||||
"ARM64"
|
||||
],
|
||||
"related": [
|
||||
"3324",
|
||||
"3325"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -46,7 +46,6 @@
|
|||
<url><loc>https://msdl.tech-latest.com/product/win11-ent</loc><lastmod>2026-05-18</lastmod><priority>0.8</priority><changefreq>monthly</changefreq></url>
|
||||
|
||||
<!-- Product Pages -->
|
||||
<url><loc>https://msdl.tech-latest.com/product/48</loc><lastmod>2026-05-12</lastmod><priority>0.8</priority><changefreq>monthly</changefreq></url>
|
||||
<url><loc>https://msdl.tech-latest.com/product/52</loc><lastmod>2026-05-12</lastmod><priority>0.8</priority><changefreq>monthly</changefreq></url>
|
||||
<url><loc>https://msdl.tech-latest.com/product/2378</loc><lastmod>2026-05-12</lastmod><priority>0.8</priority><changefreq>monthly</changefreq></url>
|
||||
<url><loc>https://msdl.tech-latest.com/product/2618</loc><lastmod>2026-05-12</lastmod><priority>0.8</priority><changefreq>monthly</changefreq></url>
|
||||
|
|
@ -63,5 +62,9 @@
|
|||
<url><loc>https://msdl.tech-latest.com/product/3266</loc><lastmod>2026-05-12</lastmod><priority>0.8</priority><changefreq>monthly</changefreq></url>
|
||||
<url><loc>https://msdl.tech-latest.com/product/3267</loc><lastmod>2026-05-12</lastmod><priority>0.8</priority><changefreq>monthly</changefreq></url>
|
||||
<url><loc>https://msdl.tech-latest.com/product/3321</loc><lastmod>2026-05-12</lastmod><priority>0.8</priority><changefreq>monthly</changefreq></url>
|
||||
<url><loc>https://msdl.tech-latest.com/product/3322</loc><lastmod>2026-07-14</lastmod><priority>0.8</priority><changefreq>monthly</changefreq></url>
|
||||
<url><loc>https://msdl.tech-latest.com/product/3323</loc><lastmod>2026-07-14</lastmod><priority>0.8</priority><changefreq>monthly</changefreq></url>
|
||||
<url><loc>https://msdl.tech-latest.com/product/3324</loc><lastmod>2026-05-12</lastmod><priority>0.8</priority><changefreq>monthly</changefreq></url>
|
||||
<url><loc>https://msdl.tech-latest.com/product/3325</loc><lastmod>2026-07-14</lastmod><priority>0.8</priority><changefreq>monthly</changefreq></url>
|
||||
<url><loc>https://msdl.tech-latest.com/product/3326</loc><lastmod>2026-07-14</lastmod><priority>0.8</priority><changefreq>monthly</changefreq></url>
|
||||
</urlset>
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ const RELEASES_URL = 'https://minxl.ink/msdl-github-release'
|
|||
|
||||
function InstallSteps() {
|
||||
const [copied, setCopied] = useState(false)
|
||||
const [brewCopied, setBrewCopied] = useState(false)
|
||||
const [curlCopied, setCurlCopied] = useState(false)
|
||||
|
||||
function handleCopy() {
|
||||
navigator.clipboard.writeText('winget install starkSV.msdl')
|
||||
|
|
@ -22,6 +24,18 @@ function InstallSteps() {
|
|||
setTimeout(() => setCopied(false), 2000)
|
||||
}
|
||||
|
||||
function handleBrewCopy() {
|
||||
navigator.clipboard.writeText('brew tap starkSV/msdl && brew install msdl-cli')
|
||||
setBrewCopied(true)
|
||||
setTimeout(() => setBrewCopied(false), 2000)
|
||||
}
|
||||
|
||||
function handleCurlCopy() {
|
||||
navigator.clipboard.writeText('curl -fsSL https://api.msdl.tech-latest.com/install.sh | bash')
|
||||
setCurlCopied(true)
|
||||
setTimeout(() => setCurlCopied(false), 2000)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-2">
|
||||
<p className="text-[10px] font-mono font-semibold uppercase tracking-widest text-zinc-600">1. Install</p>
|
||||
|
|
@ -37,6 +51,24 @@ function InstallSteps() {
|
|||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div className="rounded-lg border border-emerald-500/20 bg-emerald-500/5 p-3 space-y-2">
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-[10px] font-mono font-semibold tracking-widest uppercase text-emerald-500">macOS / Linux · Homebrew</span>
|
||||
<span className="text-[10px] px-1.5 py-0.5 rounded border border-emerald-500/20 text-emerald-600 font-mono">recommended</span>
|
||||
</div>
|
||||
<div className="flex items-center justify-between gap-2 rounded bg-black/30 px-3 py-2">
|
||||
<code className="text-[12px] font-mono text-zinc-300">brew tap starkSV/msdl && brew install msdl-cli</code>
|
||||
<button onClick={handleBrewCopy} className="flex-shrink-0 text-zinc-500 hover:text-white transition-colors">
|
||||
{brewCopied ? <Check size={12} className="text-green-400" /> : <Copy size={12} />}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center justify-between gap-2 rounded-lg bg-white/4 border border-white/7 px-3 py-2">
|
||||
<code className="text-[11px] font-mono text-zinc-400 truncate">curl -fsSL https://api.msdl.tech-latest.com/install.sh | bash</code>
|
||||
<button onClick={handleCurlCopy} className="flex-shrink-0 text-zinc-500 hover:text-white transition-colors">
|
||||
{curlCopied ? <Check size={12} className="text-green-400" /> : <Copy size={12} />}
|
||||
</button>
|
||||
</div>
|
||||
<a
|
||||
href={RELEASES_URL}
|
||||
target="_blank"
|
||||
|
|
|
|||
|
|
@ -45,6 +45,11 @@ export default function Dock() {
|
|||
|
||||
return (
|
||||
<>
|
||||
{/* Full-width shelf so the floating desktop dock separates from scrolled content — page bg is
|
||||
near-black (zinc-950), so plain color gradients are invisible; backdrop-blur is what reads.
|
||||
Desktop only: the mobile dock is already a flush full-width bar with no gap to bridge. */}
|
||||
<div className="hidden sm:block fixed inset-x-0 bottom-0 h-24 backdrop-blur-md bg-gradient-to-t from-zinc-950/90 via-zinc-950/60 to-transparent pointer-events-none z-40" />
|
||||
|
||||
{/* ── DESKTOP DOCK (sm+) — centered floating pill ── */}
|
||||
<div className="hidden sm:flex fixed bottom-5 left-0 right-0 justify-center z-50 pointer-events-none">
|
||||
<motion.div
|
||||
|
|
|
|||
|
|
@ -1,12 +1,23 @@
|
|||
import { useState } from 'react'
|
||||
import { useState, type ReactNode } from 'react'
|
||||
import { motion, AnimatePresence } from 'motion/react'
|
||||
import { ChevronRight } from 'lucide-react'
|
||||
|
||||
interface FaqItem {
|
||||
q: string
|
||||
a: string
|
||||
a: ReactNode
|
||||
}
|
||||
|
||||
const repoLink = (
|
||||
<a
|
||||
href="https://github.com/starkSV/windows-iso-downloader"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="text-blue-400 hover:text-blue-300 transition-colors"
|
||||
>
|
||||
GitHub
|
||||
</a>
|
||||
)
|
||||
|
||||
const faqs: FaqItem[] = [
|
||||
{
|
||||
q: 'Is this legal?',
|
||||
|
|
@ -34,7 +45,13 @@ const faqs: FaqItem[] = [
|
|||
},
|
||||
{
|
||||
q: 'Is MSDL open source?',
|
||||
a: 'Yes. Both the frontend and the backend proxy are fully open source. The backend implements the same session-based flow as the Fido PowerShell script, ported to Go and Node.js. You can self-host the entire stack.',
|
||||
a: (
|
||||
<>
|
||||
Yes. Both the frontend and the backend proxy are fully open source, on {repoLink}. The
|
||||
backend and the standalone CLI both implement the same session-based flow as the Fido
|
||||
PowerShell script, ported to Go. You can self-host the entire stack.
|
||||
</>
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { motion } from 'motion/react'
|
||||
import { useState, useEffect } from 'react'
|
||||
import { evalProducts } from '../data/evalProducts'
|
||||
|
||||
interface Stat {
|
||||
value: string
|
||||
|
|
@ -7,14 +8,14 @@ interface Stat {
|
|||
}
|
||||
|
||||
export default function StatsBar() {
|
||||
const [totalReleases, setTotalReleases] = useState('17')
|
||||
const [totalReleases, setTotalReleases] = useState((17 + evalProducts.length).toString())
|
||||
|
||||
useEffect(() => {
|
||||
fetch('/data/products.json')
|
||||
.then(r => r.json())
|
||||
.then((data: Record<string, { active?: boolean }>) => {
|
||||
const activeCount = Object.values(data).filter(p => p.active !== false).length
|
||||
setTotalReleases(activeCount.toString())
|
||||
setTotalReleases((activeCount + evalProducts.length).toString())
|
||||
})
|
||||
.catch(() => {})
|
||||
}, [])
|
||||
|
|
|
|||
|
|
@ -49,16 +49,8 @@ export default function AboutPage() {
|
|||
<Section title="How it works">
|
||||
<p>
|
||||
Our backend replicates the session-based authentication flow that Microsoft uses to
|
||||
serve download links to end users. The same approach is used by{' '}
|
||||
<a
|
||||
href="https://github.com/pbatard/Fido"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="text-blue-400 hover:text-blue-300 transition-colors"
|
||||
>
|
||||
Fido
|
||||
</a>{' '}
|
||||
(the PowerShell script bundled with Rufus). The flow is:
|
||||
serve download links to end users — the same technique used by Rufus (credited below).
|
||||
The flow is:
|
||||
</p>
|
||||
<ol className="list-decimal list-inside space-y-1 text-zinc-500 text-sm">
|
||||
<li>Register a session with Microsoft's tracking endpoint</li>
|
||||
|
|
@ -87,9 +79,22 @@ export default function AboutPage() {
|
|||
</code>
|
||||
</p>
|
||||
<p>
|
||||
Product IDs are maintained manually based on Microsoft's release cadence.
|
||||
Windows 11 25H2, 24H2, Windows 10 22H2, and Windows 8.1 are currently listed.
|
||||
New releases are added as Microsoft publishes them.
|
||||
Product IDs are maintained manually based on Microsoft's release cadence, covering
|
||||
consumer Windows (11, 10, 8.1) as well as Windows Server (2016–2025) and Windows 11
|
||||
Enterprise evaluation editions. New releases are added as Microsoft publishes them —
|
||||
see the <a href="/products" className="text-blue-400 hover:text-blue-300 transition-colors">full catalog</a>.
|
||||
</p>
|
||||
</Section>
|
||||
|
||||
<Section title="CLI tool">
|
||||
<p>
|
||||
MSDL also ships as a standalone command-line tool,{' '}
|
||||
<code className="text-[11px] font-mono bg-white/5 border border-white/7 px-1.5 py-0.5 rounded text-zinc-400">msdl</code>.
|
||||
It runs the same Microsoft session flow directly from your own machine instead of through
|
||||
our backend — useful when Microsoft is rate-limiting our server, since a request from
|
||||
your own connection isn't affected by that. See the{' '}
|
||||
<a href="/cli" className="text-blue-400 hover:text-blue-300 transition-colors">CLI page</a> for
|
||||
install instructions.
|
||||
</p>
|
||||
</Section>
|
||||
|
||||
|
|
|
|||
|
|
@ -107,6 +107,21 @@ export default function CliPage() {
|
|||
<CodeBlock code="winget install starkSV.msdl" />
|
||||
</div>
|
||||
|
||||
{/* Homebrew — macOS/Linux primary */}
|
||||
<div className="rounded-xl border border-emerald-500/20 bg-emerald-500/5 p-4 space-y-2.5">
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-[11px] font-mono font-semibold tracking-widest uppercase text-emerald-500">macOS / Linux · Homebrew</span>
|
||||
<span className="text-[10px] px-1.5 py-0.5 rounded border border-emerald-500/20 text-emerald-600 font-mono">recommended</span>
|
||||
</div>
|
||||
<CodeBlock code="brew tap starkSV/msdl && brew install msdl-cli" />
|
||||
</div>
|
||||
|
||||
{/* No Homebrew — curl installer */}
|
||||
<div className="rounded-xl border border-white/7 bg-[#111113] p-4 space-y-2">
|
||||
<p className="text-[12px] text-zinc-400">No Homebrew? One-line installer (auto-detects OS/arch, incl. Termux):</p>
|
||||
<CodeBlock code="curl -fsSL https://api.msdl.tech-latest.com/install.sh | bash" />
|
||||
</div>
|
||||
|
||||
{/* Manual / other platforms */}
|
||||
<div className="rounded-xl border border-white/7 bg-[#111113] p-4 space-y-3">
|
||||
<div className="flex items-center justify-between">
|
||||
|
|
@ -128,7 +143,8 @@ export default function CliPage() {
|
|||
{ os: 'Windows (manual)', file: 'msdl-windows-amd64.exe', rename: 'msdl.exe' },
|
||||
{ os: 'macOS (Apple Silicon)', file: 'msdl-darwin-arm64', rename: 'msdl' },
|
||||
{ os: 'macOS (Intel)', file: 'msdl-darwin-amd64', rename: 'msdl' },
|
||||
{ os: 'Linux', file: 'msdl-linux-amd64', rename: 'msdl' },
|
||||
{ os: 'Linux (x86_64)', file: 'msdl-linux-amd64', rename: 'msdl' },
|
||||
{ os: 'Linux (ARM64 / Termux)', file: 'msdl-linux-arm64', rename: 'msdl' },
|
||||
].map(p => (
|
||||
<div key={p.os} className="px-3 py-2.5 rounded-lg bg-white/3 border border-white/5 space-y-1">
|
||||
<p className="text-zinc-500">{p.os}</p>
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ export default function DisclaimerPage() {
|
|||
>
|
||||
<div>
|
||||
<h1 className="text-2xl font-bold text-white mb-1">Disclaimer</h1>
|
||||
<p className="text-xs font-mono text-zinc-600">Last updated: May 18, 2026</p>
|
||||
<p className="text-xs font-mono text-zinc-600">Last updated: July 16, 2026</p>
|
||||
</div>
|
||||
|
||||
<div className="p-4 rounded-xl border border-amber-500/15 bg-amber-500/6 text-amber-400/80 text-[12px] leading-relaxed">
|
||||
|
|
@ -43,7 +43,9 @@ export default function DisclaimerPage() {
|
|||
<p>
|
||||
MSDL is an independent, open-source project. It is not produced, approved, or supported
|
||||
by Microsoft Corporation. The name "Windows" and the Windows logo are registered
|
||||
trademarks of Microsoft Corporation.
|
||||
trademarks of Microsoft Corporation. This applies equally to the website and the{' '}
|
||||
<code className="text-zinc-400 text-[11px] font-mono bg-white/5 px-1 py-0.5 rounded">msdl</code>{' '}
|
||||
command-line tool.
|
||||
</p>
|
||||
</Section>
|
||||
|
||||
|
|
@ -76,8 +78,9 @@ export default function DisclaimerPage() {
|
|||
<Section title="Open source">
|
||||
<p>
|
||||
The source code is publicly available. You are free to inspect, fork, and self-host
|
||||
this project under the terms of its open-source license. The project credits the
|
||||
open-source Fido script by Pete Batard for the underlying session flow.
|
||||
this project under the terms of its open-source license. See the{' '}
|
||||
<a href="/about" className="text-blue-400 hover:text-blue-300 transition-colors">About page</a>{' '}
|
||||
for credits.
|
||||
</p>
|
||||
</Section>
|
||||
</motion.div>
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ const featured = [
|
|||
name: 'Windows 10',
|
||||
version: '22H2',
|
||||
build: '19045.2965',
|
||||
description: 'The final Windows 10 feature update. Security support until October 2025.',
|
||||
description: 'The final Windows 10 feature update. Support ended Oct 2025; ESU through Oct 2026.',
|
||||
badge: 'eol' as const,
|
||||
archs: ['x64', 'x86'],
|
||||
},
|
||||
|
|
@ -121,7 +121,7 @@ export default function HomePage() {
|
|||
<p className="text-zinc-400 text-base max-w-sm mx-auto mb-6 leading-relaxed">
|
||||
Direct links from Microsoft's CDN. No ads. No registration.
|
||||
</p>
|
||||
<div className="flex items-center justify-center gap-4 text-[12px] text-zinc-500">
|
||||
<div className="flex items-center justify-center flex-wrap gap-x-4 gap-y-1.5 text-[12px] text-zinc-500">
|
||||
<span className="flex items-center gap-1.5">
|
||||
<span className="w-1.5 h-1.5 rounded-full bg-green-500 inline-block" />
|
||||
Always official
|
||||
|
|
@ -136,6 +136,16 @@ export default function HomePage() {
|
|||
<span className="w-1.5 h-1.5 rounded-full bg-zinc-600 inline-block" />
|
||||
Free forever
|
||||
</span>
|
||||
<span className="text-zinc-700">·</span>
|
||||
<a
|
||||
href="https://github.com/starkSV/windows-iso-downloader"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="flex items-center gap-1.5 hover:text-zinc-300 transition-colors"
|
||||
>
|
||||
<span className="w-1.5 h-1.5 rounded-full bg-purple-500 inline-block" />
|
||||
<span className="underline decoration-dotted decoration-zinc-600 underline-offset-4">Open source</span>
|
||||
</a>
|
||||
</div>
|
||||
</motion.div>
|
||||
|
||||
|
|
|
|||
|
|
@ -31,14 +31,37 @@ export default function PrivacyPolicyPage() {
|
|||
>
|
||||
<div>
|
||||
<h1 className="text-2xl font-bold text-white mb-1">Privacy Policy</h1>
|
||||
<p className="text-xs font-mono text-zinc-600">Last updated: May 18, 2026</p>
|
||||
<p className="text-xs font-mono text-zinc-600">Last updated: July 16, 2026</p>
|
||||
</div>
|
||||
|
||||
<Section title="No data collected">
|
||||
<Section title="No data collected on the website">
|
||||
<p>
|
||||
MSDL does not collect, store, log, or share any personally identifiable information.
|
||||
There are no user accounts, no cookies, no analytics trackers, and no third-party
|
||||
advertising scripts of any kind.
|
||||
The MSDL website does not collect, store, log, or share any personally identifiable
|
||||
information. There are no user accounts, no cookies, no analytics trackers, and no
|
||||
third-party advertising scripts of any kind.
|
||||
</p>
|
||||
</Section>
|
||||
|
||||
<Section title="CLI tool: anonymous telemetry and link sharing">
|
||||
<p>
|
||||
The <code className="text-zinc-400 text-[11px] font-mono bg-white/5 px-1 py-0.5 rounded">msdl</code>{' '}
|
||||
CLI is a separate, optional download and sends two kinds of anonymous data by default,
|
||||
both of which you can disable:
|
||||
</p>
|
||||
<p>
|
||||
<strong className="text-zinc-400">Usage telemetry</strong> — action type (fetch, eval, list,
|
||||
interactive), platform (Windows/macOS/Linux), CLI version, and whether the run succeeded
|
||||
or failed. No personal data, IP address, or product-key information is included. Disable
|
||||
with <code className="text-zinc-400 text-[11px] font-mono bg-white/5 px-1 py-0.5 rounded">--no-telemetry</code>{' '}
|
||||
or the <code className="text-zinc-400 text-[11px] font-mono bg-white/5 px-1 py-0.5 rounded">MSDL_NO_TELEMETRY=1</code> environment variable.
|
||||
</p>
|
||||
<p>
|
||||
<strong className="text-zinc-400">Link contribution</strong> — after a successful fetch, the
|
||||
CLI can share the product ID, SKU ID, and Microsoft's raw signed-link response back to our
|
||||
backend, so the next website visitor gets a cached hit instead of a fresh Microsoft request.
|
||||
Only that Microsoft response is shared — nothing about you or your machine. Disable with{' '}
|
||||
<code className="text-zinc-400 text-[11px] font-mono bg-white/5 px-1 py-0.5 rounded">--no-contribute</code>{' '}
|
||||
or <code className="text-zinc-400 text-[11px] font-mono bg-white/5 px-1 py-0.5 rounded">MSDL_NO_CONTRIBUTE=1</code>.
|
||||
</p>
|
||||
</Section>
|
||||
|
||||
|
|
|
|||
42
issues.md
42
issues.md
|
|
@ -1,42 +0,0 @@
|
|||
# Issue Drafts
|
||||
|
||||
## Inactive products still trigger backend requests
|
||||
|
||||
### Summary
|
||||
|
||||
The new `active: false` catalog state is not fully enforced on the product detail page.
|
||||
|
||||
Inactive products still pass validation and trigger `/skuinfo` requests before the UI shows the discontinued-product panel.
|
||||
|
||||
### Affected file
|
||||
|
||||
- `frontend/src/pages/ProductDetailPage.tsx`
|
||||
|
||||
### Current behavior
|
||||
|
||||
For a catalog entry with `active: false`:
|
||||
|
||||
- the page loads the product from `products.json`
|
||||
- `setIsValidated(true)` still runs
|
||||
- the follow-up effect calls `/skuinfo?product_id=...`
|
||||
- the backend and Microsoft endpoints are still hit
|
||||
- only afterward does the page render the `Product Discontinued` state
|
||||
|
||||
### Expected behavior
|
||||
|
||||
Inactive products should fail fast on the frontend and should not trigger backend or Microsoft API requests.
|
||||
|
||||
### Risk
|
||||
|
||||
The inactive-product flow still consumes backend capacity and external request budget even though the user cannot download that release.
|
||||
|
||||
### Suggested fix
|
||||
|
||||
- gate validation on `product.active !== false`
|
||||
- avoid setting `isValidated` for inactive products
|
||||
- ensure the `/skuinfo` effect only runs for active products
|
||||
|
||||
### Prevention
|
||||
|
||||
- add a test covering an inactive catalog entry and assert that no backend request is made
|
||||
- model the route state explicitly so `active`, `not_found`, and `error` paths cannot fall through into fetch logic
|
||||
91
scripts/check-new-releases/README.md
Normal file
91
scripts/check-new-releases/README.md
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
# check-new-releases
|
||||
|
||||
Two complementary tools for finding Windows product edition IDs that exist on
|
||||
Microsoft's side but aren't yet in `cli/catalog.go` / `products.json` /
|
||||
`validContributeProducts`. Neither touches those files automatically --
|
||||
naming a new entry correctly (build number, refresh qualifiers like "V2",
|
||||
etc.) still needs a human checking Microsoft's own release-health pages.
|
||||
Both only *discover* candidates.
|
||||
|
||||
Only Windows 11 is worth checking at all: Windows 8.1 is fully frozen (no
|
||||
updates since Jan 2023) and Windows 10 is past end-of-life (ESU security
|
||||
patches only, delivered via Windows Update, not new consumer ISOs). Neither
|
||||
will ever produce a new release again.
|
||||
|
||||
## Method 1 -- `msdls_v3.py` (brute-force range scan)
|
||||
|
||||
Checks every ID in a range you specify, one by one, against Microsoft's
|
||||
SKU-info API. Use this when you already have a rough idea of where to look
|
||||
(e.g. "the last release was around 3260, check up to 3350") or want to
|
||||
directly merge results into a JSON catalog file.
|
||||
|
||||
```bash
|
||||
pip install requests
|
||||
python msdls_v3.py --first 3320 --last 3330
|
||||
```
|
||||
|
||||
Optionally write results to a JSON file (merges with an existing one if
|
||||
present, preserving fields like `badge`/`archs`/`related`, marking IDs that
|
||||
stopped responding as `"active": false`):
|
||||
|
||||
```bash
|
||||
python msdls_v3.py --first 3320 --last 3330 --write scan-results.json
|
||||
```
|
||||
|
||||
Don't point `--write` directly at `frontend/public/data/products.json` --
|
||||
write to a scratch file first and review before merging into the real
|
||||
catalog.
|
||||
|
||||
**How it identifies a release:** reads `ProductDisplayName` from the first
|
||||
SKU in the API response (e.g. `"Windows 11 25H2__V2"`). This field name was
|
||||
wrong in an earlier version of this script (checked `EditionName`/
|
||||
`ReleaseName`/`FriendlyName`, none of which exist in the real response) --
|
||||
confirmed live and fixed 2026-07-14.
|
||||
|
||||
**Session handling:** registers the session via `vlscppe`'s permit endpoint
|
||||
before scanning. A single SKU lookup was confirmed to work without this, but
|
||||
this script's use case -- many rapid consecutive requests across a range --
|
||||
is a much higher-risk pattern for triggering a Sentinel block than a
|
||||
one-shot lookup, so the permit call is worth keeping. The CLI/backend's full
|
||||
session flow additionally replays an `ov-df.microsoft.com` fingerprint; add
|
||||
that too if the permit call alone isn't enough under real scanning load.
|
||||
|
||||
## Method 2 -- `main.go` (auto-discovery)
|
||||
|
||||
Scrapes Microsoft's public `/windows11` download page for the *current*
|
||||
flagship product edition ID (no range-guessing needed), compares it against
|
||||
`cli/catalog.go`, and if it's new, probes a bounded range of adjacent IDs to
|
||||
discover the accompanying variant family (Home China, Pro China, ARM64).
|
||||
Use this when you just want to know "is there anything new" without knowing
|
||||
where to look.
|
||||
|
||||
```bash
|
||||
cd scripts/check-new-releases
|
||||
go run .
|
||||
```
|
||||
|
||||
If a new release is found, it makes real requests to Microsoft to probe the
|
||||
variant family -- let it finish rather than re-running immediately.
|
||||
|
||||
Flags:
|
||||
- `-catalog <path>` — path to `cli/catalog.go` (default: `../../cli/catalog.go`)
|
||||
- `-probe-range <n>` — how many adjacent IDs to check after a new flagship is found (default: 20)
|
||||
|
||||
**Known limitation:** the ARM64 family's offset from the flagship isn't
|
||||
consistent release to release (observed: +3 for the 25H2 family, +18 for
|
||||
24H2's). The default probe range of 20 covers both known cases, but if no
|
||||
ARM64 variant shows up, try a wider `-probe-range`.
|
||||
|
||||
## Which one to use
|
||||
|
||||
| | Method 1 (Python) | Method 2 (Go) |
|
||||
|---|---|---|
|
||||
| Need to guess a starting range? | Yes | No -- finds the current flagship automatically |
|
||||
| Gets the real Microsoft name? | Yes (`ProductDisplayName`) | Yes (`ProductDisplayName`) |
|
||||
| Can write results to a catalog-shaped JSON? | Yes (`--write`) | No -- prints a report only |
|
||||
| Best for | "I want to scan a specific range and get a mergeable file" | "Just tell me if anything's new" |
|
||||
|
||||
Both were verified end-to-end against live Microsoft endpoints on
|
||||
2026-07-14, which surfaced a real gap: product IDs `3322`/`3323`/`3325`/`3326`
|
||||
(Home/Pro China variants of the 25H2 "V2" refresh) existed on Microsoft's
|
||||
side but weren't yet in the catalog. Since fixed.
|
||||
3
scripts/check-new-releases/go.mod
Normal file
3
scripts/check-new-releases/go.mod
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
module check-new-releases
|
||||
|
||||
go 1.25.6
|
||||
321
scripts/check-new-releases/main.go
Normal file
321
scripts/check-new-releases/main.go
Normal file
|
|
@ -0,0 +1,321 @@
|
|||
// check-new-releases scrapes Microsoft's public Windows 11 download page for
|
||||
// the current "flagship" product edition ID and compares it against the
|
||||
// catalog already tracked in cli/catalog.go. Windows 8.1 and Windows 10 are
|
||||
// intentionally not checked here -- 8.1 is fully frozen and Windows 10 is
|
||||
// past end-of-life, so neither will ever produce a new consumer ISO release.
|
||||
//
|
||||
// If a new flagship ID is found, it probes a bounded range of adjacent IDs to
|
||||
// discover the accompanying variant family (Home China, Pro China, ARM64,
|
||||
// ...), since Microsoft allocates those in a cluster near the flagship ID
|
||||
// but the exact offset isn't consistent release to release (observed gaps:
|
||||
// +3 for the 25H2 family, +18 for 24H2's ARM64 family).
|
||||
//
|
||||
// This tool only *discovers* candidate IDs -- it does not touch cli/catalog.go,
|
||||
// products.json, or validContributeProducts automatically. Naming a new entry
|
||||
// correctly (build number, "Updated Oct"-style qualifiers, etc.) still needs a
|
||||
// human checking Microsoft's own release-health pages.
|
||||
//
|
||||
// Usage: go run . [-catalog path/to/cli/catalog.go] [-probe-range N]
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// --- Copied from cli/microsoft.go (not imported -- cli/ is package main and
|
||||
// can't be imported from elsewhere). This is a standalone maintenance tool,
|
||||
// not part of the shipped CLI/backend; keep in sync manually if the real
|
||||
// session flow ever changes. ---
|
||||
|
||||
const (
|
||||
msUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
msProfile = "606624d44113"
|
||||
msLocale = "en-US"
|
||||
msOrgID = "y6jn8c31"
|
||||
msCustomer = "560dc9f3-1aa5-4a2f-b63c-9e18f8d0e175"
|
||||
)
|
||||
|
||||
var (
|
||||
reW = regexp.MustCompile(`[&?]w=([^&"'\s]+)`)
|
||||
reRt = regexp.MustCompile(`rticks[="]+\+?\s*(\d{10,})`)
|
||||
)
|
||||
|
||||
type skuLang struct {
|
||||
ID string `json:"Id"`
|
||||
Language string `json:"Language"`
|
||||
ProductDisplayName string `json:"ProductDisplayName"`
|
||||
}
|
||||
|
||||
type simpleCookieJar struct {
|
||||
mu sync.Mutex
|
||||
cookies []*http.Cookie
|
||||
}
|
||||
|
||||
func (j *simpleCookieJar) SetCookies(_ *url.URL, cookies []*http.Cookie) {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
for _, c := range cookies {
|
||||
found := false
|
||||
for i, existing := range j.cookies {
|
||||
if existing.Name == c.Name {
|
||||
j.cookies[i] = c
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
j.cookies = append(j.cookies, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (j *simpleCookieJar) Cookies(_ *url.URL) []*http.Cookie {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
out := make([]*http.Cookie, len(j.cookies))
|
||||
copy(out, j.cookies)
|
||||
return out
|
||||
}
|
||||
|
||||
func newSessionID() string {
|
||||
b := make([]byte, 16)
|
||||
rand.Read(b)
|
||||
b[6] = (b[6] & 0x0f) | 0x40
|
||||
b[8] = (b[8] & 0x3f) | 0x80
|
||||
return fmt.Sprintf("%s-%s-%s-%s-%s",
|
||||
hex.EncodeToString(b[0:4]), hex.EncodeToString(b[4:6]),
|
||||
hex.EncodeToString(b[6:8]), hex.EncodeToString(b[8:10]),
|
||||
hex.EncodeToString(b[10:16]))
|
||||
}
|
||||
|
||||
func newSession() (*http.Client, string) {
|
||||
sessionID := newSessionID()
|
||||
jar := &simpleCookieJar{}
|
||||
client := &http.Client{Timeout: 15 * time.Second, Jar: jar}
|
||||
|
||||
q1 := url.Values{}
|
||||
q1.Set("org_id", msOrgID)
|
||||
q1.Set("session_id", sessionID)
|
||||
req1, _ := http.NewRequest("GET", "https://vlscppe.microsoft.com/tags?"+q1.Encode(), nil)
|
||||
req1.Header.Set("User-Agent", msUA)
|
||||
client.Do(req1)
|
||||
|
||||
q2 := url.Values{}
|
||||
q2.Set("instanceId", msCustomer)
|
||||
q2.Set("PageId", "si")
|
||||
q2.Set("session_id", sessionID)
|
||||
req2, _ := http.NewRequest("GET", "https://ov-df.microsoft.com/mdt.js?"+q2.Encode(), nil)
|
||||
req2.Header.Set("User-Agent", msUA)
|
||||
resp2, err := client.Do(req2)
|
||||
if err != nil {
|
||||
return client, sessionID
|
||||
}
|
||||
body, _ := io.ReadAll(resp2.Body)
|
||||
resp2.Body.Close()
|
||||
|
||||
wMatch := reW.FindStringSubmatch(string(body))
|
||||
rtMatch := reRt.FindStringSubmatch(string(body))
|
||||
if len(wMatch) > 1 && len(rtMatch) > 1 {
|
||||
q3 := url.Values{}
|
||||
q3.Set("session_id", sessionID)
|
||||
q3.Set("CustomerId", msCustomer)
|
||||
q3.Set("PageId", "si")
|
||||
q3.Set("w", wMatch[1])
|
||||
q3.Set("mdt", fmt.Sprintf("%d", time.Now().UnixMilli()))
|
||||
q3.Set("rticks", rtMatch[1])
|
||||
req3, _ := http.NewRequest("GET", "https://ov-df.microsoft.com/?"+q3.Encode(), nil)
|
||||
req3.Header.Set("User-Agent", msUA)
|
||||
client.Do(req3)
|
||||
}
|
||||
return client, sessionID
|
||||
}
|
||||
|
||||
type msErrorEntry struct {
|
||||
Type float64 `json:"Type"`
|
||||
Value string `json:"Value"`
|
||||
}
|
||||
|
||||
func firstError(errs []msErrorEntry) string {
|
||||
if len(errs) == 0 {
|
||||
return ""
|
||||
}
|
||||
if int(errs[0].Type) == 9 {
|
||||
if errs[0].Value != "" {
|
||||
return errs[0].Value
|
||||
}
|
||||
return "Your IP has been temporarily blocked by Microsoft (Code 715-123130)"
|
||||
}
|
||||
if errs[0].Value != "" {
|
||||
return errs[0].Value
|
||||
}
|
||||
return "Microsoft API error"
|
||||
}
|
||||
|
||||
func fetchLanguages(client *http.Client, sessionID, productID string) ([]skuLang, error) {
|
||||
q := url.Values{}
|
||||
q.Set("profile", msProfile)
|
||||
q.Set("productEditionId", productID)
|
||||
q.Set("SKU", "undefined")
|
||||
q.Set("friendlyFileName", "undefined")
|
||||
q.Set("Locale", msLocale)
|
||||
q.Set("sessionID", sessionID)
|
||||
|
||||
req, _ := http.NewRequest("GET", "https://www.microsoft.com/software-download-connector/api/getskuinformationbyproductedition?"+q.Encode(), nil)
|
||||
req.Header.Set("User-Agent", msUA)
|
||||
req.Header.Set("Referer", "https://www.microsoft.com/en-us/software-download/windows11")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("Microsoft returned HTTP %d", resp.StatusCode)
|
||||
}
|
||||
if len(body) > 0 && body[0] == '"' {
|
||||
var unquoted string
|
||||
json.Unmarshal(body, &unquoted)
|
||||
body = []byte(unquoted)
|
||||
}
|
||||
|
||||
var data struct {
|
||||
Skus []skuLang `json:"Skus"`
|
||||
Errors []msErrorEntry `json:"Errors"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &data); err != nil {
|
||||
return nil, fmt.Errorf("invalid SKU response: %w", err)
|
||||
}
|
||||
if msg := firstError(data.Errors); msg != "" {
|
||||
return nil, fmt.Errorf("%s", msg)
|
||||
}
|
||||
if len(data.Skus) == 0 {
|
||||
return nil, fmt.Errorf("no languages found for this product")
|
||||
}
|
||||
return data.Skus, nil
|
||||
}
|
||||
|
||||
// --- New logic specific to this tool ---
|
||||
|
||||
var catalogEntryRe = regexp.MustCompile(`\{"(\d+)",\s*"([^"]*)"\}`)
|
||||
|
||||
func knownCatalogIDs(catalogPath string) (map[string]string, error) {
|
||||
data, err := os.ReadFile(catalogPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
matches := catalogEntryRe.FindAllStringSubmatch(string(data), -1)
|
||||
out := make(map[string]string, len(matches))
|
||||
for _, m := range matches {
|
||||
out[m[1]] = m[2]
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
var flagshipRe = regexp.MustCompile(`<option value="(\d+)">Windows`)
|
||||
|
||||
func scrapeFlagshipID(pageURL string) (string, error) {
|
||||
req, _ := http.NewRequest("GET", pageURL, nil)
|
||||
req.Header.Set("User-Agent", msUA)
|
||||
client := &http.Client{Timeout: 15 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("fetching download page: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
m := flagshipRe.FindStringSubmatch(string(body))
|
||||
if len(m) < 2 {
|
||||
return "", fmt.Errorf("could not find a product edition ID on the page -- Microsoft may have changed the page layout")
|
||||
}
|
||||
return m[1], nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
catalogPath := flag.String("catalog", "../../cli/catalog.go", "path to cli/catalog.go")
|
||||
probeRange := flag.Int("probe-range", 20, "how many adjacent IDs to probe after a new flagship is found")
|
||||
flag.Parse()
|
||||
|
||||
const windows11Page = "https://www.microsoft.com/en-us/software-download/windows11"
|
||||
fmt.Printf("Checking %s for the current flagship product edition ID...\n", windows11Page)
|
||||
|
||||
flagshipID, err := scrapeFlagshipID(windows11Page)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Printf("Flagship ID found: %s\n", flagshipID)
|
||||
|
||||
known, err := knownCatalogIDs(*catalogPath)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error reading catalog:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Printf("Loaded %d known product IDs from %s\n\n", len(known), *catalogPath)
|
||||
|
||||
if name, ok := known[flagshipID]; ok {
|
||||
fmt.Printf("Up to date -- flagship ID %s is already in the catalog as %q\n", flagshipID, name)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("NEW RELEASE DETECTED -- product edition ID %s is not in the current catalog.\n\n", flagshipID)
|
||||
fmt.Println("Probing adjacent IDs to discover the variant family (this makes real")
|
||||
fmt.Println("requests to Microsoft -- allow it to finish rather than re-running):")
|
||||
fmt.Println()
|
||||
|
||||
client, sessionID := newSession()
|
||||
idNum, err := strconv.Atoi(flagshipID)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error: flagship ID is not numeric:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fmt.Printf("%-8s %-8s %s\n", "ID", "STATUS", "NOTES")
|
||||
for offset := 0; offset <= *probeRange; offset++ {
|
||||
candidate := strconv.Itoa(idNum + offset)
|
||||
langs, err := fetchLanguages(client, sessionID, candidate)
|
||||
time.Sleep(300 * time.Millisecond) // be a good citizen -- don't hammer Microsoft
|
||||
if err != nil {
|
||||
continue // not a valid product ID; skip silently, only report hits
|
||||
}
|
||||
realName := ""
|
||||
if len(langs) > 0 {
|
||||
realName = langs[0].ProductDisplayName
|
||||
}
|
||||
note := fmt.Sprintf("%q -- %d language(s)", realName, len(langs))
|
||||
switch {
|
||||
case len(langs) == 1:
|
||||
note += " -- likely a China-only single-edition variant"
|
||||
case len(langs) > 30:
|
||||
note += " -- likely a worldwide multi-edition variant"
|
||||
}
|
||||
if existingName, ok := known[candidate]; ok {
|
||||
note += fmt.Sprintf(" (already in catalog as %q)", existingName)
|
||||
}
|
||||
fmt.Printf("%-8s %-8s %s\n", candidate, "FOUND", note)
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
fmt.Println("Note: the ARM64 family's offset from the flagship isn't consistent release")
|
||||
fmt.Println("to release (seen +3 for 25H2, +18 for 24H2) -- if no ARM64 variant showed up")
|
||||
fmt.Println("above, try a wider -probe-range.")
|
||||
fmt.Println()
|
||||
fmt.Println("Next steps: confirm the build number from Microsoft's release-health pages,")
|
||||
fmt.Println("then add entries to cli/catalog.go, frontend/public/data/products.json,")
|
||||
fmt.Println("and the validContributeProducts map in backend/main.go.")
|
||||
}
|
||||
|
|
@ -12,8 +12,27 @@ LOCALE = "en-US"
|
|||
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
|
||||
def setup_session():
|
||||
"""Initializes a session with Microsoft tracking servers."""
|
||||
"""Initializes a session with Microsoft tracking servers.
|
||||
|
||||
Registers the session_id via vlscppe's "permit" endpoint before use --
|
||||
confirmed live that getskuinformationbyproductedition still returns valid
|
||||
data without this for a single lookup, but this script's brute-force
|
||||
range scan makes many rapid consecutive requests, which is a much
|
||||
higher-risk pattern for triggering a Sentinel block than the single-shot
|
||||
case that was tested. The CLI/backend's full session flow additionally
|
||||
replays an ov-df.microsoft.com fingerprint; add that too if this alone
|
||||
turns out not to be enough under real scanning load.
|
||||
"""
|
||||
session_id = str(uuid.uuid4())
|
||||
try:
|
||||
requests.get(
|
||||
"https://vlscppe.microsoft.com/tags",
|
||||
params={"org_id": "y6jn8c31", "session_id": session_id},
|
||||
headers={"User-Agent": UA},
|
||||
timeout=10,
|
||||
)
|
||||
except Exception as e:
|
||||
logging.warning(f"Session permit call failed (continuing anyway): {e}")
|
||||
return session_id
|
||||
|
||||
def get_product(product_id, session_id):
|
||||
|
|
@ -66,13 +85,15 @@ def scan_id(product_id):
|
|||
# MS usually puts the release name in the first SKU
|
||||
skus = data.get("Skus", [])
|
||||
if skus:
|
||||
# Check multiple possible name keys
|
||||
s = skus[0]
|
||||
# ReleaseName is often "Windows 10 Version 22H2 (Updated Oct 2025)"
|
||||
name = s.get("EditionName") or s.get("ReleaseName") or s.get("FriendlyName")
|
||||
# Confirmed live 2026-07-14: the real field is "ProductDisplayName"
|
||||
# (e.g. "Windows 11 25H2"), not EditionName/ReleaseName/FriendlyName --
|
||||
# those don't exist in the actual response and always fell through to
|
||||
# the generic fallback below, silently.
|
||||
name = s.get("ProductDisplayName") or s.get("LocalizedProductDisplayName")
|
||||
if name:
|
||||
return name
|
||||
|
||||
|
||||
return f"Windows Product {product_id}"
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
PackageIdentifier: starkSV.msdl
|
||||
PackageVersion: 0.3.1
|
||||
PackageVersion: 0.3.7
|
||||
Platform:
|
||||
- Windows.Desktop
|
||||
MinimumOSVersion: 10.0.0.0
|
||||
|
|
@ -8,7 +8,7 @@ Commands:
|
|||
- msdl
|
||||
Installers:
|
||||
- Architecture: x64
|
||||
InstallerUrl: https://github.com/starkSV/windows-iso-downloader/releases/download/cli%2Fv0.3.1/msdl-windows-amd64.exe
|
||||
InstallerSha256: A1EAB63420395C4E9572E588FFEA49E55A14ACA6403290282C841B8CFEBB7554
|
||||
InstallerUrl: https://github.com/starkSV/windows-iso-downloader/releases/download/cli%2Fv0.3.7/msdl-windows-amd64.exe
|
||||
InstallerSha256: 44E9225BFE12DCE801301C2A68CBC114F57BA3313F775740BFC9A83CD87A431F
|
||||
ManifestType: installer
|
||||
ManifestVersion: 1.6.0
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
PackageIdentifier: starkSV.msdl
|
||||
PackageVersion: 0.3.1
|
||||
PackageVersion: 0.3.7
|
||||
PackageLocale: en-US
|
||||
Publisher: starkSV
|
||||
PublisherUrl: https://github.com/starkSV
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
PackageIdentifier: starkSV.msdl
|
||||
PackageVersion: 0.3.1
|
||||
PackageVersion: 0.3.7
|
||||
DefaultLocale: en-US
|
||||
ManifestType: version
|
||||
ManifestVersion: 1.6.0
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue