From 6704a8863446a50e4d399238aec96911c1b333b9 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 18 Nov 2026 20:08:27 +0100 Subject: [PATCH 1/6] Fetch metadata again for downloads of versions it doesn't list A cargo, pub, conda, RubyGems, Hex or PyPI download looked its publish time up in metadata that may have been cached, within metadata_ttl or stale, before the version was released. Not finding the version, it let the download through cooldown. As for npm (d073fdc), such a copy is now fetched again and the download decided on what upstream lists now; if it can't be fetched, the lookup fails. A copy just read from upstream is not fetched twice, and a version listed without a time is let through as before. Conda's repodata index now keeps untimed files, so they count as listed. --- docs/configuration.md | 4 +- internal/handler/cargo.go | 41 ++-- internal/handler/conda.go | 42 +++- internal/handler/cooldown_times.go | 43 +++- internal/handler/download_cooldown.go | 48 ++++ .../handler/download_cooldown_missing_test.go | 207 ++++++++++++++++++ internal/handler/pub.go | 42 ++-- internal/handler/pypi.go | 12 +- 8 files changed, 377 insertions(+), 62 deletions(-) create mode 100644 internal/handler/download_cooldown_missing_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 72fc008b..aa997f7b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1390,11 +1390,11 @@ Maven caveats: Helm uses each release's `created` time from the repository's `index.yaml`. Releases in cooldown are left out of the index the proxy serves, and their archives aren't served; a release without `created` is let through. Charts pulled over OCI have no publish time and aren't held back. -Cargo, RubyGems, Hex, pub.dev, Conda and Helm check artifact downloads against cooldown, including files cached before the policy changed. When a publish-time lookup fails, an already-cached file is still served; an uncached download fails with 502, or 503 with `Retry-After` when the rate-limit gate holds the request. Conda checks timestamps per file because builds of one version can have different publish times. As in conda, a repodata `timestamp` is read as milliseconds when it is above 253402300799 (the end of year 9999 in seconds) and as seconds otherwise, since some older entries use seconds. +Cargo, RubyGems, Hex, pub.dev, Conda and Helm check artifact downloads against cooldown, including files cached before the policy changed. When a publish-time lookup fails, an already-cached file is still served; an uncached download fails with 502, or 503 with `Retry-After` when the rate-limit gate holds the request. Metadata cached within `metadata_ttl`, or served stale, can predate a release; a download of a version (or, for Conda, a file) it doesn't list fetches it again, as for npm and PyPI, and is decided on what the upstream lists now. If it can't be fetched, the lookup has failed. Helm charts are downloaded by digest and aren't covered by this yet. Conda checks timestamps per file because builds of one version can have different publish times. As in conda, a repodata `timestamp` is read as milliseconds when it is above 253402300799 (the end of year 9999 in seconds) and as seconds otherwise, since some older entries use seconds. Conda repodata and Helm indexes are parsed once per cached document for download checks. With `cache_metadata` off, the parsed index is kept for one minute, including the fetch, so an install can reuse it across files. Conda streams repodata when its compressed copy exceeds `metadata_max_size`, under the same decompressed size bound as the metadata filter. -RubyGems, Hex and PyPI keep their publish times outside the metadata clients read: the RubyGems versions API, the Hex HTTP API (`upstream.hex_api`) and the PyPI JSON API. The proxy fetches them alongside each compact index, package or simple page it filters, and for PyPI downloads. Cooldown fails closed: when it applies to a package and those times can't be fetched or parsed, or the metadata can't be rewritten, the request fails with 502, or with 503 and `Retry-After` while the upstream is rate limiting the proxy, instead of serving the metadata unfiltered. The same goes for npm, Composer and pub.dev metadata that can't be rewritten, and for npm tarball downloads whose publish time can't be read. A PyPI file or npm tarball that is already cached is still served when its publish time can't be read; only a download that would have to be fetched fails. A PyPI release's time is the upload time of its newest file. The proxy stores it when a download looks it up, and serves files of that release that are already cached against the stored time without reading the JSON API again. A file that isn't cached yet is always looked up, since it may have been added to the release since, and the newer time is stored. Packages exempted with a duration of `0` are served as before and don't need the times. A PyPI package the JSON API returns 404 for has no times to go by and is served unfiltered. +RubyGems, Hex and PyPI keep their publish times outside the metadata clients read: the RubyGems versions API, the Hex HTTP API (`upstream.hex_api`) and the PyPI JSON API. The proxy fetches them alongside each compact index, package or simple page it filters, and for PyPI downloads. Cooldown fails closed: when it applies to a package and those times can't be fetched or parsed, or the metadata can't be rewritten, the request fails with 502, or with 503 and `Retry-After` while the upstream is rate limiting the proxy, instead of serving the metadata unfiltered. The same goes for npm, Composer and pub.dev metadata that can't be rewritten, and for npm tarball downloads whose publish time can't be read. A PyPI file or npm tarball that is already cached is still served when its publish time can't be read; only a download that would have to be fetched fails. A PyPI release's time is the upload time of its newest file. The proxy stores it when a download looks it up, and serves files of that release that are already cached against the stored time without reading the JSON API again. A file that isn't cached yet is always looked up, since it may have been added to the release since, and the newer time is stored. A JSON API document cached before the release doesn't list it, and is fetched again. Packages exempted with a duration of `0` are served as before and don't need the times. A PyPI package the JSON API returns 404 for has no times to go by and is served unfiltered. For RubyGems and Hex, both documents go through the metadata cache. With [`cache_metadata`](#metadata-caching) they are served from it within `metadata_ttl` without asking the upstream, revalidated with their ETags after it, and filtered from the last cached copies, marked stale, while the upstream is down; without it they are still fetched once for concurrent requests. Each document has its own TTL, so the compact index or Hex package can be newer than the cached times and list a version they don't have. The times are then fetched again before filtering, and if that fails, such versions are left out until the times can be read. The filtered index or package carries an ETag of its own, so Bundler and Mix revalidate it with `304 Not Modified`. Publish times are compared with the current time on each request, and the filtered result is kept until the first version it held back leaves cooldown (see [rewritten metadata cache](#rewritten-metadata-cache)); a newly published version shows up within `metadata_ttl`, as with other cached metadata. Upstream errors are answered as for other cached metadata (404, or 502) rather than relayed. diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index a144d3a4..19897194 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -194,28 +194,29 @@ func (h *CargoHandler) applyCooldownFiltering(downstreamResponse http.ResponseWr // publishTime reads a crate version's pubtime from the sparse index, sharing // the index cache with handleIndex. Entries published before crates.io -// recorded pubtime have none. +// recorded pubtime have none. A cached index that doesn't list the version +// is fetched again (readListed). func (h *CargoHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { upstreamURL := fmt.Sprintf("%s/%s", h.indexURL, h.buildIndexPath(name)) - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "cargo", name, upstreamURL, "text/plain") - if err != nil { - return time.Time{}, false, err - } - for line := range bytes.SplitSeq(body, []byte("\n")) { - if len(bytes.TrimSpace(line)) == 0 { - continue - } - var entry crateIndexEntry - if err := json.Unmarshal(line, &entry); err != nil { - return time.Time{}, false, err - } - if entry.Version != version { - continue - } - published, err := time.Parse(time.RFC3339, entry.PublishTime) - return published, err == nil, nil - } - return time.Time{}, false, nil + listed, err := readListed(r.Context(), h.proxy, "cargo", name, upstreamURL, "text/plain", "", + func(res metadataResult) (listedTime, bool, error) { + for line := range bytes.SplitSeq(res.body, []byte("\n")) { + if len(bytes.TrimSpace(line)) == 0 { + continue + } + var entry crateIndexEntry + if err := json.Unmarshal(line, &entry); err != nil { + return listedTime{}, false, err + } + if entry.Version != version { + continue + } + published, err := time.Parse(time.RFC3339, entry.PublishTime) + return listedTime{published, err == nil}, true, nil + } + return listedTime{}, false, nil + }) + return listed.published, listed.ok, err } // buildIndexPath builds the sparse index path for a crate name. diff --git a/internal/handler/conda.go b/internal/handler/conda.go index c9db7a7b..44b21ef4 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -552,25 +552,44 @@ func (h *CondaHandler) metadataCacheKey(urlPath string) string { } // publishTime streams compressed repodata when it exceeds metadata_max_size. +// Repodata that doesn't list the file can predate it, so it is fetched +// again and the file looked up in what upstream lists now (readListed). func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename string) (time.Time, bool, error) { repodataPath := "/" + channel + "/" + arch + "/repodata.json" cacheKey := h.metadataCacheKey(repodataPath) - index, err := publishIndex(h.proxy, "conda\x00"+cacheKey, func() metadataResult { - return h.proxy.lookupMetadata(r.Context(), "conda", cacheKey, h.upstreamURL+repodataPath, "*/*", "gzip", nil) - }, func(res metadataResult) (map[string]time.Time, error) { + upstreamURL := h.upstreamURL + repodataPath + build := func(res metadataResult) (map[string]time.Time, error) { if errors.Is(res.err, ErrMetadataTooLarge) { - return h.streamRepodataIndex(r, h.upstreamURL+repodataPath) + return h.streamRepodataIndex(r, upstreamURL) } if res.err != nil { return nil, res.err } return h.indexRepodata(res.body, res.contentEncoding) - }) + } + // loaded is what the index was built from, unless an index built + // earlier was reused. + var loaded *metadataResult + index, err := publishIndex(h.proxy, "conda\x00"+cacheKey, func() metadataResult { + res := h.proxy.lookupMetadata(r.Context(), "conda", cacheKey, upstreamURL, "*/*", "gzip", nil) + loaded = &res + return res + }, build) + if err != nil { + return time.Time{}, false, err + } + published, listed := index[filename] + if listed || loaded != nil && !loaded.fromCache && !loaded.stale { + // Listed, or just read from upstream, streamed past + // metadata_max_size included. + return published, listed && !published.IsZero(), nil + } + index, err = build(h.proxy.refetchLacking(r.Context(), "conda", cacheKey, upstreamURL, "*/*", "gzip", nil)) if err != nil { return time.Time{}, false, err } - published, ok := index[filename] - return published, ok, nil + published = index[filename] + return published, !published.IsZero(), nil } // indexRepodata indexes a cached repodata.json body. @@ -614,7 +633,7 @@ func (h *CondaHandler) streamRepodataIndex(r *http.Request, upstreamURL string) } // readRepodataIndex maps each package file in a repodata.json to its -// timestamp, leaving out files without one. It reads the document as it +// timestamp, or the zero time for a file without one. It reads the document as it // streams, under the same bound as filterRepodata, so the decompressed size // is not limited by metadata_max_size. func readRepodataIndex(r io.Reader) (map[string]time.Time, error) { @@ -629,7 +648,12 @@ func readRepodataIndex(r io.Reader) (map[string]time.Time, error) { return err } var e repodataEntry - if json.Unmarshal(raw, &e) == nil && e.Timestamp != 0 { + if json.Unmarshal(raw, &e) != nil { + return nil + } + // A file without a timestamp is listed with the zero time. + index[entry.Key] = time.Time{} + if e.Timestamp != 0 { index[entry.Key] = condaTime(e.Timestamp) } return nil diff --git a/internal/handler/cooldown_times.go b/internal/handler/cooldown_times.go index fa9120da..dbb08a54 100644 --- a/internal/handler/cooldown_times.go +++ b/internal/handler/cooldown_times.go @@ -17,24 +17,39 @@ import ( // through. type publishTimes map[string]time.Time +// lookupPublishTime reads version's publish time for a download. A +// document clients read, cached within metadata_ttl or served stale, that +// doesn't list the version can predate it, so both documents are fetched +// again, as readListed does. One that lists it brought the times document up +// to date already (fetchTimedMetadata). func (p *Proxy) lookupPublishTime(ctx context.Context, t timedMetadata, version string, listed func(metadataResult) ([]string, error)) (time.Time, bool, bool, error) { + published, ok, withholdUnknown, docs, err := p.readPublishTime(ctx, t, version, listed, false) + if err != nil || ok || withholdUnknown || slices.Contains(docs.versions, version) || !docs.doc.fromCache && !docs.doc.stale { + return published, ok, withholdUnknown, err + } + published, ok, withholdUnknown, _, err = p.readPublishTime(ctx, t, version, listed, true) + return published, ok, withholdUnknown, err +} + +func (p *Proxy) readPublishTime(ctx context.Context, t timedMetadata, version string, listed func(metadataResult) ([]string, error), refetch bool) (time.Time, bool, bool, timedDocuments, error) { var versions []string - docs, err := p.fetchTimedMetadata(ctx, t, func(doc metadataResult) ([]string, error) { + docs, err := p.fetchTimedMetadataFor(ctx, t, func(doc metadataResult) ([]string, error) { var err error versions, err = listed(doc) return versions, err - }) + }, refetch) + docs.versions = versions switch { case docs.doc.err != nil: - return time.Time{}, false, false, docs.doc.err + return time.Time{}, false, false, docs, docs.doc.err case docs.times.err != nil: - return time.Time{}, false, false, docs.times.err + return time.Time{}, false, false, docs, docs.times.err case err != nil: - return time.Time{}, false, false, err + return time.Time{}, false, false, docs, err } published, ok := docs.published[version] withholdUnknown := !ok && docs.withholdUnknown && slices.Contains(versions, version) - return published, ok && !published.IsZero(), withholdUnknown, nil + return published, ok && !published.IsZero(), withholdUnknown, docs, nil } // timedMetadata describes metadata whose publish times live in a second @@ -62,6 +77,8 @@ type timedDocuments struct { // doesn't, and that document is a stale copy: those versions are newer // than the copy, so they are left out rather than let through. withholdUnknown bool + // versions are the versions doc lists, as readPublishTime read them. + versions []string } // fetchTimedMetadata fetches both documents concurrently. The document @@ -75,6 +92,12 @@ type timedDocuments struct { // from a times document read from the cache, the times document is fetched // again before filtering. func (p *Proxy) fetchTimedMetadata(ctx context.Context, t timedMetadata, listed func(metadataResult) ([]string, error)) (timedDocuments, error) { + return p.fetchTimedMetadataFor(ctx, t, listed, false) +} + +// fetchTimedMetadataFor is fetchTimedMetadata, fetching both documents again +// from upstream when refetch is set (refetchLacking). +func (p *Proxy) fetchTimedMetadataFor(ctx context.Context, t timedMetadata, listed func(metadataResult) ([]string, error), refetch bool) (timedDocuments, error) { var res timedDocuments var published publishTimes validate := func(body []byte) error { @@ -86,11 +109,15 @@ func (p *Proxy) fetchTimedMetadata(ctx context.Context, t timedMetadata, listed } done := make(chan struct{}) + lookup := p.lookupMetadata + if refetch { + lookup = p.refetchLacking + } go func() { defer close(done) - res.times = p.lookupMetadata(ctx, t.ecosystem, t.timesKey, t.timesURL, t.timesAccept, "", validate) + res.times = lookup(ctx, t.ecosystem, t.timesKey, t.timesURL, t.timesAccept, "", validate) }() - res.doc = p.lookupMetadata(ctx, t.ecosystem, t.docKey, t.docURL, "*/*", "identity", nil) + res.doc = lookup(ctx, t.ecosystem, t.docKey, t.docURL, "*/*", "identity", nil) <-done if res.doc.err != nil || res.times.err != nil { diff --git a/internal/handler/download_cooldown.go b/internal/handler/download_cooldown.go index 4ede9a8f..11e98279 100644 --- a/internal/handler/download_cooldown.go +++ b/internal/handler/download_cooldown.go @@ -1,6 +1,8 @@ package handler import ( + "context" + "fmt" "net/http" "sync" "time" @@ -157,3 +159,49 @@ func publishIndex[V any](p *Proxy, key string, load func() metadataResult, build } return v.(V), nil } + +// listedTime is a version's publish time as a metadata document gives it. +// ok is false when the document lists the version without a usable time. +type listedTime struct { + published time.Time + ok bool +} + +// readListed reads a version's entry from a metadata document fetched +// through the metadata cache. read reports whether the document lists the +// version at all. A copy cached within metadata_ttl, or served stale, can +// predate the version, and a download of it would find no publish time and +// be let through cooldown; such a copy is fetched again and the version read +// from what upstream lists now. If it can't be fetched, the lookup fails, as +// when a publish time can't be read. +func readListed[V any](ctx context.Context, p *Proxy, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding string, read func(metadataResult) (V, bool, error)) (V, error) { //nolint:ireturn // V is what read returns. + res := p.lookupMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, nil) + if res.err != nil { + var zero V + return zero, res.err + } + value, listed, err := read(res) + if err != nil || listed || !res.fromCache && !res.stale { + return value, err + } + res = p.refetchLacking(ctx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, nil) + if res.err != nil { + var zero V + return zero, res.err + } + value, _, err = read(res) + return value, err +} + +// refetchLacking fetches a cached metadata document again from upstream, +// for a download of a version the cached copy doesn't list. A stale copy is +// not upstream's answer, so it is an error. +func (p *Proxy) refetchLacking(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding string, validate func([]byte) error) metadataResult { + p.Logger.Info("cooldown: cached metadata lacks the version, fetching it again", + "ecosystem", ecosystem, "url", upstreamURL) + res := p.refreshMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate) + if res.err == nil && res.stale { + res.err = fmt.Errorf("%s metadata could not be fetched again: %w", ecosystem, res.upstreamErr) + } + return res +} diff --git a/internal/handler/download_cooldown_missing_test.go b/internal/handler/download_cooldown_missing_test.go new file mode 100644 index 00000000..af50f770 --- /dev/null +++ b/internal/handler/download_cooldown_missing_test.go @@ -0,0 +1,207 @@ +package handler + +import ( + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries/fetch" +) + +// TestDownloadCooldownRefetchesMetadataLackingTheVersion caches each +// ecosystem's metadata before a release, within metadata_ttl. A download of +// the new release then has no publish time in the cached copy; the metadata +// is fetched again and the download decided on what upstream lists now. If +// that fetch fails, the download is refused as when no time can be read. +func TestDownloadCooldownRefetchesMetadataLackingTheVersion(t *testing.T) { + now := time.Now().Truncate(time.Second) + before := map[string]time.Time{cooldownOldVersion: now.Add(-30 * 24 * time.Hour), cooldownUntimed: {}} + after := map[string]time.Time{ + cooldownOldVersion: now.Add(-30 * 24 * time.Hour), + cooldownRecentVersion: now.Add(-time.Hour), + cooldownUntimed: {}, + } + + for _, c := range downloadCooldownCases() { + if c.ecosystem == "helm" { + // Charts are downloaded by digest, which a stale index lacks + // differently; not covered here. + continue + } + // expired has the cached copy past metadata_ttl, so it is served + // stale while the upstream is down. + for _, variant := range []struct { + name string + upstreamFails, expired bool + }{{"released", false, false}, {"upstream-down", true, false}, {"stale", true, true}} { + t.Run(c.ecosystem+"/"+variant.name, func(t *testing.T) { + checkRefetch(t, c, before, after, variant.upstreamFails, variant.expired) + }) + } + } +} + +// checkRefetch runs one case of +// TestDownloadCooldownRefetchesMetadataLackingTheVersion. +func checkRefetch(t *testing.T, c downloadCooldownCase, before, after map[string]time.Time, upstreamFails, expired bool) { + t.Helper() + var released, down atomic.Bool + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + switch { + case down.Load(): + http.Error(w, "unavailable", http.StatusServiceUnavailable) + case released.Load(): + c.metadata(w, r, after) + default: + c.metadata(w, r, before) + } + })) + t.Cleanup(upstream.Close) + proxy, _, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + if expired { + proxy.MetadataTTL = time.Nanosecond + } + fetcher.fetchErr = errors.New("artifact must be served from the cache") + proxy.Cooldown = &cooldown.Config{Packages: map[string]string{ + canonicalPackagePURL(c.ecosystem, "demo"): "7d", + }} + handler := c.handler(proxy, upstream.URL) + get := func(version string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, c.path(version), nil)) + return w + } + + // Caches the metadata, which doesn't list the release yet. + c.seedCachedArtifact(t, proxy, cooldownOldVersion) + if w := get(cooldownOldVersion); w.Code != http.StatusOK { + t.Fatalf("old version: status %d, body %q", w.Code, w.Body.String()) + } + // A version listed without a time is not fetched again. + if !expired { + warmed := requests.Load() + c.seedCachedArtifact(t, proxy, cooldownUntimed) + if w := get(cooldownUntimed); w.Code != http.StatusOK || requests.Load() != warmed { + t.Fatalf("untimed version: status %d, %d more upstream requests; want 200 and none", + w.Code, requests.Load()-warmed) + } + } + + released.Store(true) + down.Store(upstreamFails) + if upstreamFails { + // Fetchable, so only the cooldown check stops it. + fetcher.fetchErr = nil + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("artifact"))} + } else { + c.seedCachedArtifact(t, proxy, cooldownRecentVersion) + } + w := get(cooldownRecentVersion) + switch { + case !upstreamFails && w.Code != http.StatusNotFound: + t.Errorf("release in cooldown: status %d, want 404 (body %q)", w.Code, w.Body.String()) + case upstreamFails && w.Code < http.StatusInternalServerError: + t.Errorf("release with upstream down: status %d, want a 5xx (body %q)", w.Code, w.Body.String()) + } +} + +// TestDownloadCooldownFetchesUnlistedVersionOnce checks a download of a +// version that metadata just fetched from upstream doesn't list makes no +// second metadata request. +func TestDownloadCooldownFetchesUnlistedVersionOnce(t *testing.T) { + times := map[string]time.Time{cooldownOldVersion: time.Now().Add(-30 * 24 * time.Hour)} + for _, c := range downloadCooldownCases() { + if c.ecosystem == "helm" { + continue + } + t.Run(c.ecosystem, func(t *testing.T) { + requests := func(version string) int32 { + var n atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n.Add(1) + c.metadata(w, r, times) + })) + defer upstream.Close() + proxy, _, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + fetcher.fetchErr = errors.New("artifact must be served from the cache") + proxy.Cooldown = &cooldown.Config{Packages: map[string]string{ + canonicalPackagePURL(c.ecosystem, "demo"): "7d", + }} + c.seedCachedArtifact(t, proxy, version) + c.handler(proxy, upstream.URL).ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, c.path(version), nil)) + return n.Load() + } + if listed, unlisted := requests(cooldownOldVersion), requests(cooldownRecentVersion); unlisted != listed { + t.Errorf("metadata requests for an unlisted version = %d, for a listed one %d", unlisted, listed) + } + }) + } +} + +// TestPyPIDownloadCooldownRefetchesReleasesLackingTheVersion is +// TestDownloadCooldownRefetchesMetadataLackingTheVersion for PyPI, whose +// release times come from the JSON API. +func TestPyPIDownloadCooldownRefetchesReleasesLackingTheVersion(t *testing.T) { + now := time.Now() + old := `"1.0.0": [{"upload_time_iso_8601": "` + now.Add(-30*24*time.Hour).Format(time.RFC3339) + `"}]` + recent := `"2.0.0": [{"upload_time_iso_8601": "` + now.Add(-time.Hour).Format(time.RFC3339) + `"}]` + for _, upstreamFails := range []bool{false, true} { + t.Run(fmt.Sprintf("upstream-down=%t", upstreamFails), func(t *testing.T) { + var released, down atomic.Bool + var metadataRequests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + metadataRequests.Add(1) + switch { + case down.Load(): + http.Error(w, "unavailable", http.StatusServiceUnavailable) + case released.Load(): + _, _ = io.WriteString(w, `{"releases": {`+old+`, `+recent+`}}`) + default: + _, _ = io.WriteString(w, `{"releases": {`+old+`}}`) + } + })) + defer upstream.Close() + proxy, _, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.Cooldown = &cooldown.Config{Default: "7d"} + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package data"))} + h := &PyPIHandler{proxy: proxy, upstreamURL: upstream.URL, proxyURL: "http://localhost"} + get := func(filename string) int { + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/packages/packages/ab/cd/ef0123456789/"+filename, nil)) + return w.Code + } + + // An unlisted version right after a fetch makes no second one. + if code := get("newpkg-9.0.0.tar.gz"); code != http.StatusOK || metadataRequests.Load() != 1 { + t.Fatalf("unlisted version: status %d, %d metadata requests; want 200 and 1", code, metadataRequests.Load()) + } + released.Store(true) + down.Store(upstreamFails) + code := get("newpkg-2.0.0.tar.gz") + switch { + case !upstreamFails && code != http.StatusNotFound: + t.Errorf("release in cooldown: status %d, want 404", code) + case upstreamFails && code < http.StatusInternalServerError: + t.Errorf("release with upstream down: status %d, want a 5xx", code) + } + }) + } +} diff --git a/internal/handler/pub.go b/internal/handler/pub.go index c2e28429..f6a8b379 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -149,27 +149,27 @@ func (h *PubHandler) discardVersionPage(name, version, filename string) { // sharing the metadata cache with handlePackageMetadata. func (h *PubHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { upstreamURL := fmt.Sprintf("%s/api/packages/%s", h.upstreamURL, name) - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "pub", name, upstreamURL) - if err != nil { - return time.Time{}, false, err - } - var metadata struct { - Versions []struct { - Version string `json:"version"` - Published string `json:"published"` - } `json:"versions"` - } - if err := json.Unmarshal(body, &metadata); err != nil { - return time.Time{}, false, err - } - for _, v := range metadata.Versions { - if v.Version != version { - continue - } - published, err := time.Parse(time.RFC3339, v.Published) - return published, err == nil, nil - } - return time.Time{}, false, nil + listed, err := readListed(r.Context(), h.proxy, "pub", name, upstreamURL, contentTypeJSON, "", + func(res metadataResult) (listedTime, bool, error) { + var metadata struct { + Versions []struct { + Version string `json:"version"` + Published string `json:"published"` + } `json:"versions"` + } + if err := json.Unmarshal(res.body, &metadata); err != nil { + return listedTime{}, false, err + } + for _, v := range metadata.Versions { + if v.Version != version { + continue + } + published, err := time.Parse(time.RFC3339, v.Published) + return listedTime{published, err == nil}, true, nil + } + return listedTime{}, false, nil + }) + return listed.published, listed.ok, err } // handlePackageMetadata proxies package metadata and rewrites archive URLs. diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 29022da7..1ada8cb7 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -489,11 +489,19 @@ func (h *PyPIHandler) versionInCooldown(r *http.Request, name, version string) ( return false, nil } - times, err := h.fetchReleaseTimes(r, name) + jsonURL := fmt.Sprintf("%s/pypi/%s/json", h.upstreamURL, name) + publishedAt, err := readListed(r.Context(), h.proxy, "pypi", name+"/json", jsonURL, contentTypeJSON, "", + func(res metadataResult) (time.Time, bool, error) { + times, err := h.parseReleaseTimes(res.body) + published, listed := times[version] + return published, listed, err + }) + if errors.Is(err, ErrUpstreamNotFound) { + return false, nil + } if err != nil { return false, err } - publishedAt := times[version] if publishedAt.IsZero() { return false, nil } From e0214f1fc572ab807605c376f090668c2fbcf6f6 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 18 Nov 2026 20:26:13 +0100 Subject: [PATCH 2/6] Keep the conda index fetched again for a newly released file Without the metadata cache, or for repodata past metadata_max_size, the index publishIndex kept for a minute still lacked the file, so every download of it fetched and parsed the repodata again until that entry expired. The refetched index now replaces it. --- internal/handler/conda.go | 9 +++- internal/handler/download_cooldown.go | 39 ++++++++------- .../handler/download_cooldown_missing_test.go | 48 +++++++++++++++++++ 3 files changed, 78 insertions(+), 18 deletions(-) diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 44b21ef4..4a628df6 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -570,7 +570,8 @@ func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename stri // loaded is what the index was built from, unless an index built // earlier was reused. var loaded *metadataResult - index, err := publishIndex(h.proxy, "conda\x00"+cacheKey, func() metadataResult { + indexKey := "conda\x00" + cacheKey + index, err := publishIndex(h.proxy, indexKey, func() metadataResult { res := h.proxy.lookupMetadata(r.Context(), "conda", cacheKey, upstreamURL, "*/*", "gzip", nil) loaded = &res return res @@ -584,10 +585,14 @@ func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename stri // metadata_max_size included. return published, listed && !published.IsZero(), nil } - index, err = build(h.proxy.refetchLacking(r.Context(), "conda", cacheKey, upstreamURL, "*/*", "gzip", nil)) + res := h.proxy.refetchLacking(r.Context(), "conda", cacheKey, upstreamURL, "*/*", "gzip", nil) + index, err = build(res) if err != nil { return time.Time{}, false, err } + // Later downloads use the refetched index, not the one that lacked + // the file. + storePublishIndex(h.proxy, indexKey, res, index) published = index[filename] return published, !published.IsZero(), nil } diff --git a/internal/handler/download_cooldown.go b/internal/handler/download_cooldown.go index 11e98279..1ec5c8a4 100644 --- a/internal/handler/download_cooldown.go +++ b/internal/handler/download_cooldown.go @@ -135,22 +135,7 @@ func publishIndex[V any](p *Proxy, key string, load func() metadataResult, build if err != nil { return nil, err } - c.mu.Lock() - defer c.mu.Unlock() - if c.entries == nil { - c.entries = make(map[string]publishIndexEntry) - } - if _, ok := c.entries[key]; !ok && len(c.entries) >= maxPublishIndexes { - for evict := range c.entries { - delete(c.entries, evict) - break - } - } - entry := publishIndexEntry{digest: res.digest, index: index} - if res.digest == "" || !p.CacheMetadata { - entry.expires = p.clock().Add(time.Minute) - } - c.entries[key] = entry + storePublishIndex(p, key, res, index) return index, nil }) if err != nil { @@ -167,6 +152,28 @@ type listedTime struct { ok bool } +// storePublishIndex keeps index, built from res, as publishIndex's entry for +// key, replacing one built from older bytes. +func storePublishIndex(p *Proxy, key string, res metadataResult, index any) { + c := &p.sharedProxy().publishIndexes + c.mu.Lock() + defer c.mu.Unlock() + if c.entries == nil { + c.entries = make(map[string]publishIndexEntry) + } + if _, ok := c.entries[key]; !ok && len(c.entries) >= maxPublishIndexes { + for evict := range c.entries { + delete(c.entries, evict) + break + } + } + entry := publishIndexEntry{digest: res.digest, index: index} + if res.digest == "" || !p.CacheMetadata { + entry.expires = p.clock().Add(time.Minute) + } + c.entries[key] = entry +} + // readListed reads a version's entry from a metadata document fetched // through the metadata cache. read reports whether the document lists the // version at all. A copy cached within metadata_ttl, or served stale, can diff --git a/internal/handler/download_cooldown_missing_test.go b/internal/handler/download_cooldown_missing_test.go index af50f770..2a8feb27 100644 --- a/internal/handler/download_cooldown_missing_test.go +++ b/internal/handler/download_cooldown_missing_test.go @@ -205,3 +205,51 @@ func TestPyPIDownloadCooldownRefetchesReleasesLackingTheVersion(t *testing.T) { }) } } + +// TestCondaRefetchedIndexIsReused checks that, without the metadata cache, +// a repodata index fetched again for a newly released file replaces the one +// that lacked it, so the next download of the file doesn't fetch it again. +func TestCondaRefetchedIndexIsReused(t *testing.T) { + var c downloadCooldownCase + for _, candidate := range downloadCooldownCases() { + if candidate.ecosystem == "conda" { + c = candidate + } + } + now := time.Now() + var released atomic.Bool + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + times := map[string]time.Time{cooldownOldVersion: now.Add(-30 * 24 * time.Hour)} + if released.Load() { + times[cooldownRecentVersion] = now.Add(-time.Hour) + } + c.metadata(w, r, times) + })) + defer upstream.Close() + proxy, _, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher.fetchErr = errors.New("artifact must be served from the cache") + proxy.Cooldown = &cooldown.Config{Default: "7d"} + handler := c.handler(proxy, upstream.URL) + get := func(version string) int { + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, c.path(version), nil)) + return w.Code + } + c.seedCachedArtifact(t, proxy, cooldownOldVersion) + c.seedCachedArtifact(t, proxy, cooldownRecentVersion) + if code := get(cooldownOldVersion); code != http.StatusOK { + t.Fatalf("old version: status %d", code) + } + released.Store(true) + for range 2 { + if code := get(cooldownRecentVersion); code != http.StatusNotFound { + t.Fatalf("release in cooldown: status %d, want 404", code) + } + } + if got := requests.Load(); got != 2 { + t.Errorf("repodata fetched %d times, want 2", got) + } +} From 80a1ed0b6c78bf97ba9019d8abd71bb180ca78cf Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 18 Nov 2026 20:52:19 +0100 Subject: [PATCH 3/6] Share a conda refetch among downloads that found the file missing Concurrent downloads of a newly released file, reading the same index that lacked it, each fetched and indexed repodata again; past metadata_max_size each streamed it. They now share one refetch and build per repodata. --- internal/handler/conda.go | 19 ++++-- .../handler/download_cooldown_missing_test.go | 61 +++++++++++++++++++ 2 files changed, 74 insertions(+), 6 deletions(-) diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 4a628df6..ffd98521 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -585,15 +585,22 @@ func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename stri // metadata_max_size included. return published, listed && !published.IsZero(), nil } - res := h.proxy.refetchLacking(r.Context(), "conda", cacheKey, upstreamURL, "*/*", "gzip", nil) - index, err = build(res) + // Downloads that found the file missing from the same index share one + // refetch and build, which for repodata past metadata_max_size streams + // it. Later downloads use the refetched index. + refetched, err, _ := h.proxy.sharedProxy().publishIndexes.group.Do(indexKey+"\x00refetch", func() (any, error) { + res := h.proxy.refetchLacking(r.Context(), "conda", cacheKey, upstreamURL, "*/*", "gzip", nil) + index, err := build(res) + if err != nil { + return nil, err + } + storePublishIndex(h.proxy, indexKey, res, index) + return index, nil + }) if err != nil { return time.Time{}, false, err } - // Later downloads use the refetched index, not the one that lacked - // the file. - storePublishIndex(h.proxy, indexKey, res, index) - published = index[filename] + published = refetched.(map[string]time.Time)[filename] return published, !published.IsZero(), nil } diff --git a/internal/handler/download_cooldown_missing_test.go b/internal/handler/download_cooldown_missing_test.go index 2a8feb27..ec396381 100644 --- a/internal/handler/download_cooldown_missing_test.go +++ b/internal/handler/download_cooldown_missing_test.go @@ -253,3 +253,64 @@ func TestCondaRefetchedIndexIsReused(t *testing.T) { t.Errorf("repodata fetched %d times, want 2", got) } } + +// TestCondaConcurrentRefetchesShareOneIndex checks that concurrent downloads +// of a newly released file, finding it missing from the same kept index, +// fetch and index repodata past metadata_max_size once between them. +func TestCondaConcurrentRefetchesShareOneIndex(t *testing.T) { + var c downloadCooldownCase + for _, candidate := range downloadCooldownCases() { + if candidate.ecosystem == "conda" { + c = candidate + } + } + now := time.Now() + var released atomic.Bool + var after atomic.Int32 + gate := make(chan struct{}) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + times := map[string]time.Time{cooldownOldVersion: now.Add(-30 * 24 * time.Hour)} + if released.Load() { + after.Add(1) + <-gate + times[cooldownRecentVersion] = now.Add(-time.Hour) + } + c.metadata(w, r, times) + })) + defer upstream.Close() + proxy, _, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.MetadataMaxSize = 10 // every repodata is streamed + fetcher.fetchErr = errors.New("artifact must be served from the cache") + proxy.Cooldown = &cooldown.Config{Default: "7d"} + handler := c.handler(proxy, upstream.URL) + get := func(version string) int { + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, c.path(version), nil)) + return w.Code + } + c.seedCachedArtifact(t, proxy, cooldownOldVersion) + c.seedCachedArtifact(t, proxy, cooldownRecentVersion) + if code := get(cooldownOldVersion); code != http.StatusOK { + t.Fatalf("old version: status %d", code) + } + + released.Store(true) + const downloads = 5 + codes := make(chan int, downloads) + for range downloads { + go func() { codes <- get(cooldownRecentVersion) }() + } + // Lets every download reach the refetch before upstream answers. + time.Sleep(200 * time.Millisecond) + close(gate) + for range downloads { + if code := <-codes; code != http.StatusNotFound { + t.Errorf("release in cooldown: status %d, want 404", code) + } + } + // The refetch tries the metadata cache's fetch first, then streams. + if got := after.Load(); got != 2 { + t.Errorf("repodata fetched %d times for %d downloads, want 2", got, downloads) + } +} From 9905ccc7880e2e0d21a1ab769b48ab76ba72c69b Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 19 Nov 2026 17:12:25 +0100 Subject: [PATCH 4/6] Keep conda refetches of a missing file from repeating on every download A file repodata doesn't list sent every download of it through a refetch and a fresh index. A refetch that reads the bytes the kept index was built from now reuses that index, and an index a refetch built without a digest answers "not listed" for the rest of its one-minute keep. The shared lookup, refetch and stream run on a context detached from the first client, so its hanging up doesn't fail the downloads that joined it. --- internal/handler/conda.go | 28 ++- internal/handler/download_cooldown.go | 48 +++-- .../handler/download_cooldown_missing_test.go | 171 ++++++++++++++++++ 3 files changed, 226 insertions(+), 21 deletions(-) diff --git a/internal/handler/conda.go b/internal/handler/conda.go index ffd98521..6185bdf3 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "compress/gzip" + "context" "crypto/sha256" "encoding/hex" "encoding/json" @@ -558,9 +559,12 @@ func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename stri repodataPath := "/" + channel + "/" + arch + "/repodata.json" cacheKey := h.metadataCacheKey(repodataPath) upstreamURL := h.upstreamURL + repodataPath + // The index is built once for the downloads joining it, so a client + // that hangs up doesn't fail the others. + ctx := context.WithoutCancel(r.Context()) build := func(res metadataResult) (map[string]time.Time, error) { if errors.Is(res.err, ErrMetadataTooLarge) { - return h.streamRepodataIndex(r, upstreamURL) + return h.streamRepodataIndex(ctx, upstreamURL) } if res.err != nil { return nil, res.err @@ -572,7 +576,7 @@ func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename stri var loaded *metadataResult indexKey := "conda\x00" + cacheKey index, err := publishIndex(h.proxy, indexKey, func() metadataResult { - res := h.proxy.lookupMetadata(r.Context(), "conda", cacheKey, upstreamURL, "*/*", "gzip", nil) + res := h.proxy.lookupMetadata(ctx, "conda", cacheKey, upstreamURL, "*/*", "gzip", nil) loaded = &res return res }, build) @@ -580,21 +584,26 @@ func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename stri return time.Time{}, false, err } published, listed := index[filename] - if listed || loaded != nil && !loaded.fromCache && !loaded.stale { - // Listed, or just read from upstream, streamed past - // metadata_max_size included. + if listed || loaded != nil && !loaded.fromCache && !loaded.stale || + refetchedRecently(h.proxy, indexKey) { + // Listed, just read from upstream (streamed past metadata_max_size + // included), or refetched for a missing file within the last + // minute, without a digest to check it is current. return published, listed && !published.IsZero(), nil } // Downloads that found the file missing from the same index share one // refetch and build, which for repodata past metadata_max_size streams // it. Later downloads use the refetched index. refetched, err, _ := h.proxy.sharedProxy().publishIndexes.group.Do(indexKey+"\x00refetch", func() (any, error) { - res := h.proxy.refetchLacking(r.Context(), "conda", cacheKey, upstreamURL, "*/*", "gzip", nil) + res := h.proxy.refetchLacking(ctx, "conda", cacheKey, upstreamURL, "*/*", "gzip", nil) + if index, ok := keptPublishIndex(h.proxy, indexKey, res); ok { + return index, nil + } index, err := build(res) if err != nil { return nil, err } - storePublishIndex(h.proxy, indexKey, res, index) + storePublishIndex(h.proxy, indexKey, res, index, true) return index, nil }) if err != nil { @@ -618,8 +627,8 @@ func (h *CondaHandler) indexRepodata(body []byte, encoding string) (map[string]t } // streamRepodataIndex indexes repodata.json as it streams from upstream. -func (h *CondaHandler) streamRepodataIndex(r *http.Request, upstreamURL string) (map[string]time.Time, error) { - req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) +func (h *CondaHandler) streamRepodataIndex(ctx context.Context, upstreamURL string) (map[string]time.Time, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, upstreamURL, nil) if err != nil { return nil, err } @@ -663,7 +672,6 @@ func readRepodataIndex(r io.Reader) (map[string]time.Time, error) { if json.Unmarshal(raw, &e) != nil { return nil } - // A file without a timestamp is listed with the zero time. index[entry.Key] = time.Time{} if e.Timestamp != 0 { index[entry.Key] = condaTime(e.Timestamp) diff --git a/internal/handler/download_cooldown.go b/internal/handler/download_cooldown.go index 1ec5c8a4..bcfee00d 100644 --- a/internal/handler/download_cooldown.go +++ b/internal/handler/download_cooldown.go @@ -101,6 +101,9 @@ type publishIndexEntry struct { digest string index any expires time.Time + // refetched is set for an index built by a refetch for a version the + // document didn't list. + refetched bool } // publishIndex reuses parsed documents by digest, or for a minute without one. @@ -123,19 +126,14 @@ func publishIndex[V any](p *Proxy, key string, load func() metadataResult, build return index, nil } res := load() - if res.err == nil && res.digest != "" { - c.mu.Lock() - entry, ok := c.entries[key] - c.mu.Unlock() - if ok && entry.digest == res.digest { - return entry.index, nil - } + if index, ok := keptPublishIndex(p, key, res); ok { + return index, nil } index, err := build(res) if err != nil { return nil, err } - storePublishIndex(p, key, res, index) + storePublishIndex(p, key, res, index, false) return index, nil }) if err != nil { @@ -152,9 +150,37 @@ type listedTime struct { ok bool } +// keptPublishIndex returns the index kept for key when it was built from the +// bytes res read. +func keptPublishIndex(p *Proxy, key string, res metadataResult) (any, bool) { + if res.err != nil || res.digest == "" { + return nil, false + } + c := &p.sharedProxy().publishIndexes + c.mu.Lock() + defer c.mu.Unlock() + entry, ok := c.entries[key] + if !ok || entry.digest != res.digest { + return nil, false + } + return entry.index, true +} + +// refetchedRecently reports whether the index kept for key was built by a +// refetch and is within its keep, so a version it doesn't list was missing +// upstream less than a minute ago. +func refetchedRecently(p *Proxy, key string) bool { + c := &p.sharedProxy().publishIndexes + c.mu.Lock() + defer c.mu.Unlock() + entry, ok := c.entries[key] + return ok && entry.refetched && p.clock().Before(entry.expires) +} + // storePublishIndex keeps index, built from res, as publishIndex's entry for -// key, replacing one built from older bytes. -func storePublishIndex(p *Proxy, key string, res metadataResult, index any) { +// key, replacing one built from older bytes. refetched marks an index built +// by a refetch. +func storePublishIndex(p *Proxy, key string, res metadataResult, index any, refetched bool) { c := &p.sharedProxy().publishIndexes c.mu.Lock() defer c.mu.Unlock() @@ -167,7 +193,7 @@ func storePublishIndex(p *Proxy, key string, res metadataResult, index any) { break } } - entry := publishIndexEntry{digest: res.digest, index: index} + entry := publishIndexEntry{digest: res.digest, index: index, refetched: refetched} if res.digest == "" || !p.CacheMetadata { entry.expires = p.clock().Add(time.Minute) } diff --git a/internal/handler/download_cooldown_missing_test.go b/internal/handler/download_cooldown_missing_test.go index ec396381..997704ba 100644 --- a/internal/handler/download_cooldown_missing_test.go +++ b/internal/handler/download_cooldown_missing_test.go @@ -1,11 +1,13 @@ package handler import ( + "context" "errors" "fmt" "io" "net/http" "net/http/httptest" + "reflect" "strings" "sync/atomic" "testing" @@ -314,3 +316,172 @@ func TestCondaConcurrentRefetchesShareOneIndex(t *testing.T) { t.Errorf("repodata fetched %d times for %d downloads, want 2", got, downloads) } } + +func condaCooldownCase(t *testing.T) downloadCooldownCase { + t.Helper() + for _, c := range downloadCooldownCases() { + if c.ecosystem == "conda" { + return c + } + } + t.Fatal("no conda case") + return downloadCooldownCase{} +} + +// TestCondaNeverListedFileRefetchesOncePerKeep checks downloads of a file +// repodata never lists, without a digest to revalidate it by, refetch it +// once, and use that index for the rest of its keep. +func TestCondaNeverListedFileRefetchesOncePerKeep(t *testing.T) { + c := condaCooldownCase(t) + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + c.metadata(w, r, map[string]time.Time{cooldownOldVersion: time.Now().Add(-30 * 24 * time.Hour)}) + })) + defer upstream.Close() + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.fetchErr = errors.New("not fetched") + proxy.HTTPClient = upstream.Client() + proxy.Cooldown = &cooldown.Config{Default: "7d"} + handler := c.handler(proxy, upstream.URL) + for range 3 { + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, c.path("9.9.9"), nil)) + } + // The first download reads repodata, the second refetches it. + if got := requests.Load(); got != 2 { + t.Errorf("repodata fetched %d times, want 2", got) + } +} + +// TestCondaUnchangedRefetchKeepsItsIndex checks a refetch that finds the +// repodata unchanged reuses the index kept for it instead of parsing again. +func TestCondaUnchangedRefetchKeepsItsIndex(t *testing.T) { + c := condaCooldownCase(t) + old := time.Now().Add(-30 * 24 * time.Hour) + var released atomic.Bool + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + times := map[string]time.Time{cooldownOldVersion: old} + if released.Load() { + times[cooldownRecentVersion] = time.Now().Add(-time.Hour) + } + c.metadata(w, r, times) + })) + defer upstream.Close() + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.fetchErr = errors.New("not fetched") + proxy.HTTPClient = upstream.Client() + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.Cooldown = &cooldown.Config{Default: "7d"} + handler := c.handler(proxy, upstream.URL) + kept := func() uintptr { + c := &proxy.sharedProxy().publishIndexes + c.mu.Lock() + defer c.mu.Unlock() + for _, e := range c.entries { + return reflect.ValueOf(e.index).Pointer() + } + return 0 + } + var seen []uintptr + for range 3 { + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, c.path("9.9.9"), nil)) + seen = append(seen, kept()) + } + if seen[0] == 0 || seen[1] != seen[0] || seen[2] != seen[0] { + t.Errorf("kept indexes %v, want the first one throughout", seen) + } + released.Store(true) + c.seedCachedArtifact(t, proxy, cooldownRecentVersion) + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, c.path(cooldownRecentVersion), nil)) + if w.Code != http.StatusNotFound { + t.Errorf("release in cooldown after the refetches: status %d, want 404", w.Code) + } +} + +// TestCondaSharedRefetchOutlastsTheFirstClient checks a download joining a +// refetch of repodata past metadata_max_size gets its answer when the +// client that started it hangs up. +func TestCondaSharedRefetchOutlastsTheFirstClient(t *testing.T) { + c := condaCooldownCase(t) + old := time.Now().Add(-30 * 24 * time.Hour) + var released atomic.Bool + gate := make(chan struct{}) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + times := map[string]time.Time{cooldownOldVersion: old} + if released.Load() { + <-gate + times[cooldownRecentVersion] = time.Now().Add(-time.Hour) + } + c.metadata(w, r, times) + })) + defer upstream.Close() + proxy, _, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.MetadataMaxSize = 10 // every repodata is streamed + fetcher.fetchErr = errors.New("artifact must be served from the cache") + proxy.Cooldown = &cooldown.Config{Default: "7d"} + handler := c.handler(proxy, upstream.URL) + c.seedCachedArtifact(t, proxy, cooldownOldVersion) + c.seedCachedArtifact(t, proxy, cooldownRecentVersion) + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, c.path(cooldownOldVersion), nil)) + + released.Store(true) + ctx, cancel := context.WithCancel(context.Background()) + first := make(chan struct{}) + go func() { + defer close(first) + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, c.path(cooldownRecentVersion), nil).WithContext(ctx)) + }() + time.Sleep(100 * time.Millisecond) + joined := make(chan int, 1) + go func() { + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, c.path(cooldownRecentVersion), nil)) + joined <- w.Code + }() + time.Sleep(100 * time.Millisecond) + cancel() + time.Sleep(100 * time.Millisecond) + close(gate) + if code := <-joined; code != http.StatusNotFound { + t.Errorf("joined download: status %d, want 404", code) + } + <-first +} + +// TestCondaRefetchWithDigestDoesNotHideLaterReleases checks an index a +// refetch stored with a digest, which keeps it past a minute, doesn't answer +// "not listed" for a file released after it. +func TestCondaRefetchWithDigestDoesNotHideLaterReleases(t *testing.T) { + c := condaCooldownCase(t) + var released atomic.Bool + var requests atomic.Int64 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Each response differs, so each refetch builds and stores an index. + times := map[string]time.Time{cooldownOldVersion: time.UnixMilli(requests.Add(1))} + if released.Load() { + times[cooldownRecentVersion] = time.Now().Add(-time.Hour) + } + c.metadata(w, r, times) + })) + defer upstream.Close() + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.fetchErr = errors.New("not fetched") + proxy.HTTPClient = upstream.Client() + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.Cooldown = &cooldown.Config{Default: "7d"} + handler := c.handler(proxy, upstream.URL) + c.seedCachedArtifact(t, proxy, cooldownRecentVersion) + for range 2 { + handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, c.path("9.9.9"), nil)) + } + released.Store(true) + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, c.path(cooldownRecentVersion), nil)) + if w.Code != http.StatusNotFound { + t.Errorf("release in cooldown: status %d, want 404", w.Code) + } +} From 356712b9d788e6cc6d187a3160c3fb0ad22ae517 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sat, 21 Nov 2026 19:23:44 +0000 Subject: [PATCH 5/6] Read a missing conda file from the index a concurrent refetch stored The recent-refetch check read the current entry while the lookup had used the one it replaced, so a file the refetch listed was let through. --- internal/handler/conda.go | 12 +++++++++-- internal/handler/download_cooldown.go | 9 ++++++--- .../handler/download_cooldown_missing_test.go | 20 +++++++++++++++++++ 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 6185bdf3..4e2068a7 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -584,8 +584,16 @@ func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename stri return time.Time{}, false, err } published, listed := index[filename] - if listed || loaded != nil && !loaded.fromCache && !loaded.stale || - refetchedRecently(h.proxy, indexKey) { + current := loaded != nil && !loaded.fromCache && !loaded.stale + if !listed && !current { + // A concurrent refetch may have replaced the index read above, so + // the file is looked up in the refetched one. + if recent, ok := recentRefetch(h.proxy, indexKey); ok { + published, listed = recent.(map[string]time.Time)[filename] + current = true + } + } + if listed || current { // Listed, just read from upstream (streamed past metadata_max_size // included), or refetched for a missing file within the last // minute, without a digest to check it is current. diff --git a/internal/handler/download_cooldown.go b/internal/handler/download_cooldown.go index bcfee00d..6ec4a9f9 100644 --- a/internal/handler/download_cooldown.go +++ b/internal/handler/download_cooldown.go @@ -166,15 +166,18 @@ func keptPublishIndex(p *Proxy, key string, res metadataResult) (any, bool) { return entry.index, true } -// refetchedRecently reports whether the index kept for key was built by a +// recentRefetch returns the index kept for key when it was built by a // refetch and is within its keep, so a version it doesn't list was missing // upstream less than a minute ago. -func refetchedRecently(p *Proxy, key string) bool { +func recentRefetch(p *Proxy, key string) (any, bool) { c := &p.sharedProxy().publishIndexes c.mu.Lock() defer c.mu.Unlock() entry, ok := c.entries[key] - return ok && entry.refetched && p.clock().Before(entry.expires) + if !ok || !entry.refetched || !p.clock().Before(entry.expires) { + return nil, false + } + return entry.index, true } // storePublishIndex keeps index, built from res, as publishIndex's entry for diff --git a/internal/handler/download_cooldown_missing_test.go b/internal/handler/download_cooldown_missing_test.go index 997704ba..9241d17b 100644 --- a/internal/handler/download_cooldown_missing_test.go +++ b/internal/handler/download_cooldown_missing_test.go @@ -485,3 +485,23 @@ func TestCondaRefetchWithDigestDoesNotHideLaterReleases(t *testing.T) { t.Errorf("release in cooldown: status %d, want 404", w.Code) } } + +// TestRecentRefetchReturnsTheRefetchedIndex checks a lookup that read the +// index a refetch has since replaced gets the refetched one to read from. +func TestRecentRefetchReturnsTheRefetchedIndex(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + const key = "conda\x00repodata" + storePublishIndex(proxy, key, metadataResult{}, map[string]time.Time{}, false) + if _, ok := recentRefetch(proxy, key); ok { + t.Fatal("an index no refetch built counts as refetched") + } + released := time.Now().Add(-time.Hour) + storePublishIndex(proxy, key, metadataResult{}, map[string]time.Time{"new.conda": released}, true) + index, ok := recentRefetch(proxy, key) + if !ok { + t.Fatal("refetched index not returned") + } + if got := index.(map[string]time.Time)["new.conda"]; !got.Equal(released) { + t.Errorf("new.conda published %v, want %v", got, released) + } +} From 572b0640c3ada6ba0892f7671cf04e34b67959b4 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 21 Nov 2026 21:00:35 +0100 Subject: [PATCH 6/6] Match PyPI downloads to listed releases by canonical version A download spelled 1.0 against a document listing 1.0.0 counted as unlisted and fetched the metadata again on every request. --- .../handler/download_cooldown_missing_test.go | 4 ++++ internal/handler/pypi.go | 16 ++++++++++------ internal/handler/pypi_version_spelling_test.go | 11 +++++++---- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/internal/handler/download_cooldown_missing_test.go b/internal/handler/download_cooldown_missing_test.go index 9241d17b..3e974e5a 100644 --- a/internal/handler/download_cooldown_missing_test.go +++ b/internal/handler/download_cooldown_missing_test.go @@ -195,6 +195,10 @@ func TestPyPIDownloadCooldownRefetchesReleasesLackingTheVersion(t *testing.T) { if code := get("newpkg-9.0.0.tar.gz"); code != http.StatusOK || metadataRequests.Load() != 1 { t.Fatalf("unlisted version: status %d, %d metadata requests; want 200 and 1", code, metadataRequests.Load()) } + // A listed version spelled another way isn't fetched again either. + if code := get("newpkg-1.0.tar.gz"); code != http.StatusOK || metadataRequests.Load() != 1 { + t.Fatalf("listed version spelled 1.0: status %d, %d metadata requests; want 200 and 1", code, metadataRequests.Load()) + } released.Store(true) down.Store(upstreamFails) code := get("newpkg-2.0.0.tar.gz") diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index dd856d09..a136aa4f 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -493,7 +493,7 @@ func (h *PyPIHandler) versionInCooldown(r *http.Request, name, version string) ( publishedAt, err := readListed(r.Context(), h.proxy, "pypi", name+"/json", jsonURL, contentTypeJSON, "", func(res metadataResult) (time.Time, bool, error) { times, err := h.parseReleaseTimes(res.body) - published, listed := times[version] + published, listed := releaseTime(times, version) return published, listed, err }) if errors.Is(err, ErrUpstreamNotFound) { @@ -509,16 +509,20 @@ func (h *PyPIHandler) versionInCooldown(r *http.Request, name, version string) ( return !h.proxy.cooldown().IsAllowed("pypi", canonicalPackagePURL("pypi", name), publishedAt), nil } -// releaseTime returns the newest upload time of the releases equal to version. -func releaseTime(times map[string]time.Time, version string) time.Time { +// releaseTime returns the newest upload time of the releases equal to +// version, and whether there is such a release. +func releaseTime(times map[string]time.Time, version string) (newest time.Time, listed bool) { canonical := pypi.CanonicalVersion(version) - var newest time.Time for release, published := range times { - if pypi.CanonicalVersion(release) == canonical && published.After(newest) { + if pypi.CanonicalVersion(release) != canonical { + continue + } + listed = true + if published.After(newest) { newest = published } } - return newest + return newest, listed } // storeUploadTime records a release's newest upload time under the version diff --git a/internal/handler/pypi_version_spelling_test.go b/internal/handler/pypi_version_spelling_test.go index 8f318cb2..b20cddb8 100644 --- a/internal/handler/pypi_version_spelling_test.go +++ b/internal/handler/pypi_version_spelling_test.go @@ -76,12 +76,15 @@ func TestPyPIReleaseTimeTakesTheNewestOfEqualReleases(t *testing.T) { times := map[string]time.Time{"1.0": older, "1.0.0": newer, "2.0": {}} for range 20 { for _, version := range []string{"1.0", "1.0.0", "1.0.0.0"} { - if got := releaseTime(times, version); !got.Equal(newer) { - t.Fatalf("releaseTime(%q) = %v, want %v", version, got, newer) + if got, listed := releaseTime(times, version); !got.Equal(newer) || !listed { + t.Fatalf("releaseTime(%q) = %v, %t; want %v, listed", version, got, listed, newer) } } } - if got := releaseTime(times, "3.0"); !got.IsZero() { - t.Errorf("releaseTime(3.0) = %v, want the zero time", got) + if got, listed := releaseTime(times, "2.0.0"); !got.IsZero() || !listed { + t.Errorf("releaseTime(2.0.0) = %v, %t; want the zero time, listed", got, listed) + } + if got, listed := releaseTime(times, "3.0"); !got.IsZero() || listed { + t.Errorf("releaseTime(3.0) = %v, %t; want the zero time, not listed", got, listed) } }