From 25e7a0b91b5feb8fe2491d1cfbd280ed93521bab Mon Sep 17 00:00:00 2001 From: Zachary Spector Date: Thu, 30 Apr 2026 16:58:28 +0200 Subject: [PATCH] feat: support simple JSON API for PyPI package registry (#12095) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR extends Forĝejo's PyPI package index to support [the simple JSON repository API](https://packaging.python.org/en/latest/specifications/simple-repository-api/#json-serialization). Since the existing implementation was for the HTML serialization of the same simple API, no new endpoint has been added. Instead, Forĝejo chooses between serialization schemes based on the "Accept" header in the request. This, together with CORS, will make Forĝejo compatible with [micropip](https://github.com/pyodide/micropip). ## Checklist The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. All work and communication must conform to Forgejo's [AI Agreement](https://codeberg.org/forgejo/governance/src/branch/main/AIAgreement.md). There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org). ### Tests for Go changes (can be removed for JavaScript changes) - I added test coverage for Go changes... - [x] in the `tests/integration` directory if it involves interactions with a live Forgejo server. - I ran... - [x] `make pr-go` before pushing ### Documentation - [x] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change. - [x] I did not document these changes and I do not expect someone else to do it. ### Release notes - [x] This change will be noticed by a Forgejo user or admin (feature, bug fix, performance, etc.). I suggest to include a release note for this change. - [ ] This change is not visible to a Forgejo user or admin (refactor, dependency upgrade, etc.). I think there is no need to add a release note for this change. *The decision if the pull request will be shown in the release notes is up to the mergers / release team.* The content of the `release-notes/.md` file will serve as the basis for the release notes. If the file does not exist, the title of the pull request will be used instead. ## Release notes - Features - [PR](https://codeberg.org/forgejo/forgejo/pulls/12095): Hosted PyPI packages may be accessed via the [simple JSON API](https://packaging.python.org/en/latest/specifications/simple-repository-api/#json-serialization) in addition to the simple HTML API already available. Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/12095 Reviewed-by: Mathieu Fenniak --- modules/packages/pypi/metadata.go | 23 ++++++ release-notes/12095.md | 1 + routers/api/packages/pypi/pypi.go | 86 ++++++++++++++++++++- tests/integration/api_packages_pypi_test.go | 28 ++++++- 4 files changed, 133 insertions(+), 5 deletions(-) create mode 100644 release-notes/12095.md diff --git a/modules/packages/pypi/metadata.go b/modules/packages/pypi/metadata.go index 125728c4f1..0aa90aeca5 100644 --- a/modules/packages/pypi/metadata.go +++ b/modules/packages/pypi/metadata.go @@ -13,3 +13,26 @@ type Metadata struct { License string `json:"license,omitempty"` RequiresPython string `json:"requires_python,omitempty"` } + +type FileHashesJSON struct { + SHA256 string `json:"sha256"` +} + +type FileJSON struct { + Filename string `json:"filename"` + URL string `json:"url"` + Hashes FileHashesJSON `json:"hashes"` + RequiresPython string `json:"requires-python"` + Size int64 `json:"size"` +} + +type PackageMetaJSON struct { + APIVersion string `json:"api-version"` +} + +type PackageJSON struct { + Name string `json:"name"` + Meta PackageMetaJSON `json:"meta"` + Versions []string `json:"versions"` + Files []FileJSON `json:"files"` +} diff --git a/release-notes/12095.md b/release-notes/12095.md new file mode 100644 index 0000000000..3930b07f46 --- /dev/null +++ b/release-notes/12095.md @@ -0,0 +1 @@ +Hosted PyPI packages may be accessed via the [simple JSON API](https://packaging.python.org/en/latest/specifications/simple-repository-api/#json-serialization) in addition to the simple HTML API already available. diff --git a/routers/api/packages/pypi/pypi.go b/routers/api/packages/pypi/pypi.go index 360632570e..5d8bf5d45f 100644 --- a/routers/api/packages/pypi/pypi.go +++ b/routers/api/packages/pypi/pypi.go @@ -8,11 +8,14 @@ import ( "io" "net/http" "regexp" + "slices" "sort" "strings" "unicode" packages_model "forgejo.org/models/packages" + "forgejo.org/modules/json" + "forgejo.org/modules/log" packages_module "forgejo.org/modules/packages" pypi_module "forgejo.org/modules/packages/pypi" "forgejo.org/modules/setting" @@ -44,8 +47,14 @@ func apiError(ctx *context.Context, status int, obj any) { }) } -// PackageMetadata returns the metadata for a single package -func PackageMetadata(ctx *context.Context) { +func contentTypeSupported(ctyps []string, v string) bool { + return slices.ContainsFunc(ctyps, func(ctyp string) bool { + return strings.HasPrefix(ctyp, v) + }) +} + +// HTMLPackageMetadata returns the metadata for a single package in Simple HTML per PEP691 +func HTMLPackageMetadata(ctx *context.Context) { packageName := normalizer.Replace(ctx.Params("id")) pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypePyPI, packageName) @@ -72,9 +81,82 @@ func PackageMetadata(ctx *context.Context) { ctx.Data["RegistryURL"] = setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/pypi" ctx.Data["PackageDescriptor"] = pds[0] ctx.Data["PackageDescriptors"] = pds + // Content-Type headers need to be in this order for the page to show in the browser + ctx.Resp.Header().Set("Content-Type", "application/vnd.pypi.simple.v1+html") + ctx.Resp.Header().Add("Content-Type", "text/html") ctx.HTML(http.StatusOK, "api/packages/pypi/simple") } +// JSONPackageMetadata returns the metadata for a single package in Simple JSON per PEP691 +func JSONPackageMetadata(ctx *context.Context) { + packageName := normalizer.Replace(ctx.Params("id")) + + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypePyPI, packageName) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pvs) == 0 { + apiError(ctx, http.StatusNotFound, err) + return + } + + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + // sort package descriptors by version to mimic PyPI format + slices.SortFunc(pds, func(a, b *packages_model.PackageDescriptor) int { + return strings.Compare(a.Version.Version, b.Version.Version) + }) + registryURL := setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/pypi" + versions := make([]string, len(pvs)) + for i, pv := range pvs { + versions[i] = pv.Version + } + var fileCounter int + for _, pd := range pds { + fileCounter += len(pd.Files) + } + files := make([]pypi_module.FileJSON, fileCounter) + var i int + for _, pd := range pds { + for _, file := range pd.Files { + files[i] = pypi_module.FileJSON{ + Filename: file.File.Name, + URL: registryURL + "/files/" + pd.Package.LowerName + "/" + pd.Version.Version + "/" + file.File.Name, + RequiresPython: pd.Metadata.(*pypi_module.Metadata).RequiresPython, + Hashes: pypi_module.FileHashesJSON{SHA256: file.Blob.HashSHA256}, + Size: file.Blob.Size, + } + i++ + } + } + content := pypi_module.PackageJSON{ + Name: pds[0].Package.Name, + Meta: pypi_module.PackageMetaJSON{APIVersion: "1.4"}, + Versions: versions, + Files: files, + } + ctx.Resp.Header().Set("Content-Type", "application/vnd.pypi.simple.v1+json") + ctx.Resp.Header().Add("Content-Type", "application/json") + if err := json.NewEncoder(ctx.Resp).Encode(content); err != nil { + log.Error("Render JSON failed: %v", err) + apiError(ctx, http.StatusInternalServerError, err) + } +} + +func PackageMetadata(ctx *context.Context) { + ctyp := ctx.Req.Header["Accept"] + if contentTypeSupported(ctyp, "application/vnd.pypi.simple.v1+json") { + JSONPackageMetadata(ctx) + } else { + HTMLPackageMetadata(ctx) + } +} + // DownloadPackageFile serves the content of a package func DownloadPackageFile(ctx *context.Context) { packageName := normalizer.Replace(ctx.Params("id")) diff --git a/tests/integration/api_packages_pypi_test.go b/tests/integration/api_packages_pypi_test.go index f025f8e577..45aa41c573 100644 --- a/tests/integration/api_packages_pypi_test.go +++ b/tests/integration/api_packages_pypi_test.go @@ -17,6 +17,7 @@ import ( "forgejo.org/models/packages" "forgejo.org/models/unittest" user_model "forgejo.org/models/user" + "forgejo.org/modules/json" "forgejo.org/modules/packages/pypi" "forgejo.org/tests" @@ -211,19 +212,20 @@ func TestPackagePyPI(t *testing.T) { assert.Equal(t, int64(2), pvs[0].DownloadCount) }) - t.Run("PackageMetadata", func(t *testing.T) { + hrefMatcher := regexp.MustCompile(fmt.Sprintf(`%s/files/%s/%s/test\..+#sha256=%s`, root, regexp.QuoteMeta(packageName), regexp.QuoteMeta(packageVersion), hashSHA256)) + + t.Run("PackageMetadataHTML", func(t *testing.T) { defer tests.PrintCurrentTest(t)() req := NewRequest(t, "GET", fmt.Sprintf("%s/simple/%s", root, packageName)). AddBasicAuth(user.Name) + req.Header["Accept"] = []string{"application/vnd.pypi.simple.v1+html"} resp := MakeRequest(t, req, http.StatusOK) htmlDoc := NewHTMLParser(t, resp.Body) nodes := htmlDoc.doc.Find("a").Nodes assert.Len(t, nodes, 2) - hrefMatcher := regexp.MustCompile(fmt.Sprintf(`%s/files/%s/%s/test\..+#sha256=%s`, root, regexp.QuoteMeta(packageName), regexp.QuoteMeta(packageVersion), hashSHA256)) - for _, a := range nodes { for _, att := range a.Attr { switch att.Key { @@ -237,4 +239,24 @@ func TestPackagePyPI(t *testing.T) { } } }) + + t.Run("PackageMetadataJSON", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/simple/%s", root, packageName)). + AddBasicAuth(user.Name) + req.Header["Accept"] = []string{"application/vnd.pypi.simple.v1+json"} + resp := MakeRequest(t, req, http.StatusOK) + assert.Greater(t, resp.Body.Len(), 3) + txt := make([]byte, resp.Body.Len()) + resp.Body.Read(txt) + var obj pypi.PackageJSON + require.NoError(t, json.Unmarshal(txt, &obj)) + assert.Equal(t, packageName, obj.Name) + assert.Equal(t, pypi.PackageMetaJSON{APIVersion: "1.4"}, obj.Meta) + for _, filed := range obj.Files { + hrefMatcher = regexp.MustCompile(fmt.Sprintf(`%s/files/%s/%s/test\.(tar\.gz)|(whl)`, root, regexp.QuoteMeta(packageName), regexp.QuoteMeta(packageVersion))) + assert.Regexp(t, hrefMatcher, filed.URL[21:]) + } + }) }