mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2026-05-12 22:10:25 +00:00
feat: support simple JSON API for PyPI package registry (#12095)
This PR extends Forĝejo's PyPI package index to support [the simple JSON repository API](https://packaging.python.org/en/latest/specifications/simple-repository-api/#json-serialization). Since the existing implementation was for the HTML serialization of the same simple API, no new endpoint has been added. Instead, Forĝejo chooses between serialization schemes based on the "Accept" header in the request. This, together with CORS, will make Forĝejo compatible with [micropip](https://github.com/pyodide/micropip). ## Checklist The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. All work and communication must conform to Forgejo's [AI Agreement](https://codeberg.org/forgejo/governance/src/branch/main/AIAgreement.md). There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org). ### Tests for Go changes (can be removed for JavaScript changes) - I added test coverage for Go changes... - [x] in the `tests/integration` directory if it involves interactions with a live Forgejo server. - I ran... - [x] `make pr-go` before pushing ### Documentation - [x] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change. - [x] I did not document these changes and I do not expect someone else to do it. ### Release notes - [x] This change will be noticed by a Forgejo user or admin (feature, bug fix, performance, etc.). I suggest to include a release note for this change. - [ ] This change is not visible to a Forgejo user or admin (refactor, dependency upgrade, etc.). I think there is no need to add a release note for this change. *The decision if the pull request will be shown in the release notes is up to the mergers / release team.* The content of the `release-notes/<pull request number>.md` file will serve as the basis for the release notes. If the file does not exist, the title of the pull request will be used instead. <!--start release-notes-assistant--> ## Release notes <!--URL:https://codeberg.org/forgejo/forgejo--> - Features - [PR](https://codeberg.org/forgejo/forgejo/pulls/12095): <!--number 12095 --><!--line 0 --><!--description SG9zdGVkIFB5UEkgcGFja2FnZXMgbWF5IGJlIGFjY2Vzc2VkIHZpYSB0aGUgW3NpbXBsZSBKU09OIEFQSV0oaHR0cHM6Ly9wYWNrYWdpbmcucHl0aG9uLm9yZy9lbi9sYXRlc3Qvc3BlY2lmaWNhdGlvbnMvc2ltcGxlLXJlcG9zaXRvcnktYXBpLyNqc29uLXNlcmlhbGl6YXRpb24pIGluIGFkZGl0aW9uIHRvIHRoZSBzaW1wbGUgSFRNTCBBUEkgYWxyZWFkeSBhdmFpbGFibGUu-->Hosted PyPI packages may be accessed via the [simple JSON API](https://packaging.python.org/en/latest/specifications/simple-repository-api/#json-serialization) in addition to the simple HTML API already available.<!--description--> <!--end release-notes-assistant--> Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/12095 Reviewed-by: Mathieu Fenniak <mfenniak@noreply.codeberg.org>
This commit is contained in:
Zachary Spector
Mathieu Fenniak
parent
81c46e4a7c
commit
25e7a0b91b
4 changed files with 133 additions and 5 deletions
|
|
@ -13,3 +13,26 @@ type Metadata struct {
|
|||
License string `json:"license,omitempty"`
|
||||
RequiresPython string `json:"requires_python,omitempty"`
|
||||
}
|
||||
|
||||
type FileHashesJSON struct {
|
||||
SHA256 string `json:"sha256"`
|
||||
}
|
||||
|
||||
type FileJSON struct {
|
||||
Filename string `json:"filename"`
|
||||
URL string `json:"url"`
|
||||
Hashes FileHashesJSON `json:"hashes"`
|
||||
RequiresPython string `json:"requires-python"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
type PackageMetaJSON struct {
|
||||
APIVersion string `json:"api-version"`
|
||||
}
|
||||
|
||||
type PackageJSON struct {
|
||||
Name string `json:"name"`
|
||||
Meta PackageMetaJSON `json:"meta"`
|
||||
Versions []string `json:"versions"`
|
||||
Files []FileJSON `json:"files"`
|
||||
}
|
||||
|
|
|
|||
1
release-notes/12095.md
Normal file
1
release-notes/12095.md
Normal file
|
|
@ -0,0 +1 @@
|
|||
Hosted PyPI packages may be accessed via the [simple JSON API](https://packaging.python.org/en/latest/specifications/simple-repository-api/#json-serialization) in addition to the simple HTML API already available.
|
||||
|
|
@ -8,11 +8,14 @@ import (
|
|||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
packages_model "forgejo.org/models/packages"
|
||||
"forgejo.org/modules/json"
|
||||
"forgejo.org/modules/log"
|
||||
packages_module "forgejo.org/modules/packages"
|
||||
pypi_module "forgejo.org/modules/packages/pypi"
|
||||
"forgejo.org/modules/setting"
|
||||
|
|
@ -44,8 +47,14 @@ func apiError(ctx *context.Context, status int, obj any) {
|
|||
})
|
||||
}
|
||||
|
||||
// PackageMetadata returns the metadata for a single package
|
||||
func PackageMetadata(ctx *context.Context) {
|
||||
func contentTypeSupported(ctyps []string, v string) bool {
|
||||
return slices.ContainsFunc(ctyps, func(ctyp string) bool {
|
||||
return strings.HasPrefix(ctyp, v)
|
||||
})
|
||||
}
|
||||
|
||||
// HTMLPackageMetadata returns the metadata for a single package in Simple HTML per PEP691
|
||||
func HTMLPackageMetadata(ctx *context.Context) {
|
||||
packageName := normalizer.Replace(ctx.Params("id"))
|
||||
|
||||
pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypePyPI, packageName)
|
||||
|
|
@ -72,9 +81,82 @@ func PackageMetadata(ctx *context.Context) {
|
|||
ctx.Data["RegistryURL"] = setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/pypi"
|
||||
ctx.Data["PackageDescriptor"] = pds[0]
|
||||
ctx.Data["PackageDescriptors"] = pds
|
||||
// Content-Type headers need to be in this order for the page to show in the browser
|
||||
ctx.Resp.Header().Set("Content-Type", "application/vnd.pypi.simple.v1+html")
|
||||
ctx.Resp.Header().Add("Content-Type", "text/html")
|
||||
ctx.HTML(http.StatusOK, "api/packages/pypi/simple")
|
||||
}
|
||||
|
||||
// JSONPackageMetadata returns the metadata for a single package in Simple JSON per PEP691
|
||||
func JSONPackageMetadata(ctx *context.Context) {
|
||||
packageName := normalizer.Replace(ctx.Params("id"))
|
||||
|
||||
pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypePyPI, packageName)
|
||||
if err != nil {
|
||||
apiError(ctx, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
if len(pvs) == 0 {
|
||||
apiError(ctx, http.StatusNotFound, err)
|
||||
return
|
||||
}
|
||||
|
||||
pds, err := packages_model.GetPackageDescriptors(ctx, pvs)
|
||||
if err != nil {
|
||||
apiError(ctx, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
|
||||
// sort package descriptors by version to mimic PyPI format
|
||||
slices.SortFunc(pds, func(a, b *packages_model.PackageDescriptor) int {
|
||||
return strings.Compare(a.Version.Version, b.Version.Version)
|
||||
})
|
||||
registryURL := setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/pypi"
|
||||
versions := make([]string, len(pvs))
|
||||
for i, pv := range pvs {
|
||||
versions[i] = pv.Version
|
||||
}
|
||||
var fileCounter int
|
||||
for _, pd := range pds {
|
||||
fileCounter += len(pd.Files)
|
||||
}
|
||||
files := make([]pypi_module.FileJSON, fileCounter)
|
||||
var i int
|
||||
for _, pd := range pds {
|
||||
for _, file := range pd.Files {
|
||||
files[i] = pypi_module.FileJSON{
|
||||
Filename: file.File.Name,
|
||||
URL: registryURL + "/files/" + pd.Package.LowerName + "/" + pd.Version.Version + "/" + file.File.Name,
|
||||
RequiresPython: pd.Metadata.(*pypi_module.Metadata).RequiresPython,
|
||||
Hashes: pypi_module.FileHashesJSON{SHA256: file.Blob.HashSHA256},
|
||||
Size: file.Blob.Size,
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
content := pypi_module.PackageJSON{
|
||||
Name: pds[0].Package.Name,
|
||||
Meta: pypi_module.PackageMetaJSON{APIVersion: "1.4"},
|
||||
Versions: versions,
|
||||
Files: files,
|
||||
}
|
||||
ctx.Resp.Header().Set("Content-Type", "application/vnd.pypi.simple.v1+json")
|
||||
ctx.Resp.Header().Add("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(ctx.Resp).Encode(content); err != nil {
|
||||
log.Error("Render JSON failed: %v", err)
|
||||
apiError(ctx, http.StatusInternalServerError, err)
|
||||
}
|
||||
}
|
||||
|
||||
func PackageMetadata(ctx *context.Context) {
|
||||
ctyp := ctx.Req.Header["Accept"]
|
||||
if contentTypeSupported(ctyp, "application/vnd.pypi.simple.v1+json") {
|
||||
JSONPackageMetadata(ctx)
|
||||
} else {
|
||||
HTMLPackageMetadata(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// DownloadPackageFile serves the content of a package
|
||||
func DownloadPackageFile(ctx *context.Context) {
|
||||
packageName := normalizer.Replace(ctx.Params("id"))
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import (
|
|||
"forgejo.org/models/packages"
|
||||
"forgejo.org/models/unittest"
|
||||
user_model "forgejo.org/models/user"
|
||||
"forgejo.org/modules/json"
|
||||
"forgejo.org/modules/packages/pypi"
|
||||
"forgejo.org/tests"
|
||||
|
||||
|
|
@ -211,19 +212,20 @@ func TestPackagePyPI(t *testing.T) {
|
|||
assert.Equal(t, int64(2), pvs[0].DownloadCount)
|
||||
})
|
||||
|
||||
t.Run("PackageMetadata", func(t *testing.T) {
|
||||
hrefMatcher := regexp.MustCompile(fmt.Sprintf(`%s/files/%s/%s/test\..+#sha256=%s`, root, regexp.QuoteMeta(packageName), regexp.QuoteMeta(packageVersion), hashSHA256))
|
||||
|
||||
t.Run("PackageMetadataHTML", func(t *testing.T) {
|
||||
defer tests.PrintCurrentTest(t)()
|
||||
|
||||
req := NewRequest(t, "GET", fmt.Sprintf("%s/simple/%s", root, packageName)).
|
||||
AddBasicAuth(user.Name)
|
||||
req.Header["Accept"] = []string{"application/vnd.pypi.simple.v1+html"}
|
||||
resp := MakeRequest(t, req, http.StatusOK)
|
||||
|
||||
htmlDoc := NewHTMLParser(t, resp.Body)
|
||||
nodes := htmlDoc.doc.Find("a").Nodes
|
||||
assert.Len(t, nodes, 2)
|
||||
|
||||
hrefMatcher := regexp.MustCompile(fmt.Sprintf(`%s/files/%s/%s/test\..+#sha256=%s`, root, regexp.QuoteMeta(packageName), regexp.QuoteMeta(packageVersion), hashSHA256))
|
||||
|
||||
for _, a := range nodes {
|
||||
for _, att := range a.Attr {
|
||||
switch att.Key {
|
||||
|
|
@ -237,4 +239,24 @@ func TestPackagePyPI(t *testing.T) {
|
|||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("PackageMetadataJSON", func(t *testing.T) {
|
||||
defer tests.PrintCurrentTest(t)()
|
||||
|
||||
req := NewRequest(t, "GET", fmt.Sprintf("%s/simple/%s", root, packageName)).
|
||||
AddBasicAuth(user.Name)
|
||||
req.Header["Accept"] = []string{"application/vnd.pypi.simple.v1+json"}
|
||||
resp := MakeRequest(t, req, http.StatusOK)
|
||||
assert.Greater(t, resp.Body.Len(), 3)
|
||||
txt := make([]byte, resp.Body.Len())
|
||||
resp.Body.Read(txt)
|
||||
var obj pypi.PackageJSON
|
||||
require.NoError(t, json.Unmarshal(txt, &obj))
|
||||
assert.Equal(t, packageName, obj.Name)
|
||||
assert.Equal(t, pypi.PackageMetaJSON{APIVersion: "1.4"}, obj.Meta)
|
||||
for _, filed := range obj.Files {
|
||||
hrefMatcher = regexp.MustCompile(fmt.Sprintf(`%s/files/%s/%s/test\.(tar\.gz)|(whl)`, root, regexp.QuoteMeta(packageName), regexp.QuoteMeta(packageVersion)))
|
||||
assert.Regexp(t, hrefMatcher, filed.URL[21:])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue