diff --git a/modules/packages/pypi/metadata.go b/modules/packages/pypi/metadata.go index 125728c4f1..0aa90aeca5 100644 --- a/modules/packages/pypi/metadata.go +++ b/modules/packages/pypi/metadata.go @@ -13,3 +13,26 @@ type Metadata struct { License string `json:"license,omitempty"` RequiresPython string `json:"requires_python,omitempty"` } + +type FileHashesJSON struct { + SHA256 string `json:"sha256"` +} + +type FileJSON struct { + Filename string `json:"filename"` + URL string `json:"url"` + Hashes FileHashesJSON `json:"hashes"` + RequiresPython string `json:"requires-python"` + Size int64 `json:"size"` +} + +type PackageMetaJSON struct { + APIVersion string `json:"api-version"` +} + +type PackageJSON struct { + Name string `json:"name"` + Meta PackageMetaJSON `json:"meta"` + Versions []string `json:"versions"` + Files []FileJSON `json:"files"` +} diff --git a/release-notes/12095.md b/release-notes/12095.md new file mode 100644 index 0000000000..3930b07f46 --- /dev/null +++ b/release-notes/12095.md @@ -0,0 +1 @@ +Hosted PyPI packages may be accessed via the [simple JSON API](https://packaging.python.org/en/latest/specifications/simple-repository-api/#json-serialization) in addition to the simple HTML API already available. diff --git a/routers/api/packages/pypi/pypi.go b/routers/api/packages/pypi/pypi.go index 360632570e..5d8bf5d45f 100644 --- a/routers/api/packages/pypi/pypi.go +++ b/routers/api/packages/pypi/pypi.go @@ -8,11 +8,14 @@ import ( "io" "net/http" "regexp" + "slices" "sort" "strings" "unicode" packages_model "forgejo.org/models/packages" + "forgejo.org/modules/json" + "forgejo.org/modules/log" packages_module "forgejo.org/modules/packages" pypi_module "forgejo.org/modules/packages/pypi" "forgejo.org/modules/setting" @@ -44,8 +47,14 @@ func apiError(ctx *context.Context, status int, obj any) { }) } -// PackageMetadata returns the metadata for a single package -func PackageMetadata(ctx *context.Context) { +func contentTypeSupported(ctyps []string, v string) bool { + return slices.ContainsFunc(ctyps, func(ctyp string) bool { + return strings.HasPrefix(ctyp, v) + }) +} + +// HTMLPackageMetadata returns the metadata for a single package in Simple HTML per PEP691 +func HTMLPackageMetadata(ctx *context.Context) { packageName := normalizer.Replace(ctx.Params("id")) pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypePyPI, packageName) @@ -72,9 +81,82 @@ func PackageMetadata(ctx *context.Context) { ctx.Data["RegistryURL"] = setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/pypi" ctx.Data["PackageDescriptor"] = pds[0] ctx.Data["PackageDescriptors"] = pds + // Content-Type headers need to be in this order for the page to show in the browser + ctx.Resp.Header().Set("Content-Type", "application/vnd.pypi.simple.v1+html") + ctx.Resp.Header().Add("Content-Type", "text/html") ctx.HTML(http.StatusOK, "api/packages/pypi/simple") } +// JSONPackageMetadata returns the metadata for a single package in Simple JSON per PEP691 +func JSONPackageMetadata(ctx *context.Context) { + packageName := normalizer.Replace(ctx.Params("id")) + + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypePyPI, packageName) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pvs) == 0 { + apiError(ctx, http.StatusNotFound, err) + return + } + + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + // sort package descriptors by version to mimic PyPI format + slices.SortFunc(pds, func(a, b *packages_model.PackageDescriptor) int { + return strings.Compare(a.Version.Version, b.Version.Version) + }) + registryURL := setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/pypi" + versions := make([]string, len(pvs)) + for i, pv := range pvs { + versions[i] = pv.Version + } + var fileCounter int + for _, pd := range pds { + fileCounter += len(pd.Files) + } + files := make([]pypi_module.FileJSON, fileCounter) + var i int + for _, pd := range pds { + for _, file := range pd.Files { + files[i] = pypi_module.FileJSON{ + Filename: file.File.Name, + URL: registryURL + "/files/" + pd.Package.LowerName + "/" + pd.Version.Version + "/" + file.File.Name, + RequiresPython: pd.Metadata.(*pypi_module.Metadata).RequiresPython, + Hashes: pypi_module.FileHashesJSON{SHA256: file.Blob.HashSHA256}, + Size: file.Blob.Size, + } + i++ + } + } + content := pypi_module.PackageJSON{ + Name: pds[0].Package.Name, + Meta: pypi_module.PackageMetaJSON{APIVersion: "1.4"}, + Versions: versions, + Files: files, + } + ctx.Resp.Header().Set("Content-Type", "application/vnd.pypi.simple.v1+json") + ctx.Resp.Header().Add("Content-Type", "application/json") + if err := json.NewEncoder(ctx.Resp).Encode(content); err != nil { + log.Error("Render JSON failed: %v", err) + apiError(ctx, http.StatusInternalServerError, err) + } +} + +func PackageMetadata(ctx *context.Context) { + ctyp := ctx.Req.Header["Accept"] + if contentTypeSupported(ctyp, "application/vnd.pypi.simple.v1+json") { + JSONPackageMetadata(ctx) + } else { + HTMLPackageMetadata(ctx) + } +} + // DownloadPackageFile serves the content of a package func DownloadPackageFile(ctx *context.Context) { packageName := normalizer.Replace(ctx.Params("id")) diff --git a/tests/integration/api_packages_pypi_test.go b/tests/integration/api_packages_pypi_test.go index f025f8e577..45aa41c573 100644 --- a/tests/integration/api_packages_pypi_test.go +++ b/tests/integration/api_packages_pypi_test.go @@ -17,6 +17,7 @@ import ( "forgejo.org/models/packages" "forgejo.org/models/unittest" user_model "forgejo.org/models/user" + "forgejo.org/modules/json" "forgejo.org/modules/packages/pypi" "forgejo.org/tests" @@ -211,19 +212,20 @@ func TestPackagePyPI(t *testing.T) { assert.Equal(t, int64(2), pvs[0].DownloadCount) }) - t.Run("PackageMetadata", func(t *testing.T) { + hrefMatcher := regexp.MustCompile(fmt.Sprintf(`%s/files/%s/%s/test\..+#sha256=%s`, root, regexp.QuoteMeta(packageName), regexp.QuoteMeta(packageVersion), hashSHA256)) + + t.Run("PackageMetadataHTML", func(t *testing.T) { defer tests.PrintCurrentTest(t)() req := NewRequest(t, "GET", fmt.Sprintf("%s/simple/%s", root, packageName)). AddBasicAuth(user.Name) + req.Header["Accept"] = []string{"application/vnd.pypi.simple.v1+html"} resp := MakeRequest(t, req, http.StatusOK) htmlDoc := NewHTMLParser(t, resp.Body) nodes := htmlDoc.doc.Find("a").Nodes assert.Len(t, nodes, 2) - hrefMatcher := regexp.MustCompile(fmt.Sprintf(`%s/files/%s/%s/test\..+#sha256=%s`, root, regexp.QuoteMeta(packageName), regexp.QuoteMeta(packageVersion), hashSHA256)) - for _, a := range nodes { for _, att := range a.Attr { switch att.Key { @@ -237,4 +239,24 @@ func TestPackagePyPI(t *testing.T) { } } }) + + t.Run("PackageMetadataJSON", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/simple/%s", root, packageName)). + AddBasicAuth(user.Name) + req.Header["Accept"] = []string{"application/vnd.pypi.simple.v1+json"} + resp := MakeRequest(t, req, http.StatusOK) + assert.Greater(t, resp.Body.Len(), 3) + txt := make([]byte, resp.Body.Len()) + resp.Body.Read(txt) + var obj pypi.PackageJSON + require.NoError(t, json.Unmarshal(txt, &obj)) + assert.Equal(t, packageName, obj.Name) + assert.Equal(t, pypi.PackageMetaJSON{APIVersion: "1.4"}, obj.Meta) + for _, filed := range obj.Files { + hrefMatcher = regexp.MustCompile(fmt.Sprintf(`%s/files/%s/%s/test\.(tar\.gz)|(whl)`, root, regexp.QuoteMeta(packageName), regexp.QuoteMeta(packageVersion))) + assert.Regexp(t, hrefMatcher, filed.URL[21:]) + } + }) }