peezy-tech/jojo
2
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo.git synced 2026-05-12 22:10:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[v15.0/forgejo] fix: duplicate key violates unique constraint in concurrent debian package uploads (#11833)

Browse source 
**Backport:** https://codeberg.org/forgejo/forgejo/pulls/11776

Fixes #11438.

Whenever a "unique constraint violation" error is encountered by package mutation, detect if a `xorm.ErrUniqueConstraintViolation` error occurs.  If it does, retry the entire transaction.

## Checklist

The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org).

### Tests for Go changes

(can be removed for JavaScript changes)

- I added test coverage for Go changes...
  - [ ] in their respective `*_test.go` for unit tests.
  - [ ] in the `tests/integration` directory if it involves interactions with a live Forgejo server.
- I ran...
  - [ ] `make pr-go` before pushing

### Documentation

- [ ] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change.
- [x] I did not document these changes and I do not expect someone else to do it.

### Release notes

- [x] This change will be noticed by a Forgejo user or admin (feature, bug fix, performance, etc.). I suggest to include a release note for this change.
- [ ] This change is not visible to a Forgejo user or admin (refactor, dependency upgrade, etc.). I think there is no need to add a release note for this change.

Co-authored-by: Mathieu Fenniak <mathieu@fenniak.net>
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/11833
Reviewed-by: Mathieu Fenniak <mfenniak@noreply.codeberg.org>
Co-authored-by: forgejo-backport-action <forgejo-backport-action@noreply.codeberg.org>
Co-committed-by: forgejo-backport-action <forgejo-backport-action@noreply.codeberg.org>
This commit is contained in:
forgejo-backport-action 2026-03-27 01:36:18 +01:00 • committed by Mathieu Fenniak
parent 4230ba6ed0
commit ebac8b38cb
8 changed files with 286 additions and 79 deletions
Download patch file Download diff file Expand all files Collapse all files

111
tests/integration/api_packages_debian_test.go
View file

@ -11,6 +11,7 @@ import (
"io"
"net/http"
"strings"
"sync"
"testing"
"forgejo.org/models/db"
@ -19,6 +20,7 @@ import (
user_model "forgejo.org/models/user"
"forgejo.org/modules/base"
debian_module "forgejo.org/modules/packages/debian"
"forgejo.org/modules/setting"
"forgejo.org/tests"
"github.com/blakesmith/ar"
@ -26,6 +28,32 @@ import (
"github.com/stretchr/testify/require"
)
func createDebianArchive(name, version, architecture, packageDescription string) io.Reader {
var cbuf bytes.Buffer
zw := gzip.NewWriter(&cbuf)
tw := tar.NewWriter(zw)
tw.WriteHeader(&tar.Header{
Name: "control",
Mode: 0o600,
Size: 50,
})
fmt.Fprintf(tw, "Package: %s\nVersion: %s\nArchitecture: %s\nDescription: %s\n", name, version, architecture, packageDescription)
tw.Close()
zw.Close()
var buf bytes.Buffer
aw := ar.NewWriter(&buf)
aw.WriteGlobalHeader()
hdr := &ar.Header{
Name: "control.tar.gz",
Mode: 0o600,
Size: int64(cbuf.Len()),
}
aw.WriteHeader(hdr)
aw.Write(cbuf.Bytes())
return &buf
}
func TestPackageDebian(t *testing.T) {
defer tests.PrepareTestEnv(t)()
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
@ -35,32 +63,6 @@ func TestPackageDebian(t *testing.T) {
packageVersion2 := "1.0.4"
packageDescription := "Package Description"
createArchive := func(name, version, architecture string) io.Reader {
var cbuf bytes.Buffer
zw := gzip.NewWriter(&cbuf)
tw := tar.NewWriter(zw)
tw.WriteHeader(&tar.Header{
Name: "control",
Mode: 0o600,
Size: 50,
})
fmt.Fprintf(tw, "Package: %s\nVersion: %s\nArchitecture: %s\nDescription: %s\n", name, version, architecture, packageDescription)
tw.Close()
zw.Close()
var buf bytes.Buffer
aw := ar.NewWriter(&buf)
aw.WriteGlobalHeader()
hdr := &ar.Header{
Name: "control.tar.gz",
Mode: 0o600,
Size: int64(cbuf.Len()),
}
aw.WriteHeader(hdr)
aw.Write(cbuf.Bytes())
return &buf
}
distributions := []string{"test", "gitea"}
components := []string{"main", "stable"}
architectures := []string{"all", "amd64"}
@ -97,16 +99,16 @@ func TestPackageDebian(t *testing.T) {
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusBadRequest)
req = NewRequestWithBody(t, "PUT", uploadURL, createArchive("", "", "")).
req = NewRequestWithBody(t, "PUT", uploadURL, createDebianArchive("", "", "", packageDescription)).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusBadRequest)
req = NewRequestWithBody(t, "PUT", uploadURL, createArchive(packageName, packageVersion, architecture)).
req = NewRequestWithBody(t, "PUT", uploadURL, createDebianArchive(packageName, packageVersion, architecture, packageDescription)).
AddBasicAuth(user.Name).
SetHeader("content-type", "multipart/form-data")
MakeRequest(t, req, http.StatusBadRequest)
req = NewRequestWithBody(t, "PUT", uploadURL, createArchive(packageName, packageVersion, architecture)).
req = NewRequestWithBody(t, "PUT", uploadURL, createDebianArchive(packageName, packageVersion, architecture, packageDescription)).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
@ -154,7 +156,7 @@ func TestPackageDebian(t *testing.T) {
return seen
})
req = NewRequestWithBody(t, "PUT", uploadURL, createArchive(packageName, packageVersion, architecture)).
req = NewRequestWithBody(t, "PUT", uploadURL, createDebianArchive(packageName, packageVersion, architecture, packageDescription)).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusConflict)
})
@ -171,7 +173,7 @@ func TestPackageDebian(t *testing.T) {
t.Run("Packages", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequestWithBody(t, "PUT", uploadURL, createArchive(packageName, packageVersion2, architecture)).
req := NewRequestWithBody(t, "PUT", uploadURL, createDebianArchive(packageName, packageVersion2, architecture, packageDescription)).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
@ -308,3 +310,50 @@ func TestPackageDebian(t *testing.T) {
require.Contains(t, body, fmt.Sprintf("Version: %s", packageVersion2))
})
}
func TestPackageDebianConcurrent(t *testing.T) {
if setting.Database.Type.IsSQLite3() {
// Concurrency test fails on SQLite w/ "database is locked"
t.Skip()
}
defer tests.PrepareTestEnv(t)()
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
distribution := "test"
component := "main"
architecture := "amd64"
packageName := "gitea"
packageDescription := "Package Description"
rootURL := fmt.Sprintf("/api/packages/%s/debian", user.Name)
uploadURL := fmt.Sprintf("%s/pool/%s/%s/upload", rootURL, distribution, component)
t.Run("Concurrent Upload", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
var wg sync.WaitGroup
packageCount := 10
for i := range packageCount {
wg.Go(func() {
req := NewRequestWithBody(t, "PUT", uploadURL,
createDebianArchive(packageName, fmt.Sprintf("1.0.%d", i), architecture, packageDescription)).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
})
}
wg.Wait()
url := fmt.Sprintf("%s/dists/%s/%s/binary-%s/Packages", rootURL, distribution, component, architecture)
req := NewRequest(t, "GET", url)
resp := MakeRequest(t, req, http.StatusOK)
body := resp.Body.String()
assert.Contains(t, body, fmt.Sprintf("Package: %s\n", packageName))
for i := range packageCount {
assert.Contains(t, body, fmt.Sprintf("Version: 1.0.%d\n", i))
}
})
}