jojo/models/unittest/consistency.go
Mathieu Fenniak a3c6c78e08 fix: reduce deadlocks merging PRs by using caching for repo issue count stats (#9922)
The `repository` table has quite a few "count of related objects" fields on it, including the number of issues, closed issues, pull requests, and closed pull requests.  These fields specifically will cause deadlocks during concurrent PR merges as documented in #9785.

These fields are not used in database queries.  In order to eliminate the deadlock possibility on them, I've moved them to be calculated on-demand with caching, with the cache being invalidated in the same places that the recalc used to be triggered.

I've supplemented the already in-place automated testing with manual testing performing simple close & reopen of issues & PRs, and the counts which are used in the tabs at the top of the repo page are updated correctly as expected.

Near future work:
- Similar change can probably be performed to fix #9846
- Last known deadlock identified from #9785; I'm hoping to incorporate the synthetic deadlock test in a near future PR to prevent regressions

## Checklist

The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org).

### Tests

- Tests were already in-place covering these fields; they've been adjusted from using the fields to the new accessor methods.
- I added test coverage for Go changes...
  - [ ] in their respective `*_test.go` for unit tests.
  - [ ] in the `tests/integration` directory if it involves interactions with a live Forgejo server.
- I added test coverage for JavaScript changes...
  - [ ] in `web_src/js/*.test.js` if it can be unit tested.
  - [ ] in `tests/e2e/*.test.e2e.js` if it requires interactions with a live Forgejo server (see also the [developer guide for JavaScript testing](https://codeberg.org/forgejo/forgejo/src/branch/forgejo/tests/e2e/README.md#end-to-end-tests)).

### Documentation

- [ ] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change.
- [x] I did not document these changes and I do not expect someone else to do it.

### Release notes

- [ ] I do not want this change to show in the release notes.
- [x] I want the title to show in the release notes with a link to this pull request.
- [ ] I want the content of the `release-notes/<pull request number>.md` to be be used for the release notes instead of the title.

<!--start release-notes-assistant-->

## Release notes
<!--URL:https://codeberg.org/forgejo/forgejo-->
- Bug fixes
  - [PR](https://codeberg.org/forgejo/forgejo/pulls/9922): <!--number 9922 --><!--line 0 --><!--description cmVkdWNlIGRlYWRsb2NrcyBtZXJnaW5nIFBScyBieSB1c2luZyBjYWNoaW5nIGZvciByZXBvIGlzc3VlIGNvdW50IHN0YXRz-->reduce deadlocks merging PRs by using caching for repo issue count stats<!--description-->
<!--end release-notes-assistant-->

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/9922
Reviewed-by: Gusted <gusted@noreply.codeberg.org>
Co-authored-by: Mathieu Fenniak <mathieu@fenniak.net>
Co-committed-by: Mathieu Fenniak <mathieu@fenniak.net>
2025-10-31 23:50:05 +01:00

177 lines
7.1 KiB
Go

// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package unittest
import (
"reflect"
"strconv"
"strings"
"testing"
"forgejo.org/models/db"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"xorm.io/builder"
)
const (
// these const values are copied from `models` package to prevent from cycle-import
modelsUserTypeOrganization = 1
modelsRepoWatchModeDont = 2
modelsCommentTypeComment = 0
)
var consistencyCheckMap = make(map[string]func(t *testing.T, bean any))
// CheckConsistencyFor test that all matching database entries are consistent
func CheckConsistencyFor(t *testing.T, beansToCheck ...any) {
FlushAsyncCalcs(t)
for _, bean := range beansToCheck {
sliceType := reflect.SliceOf(reflect.TypeOf(bean))
sliceValue := reflect.MakeSlice(sliceType, 0, 10)
ptrToSliceValue := reflect.New(sliceType)
ptrToSliceValue.Elem().Set(sliceValue)
require.NoError(t, db.GetEngine(db.DefaultContext).Table(bean).Find(ptrToSliceValue.Interface()))
sliceValue = ptrToSliceValue.Elem()
for i := 0; i < sliceValue.Len(); i++ {
entity := sliceValue.Index(i).Interface()
checkForConsistency(t, entity)
}
}
}
func checkForConsistency(t *testing.T, bean any) {
tb, err := db.TableInfo(bean)
require.NoError(t, err)
f := consistencyCheckMap[tb.Name]
if f == nil {
assert.FailNow(t, "unknown bean type", "%#v", bean)
}
f(t, bean)
}
func init() {
parseBool := func(v string) bool {
b, _ := strconv.ParseBool(v)
return b
}
parseInt := func(v string) int {
i, _ := strconv.Atoi(v)
return i
}
checkForUserConsistency := func(t *testing.T, bean any) {
user := reflectionWrap(bean)
AssertCountByCond(t, "repository", builder.Eq{"owner_id": user.int("ID")}, user.int("NumRepos"))
AssertCountByCond(t, "star", builder.Eq{"uid": user.int("ID")}, user.int("NumStars"))
AssertCountByCond(t, "org_user", builder.Eq{"org_id": user.int("ID")}, user.int("NumMembers"))
AssertCountByCond(t, "team", builder.Eq{"org_id": user.int("ID")}, user.int("NumTeams"))
AssertCountByCond(t, "follow", builder.Eq{"user_id": user.int("ID")}, user.int("NumFollowing"))
AssertCountByCond(t, "follow", builder.Eq{"follow_id": user.int("ID")}, user.int("NumFollowers"))
if user.int("Type") != modelsUserTypeOrganization {
assert.Equal(t, 0, user.int("NumMembers"), "Unexpected number of members for user id: %d", user.int("ID"))
assert.Equal(t, 0, user.int("NumTeams"), "Unexpected number of teams for user id: %d", user.int("ID"))
}
}
checkForRepoConsistency := func(t *testing.T, bean any) {
repo := reflectionWrap(bean)
assert.Equal(t, repo.str("LowerName"), strings.ToLower(repo.str("Name")), "repo: %+v", repo)
AssertCountByCond(t, "star", builder.Eq{"repo_id": repo.int("ID")}, repo.int("NumStars"))
AssertCountByCond(t, "milestone", builder.Eq{"repo_id": repo.int("ID")}, repo.int("NumMilestones"))
AssertCountByCond(t, "repository", builder.Eq{"fork_id": repo.int("ID")}, repo.int("NumForks"))
if repo.bool("IsFork") {
AssertExistsAndLoadMap(t, "repository", builder.Eq{"id": repo.int("ForkID")})
}
actual := GetCountByCond(t, "watch", builder.Eq{"repo_id": repo.int("ID")}.
And(builder.Neq{"mode": modelsRepoWatchModeDont}))
assert.EqualValues(t, repo.int("NumWatches"), actual,
"Unexpected number of watches for repo id: %d", repo.int("ID"))
actual = GetCountByCond(t, "milestone", builder.Eq{"is_closed": true, "repo_id": repo.int("ID")})
assert.EqualValues(t, repo.int("NumClosedMilestones"), actual,
"Unexpected number of closed milestones for repo id: %d", repo.int("ID"))
}
checkForIssueConsistency := func(t *testing.T, bean any) {
issue := reflectionWrap(bean)
typeComment := modelsCommentTypeComment
actual := GetCountByCond(t, "comment", builder.Eq{"`type`": typeComment, "issue_id": issue.int("ID")})
assert.EqualValues(t, issue.int("NumComments"), actual, "Unexpected number of comments for issue id: %d", issue.int("ID"))
if issue.bool("IsPull") {
prRow := AssertExistsAndLoadMap(t, "pull_request", builder.Eq{"issue_id": issue.int("ID")})
assert.Equal(t, parseInt(prRow["index"]), issue.int("Index"), "Unexpected index for issue id: %d", issue.int("ID"))
}
}
checkForPullRequestConsistency := func(t *testing.T, bean any) {
pr := reflectionWrap(bean)
issueRow := AssertExistsAndLoadMap(t, "issue", builder.Eq{"id": pr.int("IssueID")})
assert.True(t, parseBool(issueRow["is_pull"]))
assert.Equal(t, parseInt(issueRow["index"]), pr.int("Index"), "Unexpected index for pull request id: %d", pr.int("ID"))
}
checkForMilestoneConsistency := func(t *testing.T, bean any) {
milestone := reflectionWrap(bean)
AssertCountByCond(t, "issue", builder.Eq{"milestone_id": milestone.int("ID")}, milestone.int("NumIssues"))
actual := GetCountByCond(t, "issue", builder.Eq{"is_closed": true, "milestone_id": milestone.int("ID")})
assert.EqualValues(t, milestone.int("NumClosedIssues"), actual, "Unexpected number of closed issues for milestone id: %d", milestone.int("ID"))
completeness := 0
if milestone.int("NumIssues") > 0 {
completeness = milestone.int("NumClosedIssues") * 100 / milestone.int("NumIssues")
}
assert.Equal(t, completeness, milestone.int("Completeness"))
}
checkForLabelConsistency := func(t *testing.T, bean any) {
label := reflectionWrap(bean)
issueLabels, err := db.GetEngine(db.DefaultContext).Table("issue_label").
Where(builder.Eq{"label_id": label.int("ID")}).
Query()
require.NoError(t, err)
assert.Len(t, issueLabels, label.int("NumIssues"), "Unexpected number of issue for label id: %d", label.int("ID"))
issueIDs := make([]int, len(issueLabels))
for i, issueLabel := range issueLabels {
issueIDs[i], _ = strconv.Atoi(string(issueLabel["issue_id"]))
}
expected := int64(0)
if len(issueIDs) > 0 {
expected = GetCountByCond(t, "issue", builder.In("id", issueIDs).And(builder.Eq{"is_closed": true}))
}
assert.EqualValues(t, expected, label.int("NumClosedIssues"), "Unexpected number of closed issues for label id: %d", label.int("ID"))
}
checkForTeamConsistency := func(t *testing.T, bean any) {
team := reflectionWrap(bean)
AssertCountByCond(t, "team_user", builder.Eq{"team_id": team.int("ID")}, team.int("NumMembers"))
AssertCountByCond(t, "team_repo", builder.Eq{"team_id": team.int("ID")}, team.int("NumRepos"))
}
checkForActionConsistency := func(t *testing.T, bean any) {
action := reflectionWrap(bean)
if action.int("RepoID") != 1700 { // dangling intentional
repoRow := AssertExistsAndLoadMap(t, "repository", builder.Eq{"id": action.int("RepoID")})
assert.Equal(t, parseBool(repoRow["is_private"]), action.bool("IsPrivate"), "Unexpected is_private field for action id: %d", action.int("ID"))
}
}
consistencyCheckMap["user"] = checkForUserConsistency
consistencyCheckMap["repository"] = checkForRepoConsistency
consistencyCheckMap["issue"] = checkForIssueConsistency
consistencyCheckMap["pull_request"] = checkForPullRequestConsistency
consistencyCheckMap["milestone"] = checkForMilestoneConsistency
consistencyCheckMap["label"] = checkForLabelConsistency
consistencyCheckMap["team"] = checkForTeamConsistency
consistencyCheckMap["action"] = checkForActionConsistency
}