mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2026-05-12 22:10:25 +00:00
9abc1b0144
`DefaultMaxInSize` is an internal parameter for limiting the size of `field IN (...)` clauses in DB queries, which is a reasonable thing to do -- in addition to the errors noted when [originally introduced](https://github.com/go-gitea/gitea/pull/4594), there are technical limits that apply to each of PostgreSQL, MySQL, and SQLite which would prevent an unbounded size for a query like this. However: the size is incredibly small at 50, and, the implementation of `DefaultMaxInSize` is really wasteful with copy-and-paste coding. This PR: - introduces `GetByIDs` which fetches a `map[int64]*Model` from the database for an array of ID values, while respecting `IN` clause size limits - introduces `GetByFieldIn` which fetches a `map[int64][]*Model` from the database for an array of field values, while respecting `IN` clause size limits - uses `slices.Chunk` for other locations where queries are too complex for these implementations - bumps the `DefaultMaxInSize` parameter from 50 to 500, a conservative increase well under known limits, but 10x the current value: - PostgreSQL supports up to 1GB query text size with 65,535 parameters, but I've experienced performance degradation at high value counts - MySQL supports 64MB query text size without known limits of parameter count - SQLite supports 32,766 parameters in a query ## Checklist The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. All work and communication must conform to Forgejo's [AI Agreement](https://codeberg.org/forgejo/governance/src/branch/main/AIAgreement.md). There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org). ### Tests for Go changes - I added test coverage for Go changes... - [x] in their respective `*_test.go` for unit tests. - Refactored functions are assumed to be covered by existing tests to some extent; that assumption is probably wrong but the changes here are relatively easily reviewed for correctness as well. - [ ] in the `tests/integration` directory if it involves interactions with a live Forgejo server. - I ran... - [x] `make pr-go` before pushing ### Documentation - [ ] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change. - [x] I did not document these changes and I do not expect someone else to do it. ### Release notes - [ ] This change will be noticed by a Forgejo user or admin (feature, bug fix, performance, etc.). I suggest to include a release note for this change. - [x] This change is not visible to a Forgejo user or admin (refactor, dependency upgrade, etc.). I think there is no need to add a release note for this change. Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/11999 Reviewed-by: Andreas Ahlenstorf <aahlenst@noreply.codeberg.org> Co-authored-by: Mathieu Fenniak <mathieu@fenniak.net> Co-committed-by: Mathieu Fenniak <mathieu@fenniak.net>
64 lines
2 KiB
Go
64 lines
2 KiB
Go
// Copyright 2026 The Forgejo Authors. All rights reserved.
|
|
// SPDX-License-Identifier: GPL-3.0-or-later
|
|
|
|
package integration
|
|
|
|
import (
|
|
"fmt"
|
|
"testing"
|
|
|
|
actions_model "forgejo.org/models/actions"
|
|
"forgejo.org/models/db"
|
|
"forgejo.org/tests"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// These are basically unit tests, but by running them in the integration test suite they are tested against all
|
|
// supported database types.
|
|
|
|
func TestDatabaseDefaultMaxInSize(t *testing.T) {
|
|
defer tests.PrepareTestEnv(t)()
|
|
|
|
// Ensure there are more than db.DefaultMaxInSize objects in a table:
|
|
targetCount := db.DefaultMaxInSize * 2
|
|
for i := range targetCount {
|
|
_, err := actions_model.InsertVariable(t.Context(), 2, 2, fmt.Sprintf("VAR_%d", i), fmt.Sprintf("Value %d", i))
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
t.Run("GetByIDs", func(t *testing.T) {
|
|
defer tests.PrintCurrentTest(t)()
|
|
|
|
allActionVariables := make([]*actions_model.ActionVariable, 0, targetCount)
|
|
err := db.GetEngine(t.Context()).Find(&allActionVariables)
|
|
require.NoError(t, err)
|
|
|
|
allIDs := make([]int64, len(allActionVariables))
|
|
for i := range allActionVariables {
|
|
allIDs[i] = allActionVariables[i].ID
|
|
}
|
|
|
|
allActionVariablesAgain, err := db.GetByIDs(t.Context(), "id", allIDs, &actions_model.ActionVariable{})
|
|
require.NoError(t, err)
|
|
assert.Len(t, allActionVariablesAgain, len(allActionVariables))
|
|
})
|
|
|
|
t.Run("GetByFieldIn", func(t *testing.T) {
|
|
defer tests.PrintCurrentTest(t)()
|
|
|
|
allActionVariables := make([]*actions_model.ActionVariable, 0, targetCount)
|
|
err := db.GetEngine(t.Context()).Find(&allActionVariables)
|
|
require.NoError(t, err)
|
|
|
|
allIDs := make([]int64, len(allActionVariables))
|
|
for i := range allActionVariables {
|
|
allIDs[i] = allActionVariables[i].ID
|
|
}
|
|
|
|
allActionVariablesAgain, err := db.GetByFieldIn(t.Context(), "id", allIDs, &actions_model.ActionVariable{})
|
|
require.NoError(t, err)
|
|
assert.Len(t, allActionVariablesAgain, len(allActionVariables))
|
|
})
|
|
}
|