mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2026-05-12 22:10:25 +00:00
`DefaultMaxInSize` is an internal parameter for limiting the size of `field IN (...)` clauses in DB queries, which is a reasonable thing to do -- in addition to the errors noted when [originally introduced](https://github.com/go-gitea/gitea/pull/4594), there are technical limits that apply to each of PostgreSQL, MySQL, and SQLite which would prevent an unbounded size for a query like this. However: the size is incredibly small at 50, and, the implementation of `DefaultMaxInSize` is really wasteful with copy-and-paste coding. This PR: - introduces `GetByIDs` which fetches a `map[int64]*Model` from the database for an array of ID values, while respecting `IN` clause size limits - introduces `GetByFieldIn` which fetches a `map[int64][]*Model` from the database for an array of field values, while respecting `IN` clause size limits - uses `slices.Chunk` for other locations where queries are too complex for these implementations - bumps the `DefaultMaxInSize` parameter from 50 to 500, a conservative increase well under known limits, but 10x the current value: - PostgreSQL supports up to 1GB query text size with 65,535 parameters, but I've experienced performance degradation at high value counts - MySQL supports 64MB query text size without known limits of parameter count - SQLite supports 32,766 parameters in a query ## Checklist The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. All work and communication must conform to Forgejo's [AI Agreement](https://codeberg.org/forgejo/governance/src/branch/main/AIAgreement.md). There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org). ### Tests for Go changes - I added test coverage for Go changes... - [x] in their respective `*_test.go` for unit tests. - Refactored functions are assumed to be covered by existing tests to some extent; that assumption is probably wrong but the changes here are relatively easily reviewed for correctness as well. - [ ] in the `tests/integration` directory if it involves interactions with a live Forgejo server. - I ran... - [x] `make pr-go` before pushing ### Documentation - [ ] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change. - [x] I did not document these changes and I do not expect someone else to do it. ### Release notes - [ ] This change will be noticed by a Forgejo user or admin (feature, bug fix, performance, etc.). I suggest to include a release note for this change. - [x] This change is not visible to a Forgejo user or admin (refactor, dependency upgrade, etc.). I think there is no need to add a release note for this change. Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/11999 Reviewed-by: Andreas Ahlenstorf <aahlenst@noreply.codeberg.org> Co-authored-by: Mathieu Fenniak <mathieu@fenniak.net> Co-committed-by: Mathieu Fenniak <mathieu@fenniak.net>
215 lines
5 KiB
Go
215 lines
5 KiB
Go
// Copyright 2020 The Gitea Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package db
|
|
|
|
import (
|
|
"context"
|
|
|
|
"forgejo.org/modules/setting"
|
|
|
|
"xorm.io/builder"
|
|
"xorm.io/xorm"
|
|
)
|
|
|
|
const (
|
|
// DefaultMaxInSize represents default variables number on IN () in SQL
|
|
DefaultMaxInSize = 500
|
|
defaultFindSliceSize = 10
|
|
)
|
|
|
|
// Paginator is the base for different ListOptions types
|
|
type Paginator interface {
|
|
GetSkipTake() (skip, take int)
|
|
IsListAll() bool
|
|
}
|
|
|
|
// SetSessionPagination sets pagination for a database session
|
|
func SetSessionPagination(sess Engine, p Paginator) *xorm.Session {
|
|
skip, take := p.GetSkipTake()
|
|
|
|
return sess.Limit(take, skip)
|
|
}
|
|
|
|
// ListOptions options to paginate results
|
|
type ListOptions struct {
|
|
PageSize int
|
|
Page int // start from 1
|
|
ListAll bool // if true, then PageSize and Page will not be taken
|
|
}
|
|
|
|
var ListOptionsAll = ListOptions{ListAll: true}
|
|
|
|
var (
|
|
_ Paginator = &ListOptions{}
|
|
_ FindOptions = ListOptions{}
|
|
)
|
|
|
|
// GetSkipTake returns the skip and take values
|
|
func (opts *ListOptions) GetSkipTake() (skip, take int) {
|
|
opts.SetDefaultValues()
|
|
return (opts.Page - 1) * opts.PageSize, opts.PageSize
|
|
}
|
|
|
|
func (opts ListOptions) GetPage() int {
|
|
return opts.Page
|
|
}
|
|
|
|
func (opts ListOptions) GetPageSize() int {
|
|
return opts.PageSize
|
|
}
|
|
|
|
// IsListAll indicates PageSize and Page will be ignored
|
|
func (opts ListOptions) IsListAll() bool {
|
|
return opts.ListAll
|
|
}
|
|
|
|
// SetDefaultValues sets default values
|
|
func (opts *ListOptions) SetDefaultValues() {
|
|
if opts.PageSize <= 0 {
|
|
opts.PageSize = setting.API.DefaultPagingNum
|
|
}
|
|
if opts.PageSize > setting.API.MaxResponseItems {
|
|
opts.PageSize = setting.API.MaxResponseItems
|
|
}
|
|
if opts.Page <= 0 {
|
|
opts.Page = 1
|
|
}
|
|
}
|
|
|
|
func (opts ListOptions) ToConds() builder.Cond {
|
|
return builder.NewCond()
|
|
}
|
|
|
|
// AbsoluteListOptions absolute options to paginate results
|
|
type AbsoluteListOptions struct {
|
|
skip int
|
|
take int
|
|
}
|
|
|
|
var _ Paginator = &AbsoluteListOptions{}
|
|
|
|
// NewAbsoluteListOptions creates a list option with applied limits
|
|
func NewAbsoluteListOptions(skip, take int) *AbsoluteListOptions {
|
|
if skip < 0 {
|
|
skip = 0
|
|
}
|
|
if take <= 0 {
|
|
take = setting.API.DefaultPagingNum
|
|
}
|
|
if take > setting.API.MaxResponseItems {
|
|
take = setting.API.MaxResponseItems
|
|
}
|
|
return &AbsoluteListOptions{skip, take}
|
|
}
|
|
|
|
// IsListAll will always return false
|
|
func (opts *AbsoluteListOptions) IsListAll() bool {
|
|
return false
|
|
}
|
|
|
|
// GetSkipTake returns the skip and take values
|
|
func (opts *AbsoluteListOptions) GetSkipTake() (skip, take int) {
|
|
return opts.skip, opts.take
|
|
}
|
|
|
|
// FindOptions represents a find options
|
|
type FindOptions interface {
|
|
GetPage() int
|
|
GetPageSize() int
|
|
IsListAll() bool
|
|
ToConds() builder.Cond
|
|
}
|
|
|
|
type JoinFunc func(sess Engine) error
|
|
|
|
type FindOptionsJoin interface {
|
|
ToJoins() []JoinFunc
|
|
}
|
|
|
|
type FindOptionsOrder interface {
|
|
ToOrders() string
|
|
}
|
|
|
|
// Find represents a common find function which accept an options interface
|
|
func Find[T any](ctx context.Context, opts FindOptions) ([]*T, error) {
|
|
sess := GetEngine(ctx).Where(opts.ToConds())
|
|
|
|
if joinOpt, ok := opts.(FindOptionsJoin); ok {
|
|
for _, joinFunc := range joinOpt.ToJoins() {
|
|
if err := joinFunc(sess); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
}
|
|
if orderOpt, ok := opts.(FindOptionsOrder); ok {
|
|
if order := orderOpt.ToOrders(); order != "" {
|
|
sess.OrderBy(order)
|
|
}
|
|
}
|
|
|
|
page, pageSize := opts.GetPage(), opts.GetPageSize()
|
|
if !opts.IsListAll() && pageSize > 0 {
|
|
if page == 0 {
|
|
page = 1
|
|
}
|
|
sess.Limit(pageSize, (page-1)*pageSize)
|
|
}
|
|
|
|
findPageSize := defaultFindSliceSize
|
|
if pageSize > 0 {
|
|
findPageSize = pageSize
|
|
}
|
|
objects := make([]*T, 0, findPageSize)
|
|
if err := sess.Find(&objects); err != nil {
|
|
return nil, err
|
|
}
|
|
return objects, nil
|
|
}
|
|
|
|
// Count represents a common count function which accept an options interface
|
|
func Count[T any](ctx context.Context, opts FindOptions) (int64, error) {
|
|
sess := GetEngine(ctx).Where(opts.ToConds())
|
|
if joinOpt, ok := opts.(FindOptionsJoin); ok {
|
|
for _, joinFunc := range joinOpt.ToJoins() {
|
|
if err := joinFunc(sess); err != nil {
|
|
return 0, err
|
|
}
|
|
}
|
|
}
|
|
|
|
var object T
|
|
return sess.Count(&object)
|
|
}
|
|
|
|
// FindAndCount represents a common findandcount function which accept an options interface
|
|
func FindAndCount[T any](ctx context.Context, opts FindOptions) ([]*T, int64, error) {
|
|
sess := GetEngine(ctx).Where(opts.ToConds())
|
|
page, pageSize := opts.GetPage(), opts.GetPageSize()
|
|
if !opts.IsListAll() && pageSize > 0 && page >= 1 {
|
|
sess.Limit(pageSize, (page-1)*pageSize)
|
|
}
|
|
if joinOpt, ok := opts.(FindOptionsJoin); ok {
|
|
for _, joinFunc := range joinOpt.ToJoins() {
|
|
if err := joinFunc(sess); err != nil {
|
|
return nil, 0, err
|
|
}
|
|
}
|
|
}
|
|
if orderOpt, ok := opts.(FindOptionsOrder); ok {
|
|
if order := orderOpt.ToOrders(); order != "" {
|
|
sess.OrderBy(order)
|
|
}
|
|
}
|
|
|
|
findPageSize := defaultFindSliceSize
|
|
if pageSize > 0 {
|
|
findPageSize = pageSize
|
|
}
|
|
objects := make([]*T, 0, findPageSize)
|
|
cnt, err := sess.FindAndCount(&objects)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
return objects, cnt, nil
|
|
}
|