jojo/models/db/list.go
Mathieu Fenniak 9abc1b0144 refactor: reduce code duplication when accessing DefaultMaxInSize (#11999)
`DefaultMaxInSize` is an internal parameter for limiting the size of `field IN (...)` clauses in DB queries, which is a reasonable thing to do -- in addition to the errors noted when [originally introduced](https://github.com/go-gitea/gitea/pull/4594), there are technical limits that apply to each of PostgreSQL, MySQL, and SQLite which would prevent an unbounded size for a query like this.  However: the size is incredibly small at 50, and, the implementation of `DefaultMaxInSize` is really wasteful with copy-and-paste coding.

This PR:
- introduces `GetByIDs` which fetches a `map[int64]*Model` from the database for an array of ID values, while respecting `IN` clause size limits
- introduces `GetByFieldIn` which fetches a `map[int64][]*Model` from the database for an array of field values, while respecting `IN` clause size limits
- uses `slices.Chunk` for other locations where queries are too complex for these implementations
- bumps the `DefaultMaxInSize` parameter from 50 to 500, a conservative increase well under known limits, but 10x the current value:
    - PostgreSQL supports up to 1GB query text size with 65,535 parameters, but I've experienced performance degradation at high value counts
    - MySQL supports 64MB query text size without known limits of parameter count
    - SQLite supports 32,766 parameters in a query

## Checklist

The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. All work and communication must conform to Forgejo's [AI Agreement](https://codeberg.org/forgejo/governance/src/branch/main/AIAgreement.md). There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org).

### Tests for Go changes

- I added test coverage for Go changes...
  - [x] in their respective `*_test.go` for unit tests.
      - Refactored functions are assumed to be covered by existing tests to some extent; that assumption is probably wrong but the changes here are relatively easily reviewed for correctness as well.
  - [ ] in the `tests/integration` directory if it involves interactions with a live Forgejo server.
- I ran...
  - [x] `make pr-go` before pushing

### Documentation

- [ ] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change.
- [x] I did not document these changes and I do not expect someone else to do it.

### Release notes

- [ ] This change will be noticed by a Forgejo user or admin (feature, bug fix, performance, etc.). I suggest to include a release note for this change.
- [x] This change is not visible to a Forgejo user or admin (refactor, dependency upgrade, etc.). I think there is no need to add a release note for this change.

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/11999
Reviewed-by: Andreas Ahlenstorf <aahlenst@noreply.codeberg.org>
Co-authored-by: Mathieu Fenniak <mathieu@fenniak.net>
Co-committed-by: Mathieu Fenniak <mathieu@fenniak.net>
2026-04-05 22:03:45 +02:00

215 lines
5 KiB
Go

// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package db
import (
"context"
"forgejo.org/modules/setting"
"xorm.io/builder"
"xorm.io/xorm"
)
const (
// DefaultMaxInSize represents default variables number on IN () in SQL
DefaultMaxInSize = 500
defaultFindSliceSize = 10
)
// Paginator is the base for different ListOptions types
type Paginator interface {
GetSkipTake() (skip, take int)
IsListAll() bool
}
// SetSessionPagination sets pagination for a database session
func SetSessionPagination(sess Engine, p Paginator) *xorm.Session {
skip, take := p.GetSkipTake()
return sess.Limit(take, skip)
}
// ListOptions options to paginate results
type ListOptions struct {
PageSize int
Page int // start from 1
ListAll bool // if true, then PageSize and Page will not be taken
}
var ListOptionsAll = ListOptions{ListAll: true}
var (
_ Paginator = &ListOptions{}
_ FindOptions = ListOptions{}
)
// GetSkipTake returns the skip and take values
func (opts *ListOptions) GetSkipTake() (skip, take int) {
opts.SetDefaultValues()
return (opts.Page - 1) * opts.PageSize, opts.PageSize
}
func (opts ListOptions) GetPage() int {
return opts.Page
}
func (opts ListOptions) GetPageSize() int {
return opts.PageSize
}
// IsListAll indicates PageSize and Page will be ignored
func (opts ListOptions) IsListAll() bool {
return opts.ListAll
}
// SetDefaultValues sets default values
func (opts *ListOptions) SetDefaultValues() {
if opts.PageSize <= 0 {
opts.PageSize = setting.API.DefaultPagingNum
}
if opts.PageSize > setting.API.MaxResponseItems {
opts.PageSize = setting.API.MaxResponseItems
}
if opts.Page <= 0 {
opts.Page = 1
}
}
func (opts ListOptions) ToConds() builder.Cond {
return builder.NewCond()
}
// AbsoluteListOptions absolute options to paginate results
type AbsoluteListOptions struct {
skip int
take int
}
var _ Paginator = &AbsoluteListOptions{}
// NewAbsoluteListOptions creates a list option with applied limits
func NewAbsoluteListOptions(skip, take int) *AbsoluteListOptions {
if skip < 0 {
skip = 0
}
if take <= 0 {
take = setting.API.DefaultPagingNum
}
if take > setting.API.MaxResponseItems {
take = setting.API.MaxResponseItems
}
return &AbsoluteListOptions{skip, take}
}
// IsListAll will always return false
func (opts *AbsoluteListOptions) IsListAll() bool {
return false
}
// GetSkipTake returns the skip and take values
func (opts *AbsoluteListOptions) GetSkipTake() (skip, take int) {
return opts.skip, opts.take
}
// FindOptions represents a find options
type FindOptions interface {
GetPage() int
GetPageSize() int
IsListAll() bool
ToConds() builder.Cond
}
type JoinFunc func(sess Engine) error
type FindOptionsJoin interface {
ToJoins() []JoinFunc
}
type FindOptionsOrder interface {
ToOrders() string
}
// Find represents a common find function which accept an options interface
func Find[T any](ctx context.Context, opts FindOptions) ([]*T, error) {
sess := GetEngine(ctx).Where(opts.ToConds())
if joinOpt, ok := opts.(FindOptionsJoin); ok {
for _, joinFunc := range joinOpt.ToJoins() {
if err := joinFunc(sess); err != nil {
return nil, err
}
}
}
if orderOpt, ok := opts.(FindOptionsOrder); ok {
if order := orderOpt.ToOrders(); order != "" {
sess.OrderBy(order)
}
}
page, pageSize := opts.GetPage(), opts.GetPageSize()
if !opts.IsListAll() && pageSize > 0 {
if page == 0 {
page = 1
}
sess.Limit(pageSize, (page-1)*pageSize)
}
findPageSize := defaultFindSliceSize
if pageSize > 0 {
findPageSize = pageSize
}
objects := make([]*T, 0, findPageSize)
if err := sess.Find(&objects); err != nil {
return nil, err
}
return objects, nil
}
// Count represents a common count function which accept an options interface
func Count[T any](ctx context.Context, opts FindOptions) (int64, error) {
sess := GetEngine(ctx).Where(opts.ToConds())
if joinOpt, ok := opts.(FindOptionsJoin); ok {
for _, joinFunc := range joinOpt.ToJoins() {
if err := joinFunc(sess); err != nil {
return 0, err
}
}
}
var object T
return sess.Count(&object)
}
// FindAndCount represents a common findandcount function which accept an options interface
func FindAndCount[T any](ctx context.Context, opts FindOptions) ([]*T, int64, error) {
sess := GetEngine(ctx).Where(opts.ToConds())
page, pageSize := opts.GetPage(), opts.GetPageSize()
if !opts.IsListAll() && pageSize > 0 && page >= 1 {
sess.Limit(pageSize, (page-1)*pageSize)
}
if joinOpt, ok := opts.(FindOptionsJoin); ok {
for _, joinFunc := range joinOpt.ToJoins() {
if err := joinFunc(sess); err != nil {
return nil, 0, err
}
}
}
if orderOpt, ok := opts.(FindOptionsOrder); ok {
if order := orderOpt.ToOrders(); order != "" {
sess.OrderBy(order)
}
}
findPageSize := defaultFindSliceSize
if pageSize > 0 {
findPageSize = pageSize
}
objects := make([]*T, 0, findPageSize)
cnt, err := sess.FindAndCount(&objects)
if err != nil {
return nil, 0, err
}
return objects, cnt, nil
}