mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2026-05-12 22:10:25 +00:00
`DefaultMaxInSize` is an internal parameter for limiting the size of `field IN (...)` clauses in DB queries, which is a reasonable thing to do -- in addition to the errors noted when [originally introduced](https://github.com/go-gitea/gitea/pull/4594), there are technical limits that apply to each of PostgreSQL, MySQL, and SQLite which would prevent an unbounded size for a query like this. However: the size is incredibly small at 50, and, the implementation of `DefaultMaxInSize` is really wasteful with copy-and-paste coding. This PR: - introduces `GetByIDs` which fetches a `map[int64]*Model` from the database for an array of ID values, while respecting `IN` clause size limits - introduces `GetByFieldIn` which fetches a `map[int64][]*Model` from the database for an array of field values, while respecting `IN` clause size limits - uses `slices.Chunk` for other locations where queries are too complex for these implementations - bumps the `DefaultMaxInSize` parameter from 50 to 500, a conservative increase well under known limits, but 10x the current value: - PostgreSQL supports up to 1GB query text size with 65,535 parameters, but I've experienced performance degradation at high value counts - MySQL supports 64MB query text size without known limits of parameter count - SQLite supports 32,766 parameters in a query ## Checklist The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. All work and communication must conform to Forgejo's [AI Agreement](https://codeberg.org/forgejo/governance/src/branch/main/AIAgreement.md). There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org). ### Tests for Go changes - I added test coverage for Go changes... - [x] in their respective `*_test.go` for unit tests. - Refactored functions are assumed to be covered by existing tests to some extent; that assumption is probably wrong but the changes here are relatively easily reviewed for correctness as well. - [ ] in the `tests/integration` directory if it involves interactions with a live Forgejo server. - I ran... - [x] `make pr-go` before pushing ### Documentation - [ ] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change. - [x] I did not document these changes and I do not expect someone else to do it. ### Release notes - [ ] This change will be noticed by a Forgejo user or admin (feature, bug fix, performance, etc.). I suggest to include a release note for this change. - [x] This change is not visible to a Forgejo user or admin (refactor, dependency upgrade, etc.). I think there is no need to add a release note for this change. Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/11999 Reviewed-by: Andreas Ahlenstorf <aahlenst@noreply.codeberg.org> Co-authored-by: Mathieu Fenniak <mathieu@fenniak.net> Co-committed-by: Mathieu Fenniak <mathieu@fenniak.net>
376 lines
9.9 KiB
Go
376 lines
9.9 KiB
Go
// Copyright 2024 The Gitea Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package activities
|
|
|
|
import (
|
|
"context"
|
|
|
|
"forgejo.org/models/db"
|
|
issues_model "forgejo.org/models/issues"
|
|
access_model "forgejo.org/models/perm/access"
|
|
repo_model "forgejo.org/models/repo"
|
|
"forgejo.org/models/unit"
|
|
user_model "forgejo.org/models/user"
|
|
"forgejo.org/modules/container"
|
|
"forgejo.org/modules/log"
|
|
|
|
"xorm.io/builder"
|
|
)
|
|
|
|
// FindNotificationOptions represent the filters for notifications. If an ID is 0 it will be ignored.
|
|
type FindNotificationOptions struct {
|
|
db.ListOptions
|
|
UserID int64
|
|
RepoID int64
|
|
IssueID int64
|
|
Status []NotificationStatus
|
|
Source []NotificationSource
|
|
UpdatedAfterUnix int64
|
|
UpdatedBeforeUnix int64
|
|
}
|
|
|
|
// ToCond will convert each condition into a xorm-Cond
|
|
func (opts FindNotificationOptions) ToConds() builder.Cond {
|
|
cond := builder.NewCond()
|
|
if opts.UserID != 0 {
|
|
cond = cond.And(builder.Eq{"notification.user_id": opts.UserID})
|
|
}
|
|
if opts.RepoID != 0 {
|
|
cond = cond.And(builder.Eq{"notification.repo_id": opts.RepoID})
|
|
}
|
|
if opts.IssueID != 0 {
|
|
cond = cond.And(builder.Eq{"notification.issue_id": opts.IssueID})
|
|
}
|
|
if len(opts.Status) > 0 {
|
|
if len(opts.Status) == 1 {
|
|
cond = cond.And(builder.Eq{"notification.status": opts.Status[0]})
|
|
} else {
|
|
cond = cond.And(builder.In("notification.status", opts.Status))
|
|
}
|
|
}
|
|
if len(opts.Source) > 0 {
|
|
cond = cond.And(builder.In("notification.source", opts.Source))
|
|
}
|
|
if opts.UpdatedAfterUnix != 0 {
|
|
cond = cond.And(builder.Gte{"notification.updated_unix": opts.UpdatedAfterUnix})
|
|
}
|
|
if opts.UpdatedBeforeUnix != 0 {
|
|
cond = cond.And(builder.Lte{"notification.updated_unix": opts.UpdatedBeforeUnix})
|
|
}
|
|
return cond
|
|
}
|
|
|
|
func (opts FindNotificationOptions) ToOrders() string {
|
|
return "notification.updated_unix DESC"
|
|
}
|
|
|
|
// CreateOrUpdateIssueNotifications creates an issue notification
|
|
// for each watcher, or updates it if already exists
|
|
// receiverID > 0 just send to receiver, else send to all watcher
|
|
func CreateOrUpdateIssueNotifications(ctx context.Context, issueID, commentID, notificationAuthorID, receiverID int64) error {
|
|
ctx, committer, err := db.TxContext(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer committer.Close()
|
|
|
|
if err := createOrUpdateIssueNotifications(ctx, issueID, commentID, notificationAuthorID, receiverID); err != nil {
|
|
return err
|
|
}
|
|
|
|
return committer.Commit()
|
|
}
|
|
|
|
func createOrUpdateIssueNotifications(ctx context.Context, issueID, commentID, notificationAuthorID, receiverID int64) error {
|
|
// init
|
|
var toNotify container.Set[int64]
|
|
notifications, err := db.Find[Notification](ctx, FindNotificationOptions{
|
|
IssueID: issueID,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
issue, err := issues_model.GetIssueByID(ctx, issueID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if receiverID > 0 {
|
|
toNotify = make(container.Set[int64], 1)
|
|
toNotify.Add(receiverID)
|
|
} else {
|
|
toNotify = make(container.Set[int64], 32)
|
|
issueWatches, err := issues_model.GetIssueWatchersIDs(ctx, issueID, true)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
toNotify.AddMultiple(issueWatches...)
|
|
if !issue.IsPull || !issues_model.HasWorkInProgressPrefix(issue.Title) {
|
|
repoWatches, err := repo_model.GetRepoWatchersIDs(ctx, issue.RepoID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
toNotify.AddMultiple(repoWatches...)
|
|
}
|
|
issueParticipants, err := issue.GetParticipantIDsByIssue(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
toNotify.AddMultiple(issueParticipants...)
|
|
|
|
// dont notify user who cause notification
|
|
delete(toNotify, notificationAuthorID)
|
|
// explicit unwatch on issue
|
|
issueUnWatches, err := issues_model.GetIssueWatchersIDs(ctx, issueID, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, id := range issueUnWatches {
|
|
toNotify.Remove(id)
|
|
}
|
|
// Remove users who have the notification author blocked.
|
|
blockedAuthorIDs, err := user_model.ListBlockedByUsersID(ctx, notificationAuthorID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, id := range blockedAuthorIDs {
|
|
toNotify.Remove(id)
|
|
}
|
|
}
|
|
|
|
err = issue.LoadRepo(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// notify
|
|
for userID := range toNotify {
|
|
issue.Repo.Units = nil
|
|
user, err := user_model.GetUserByID(ctx, userID)
|
|
if err != nil {
|
|
if user_model.IsErrUserNotExist(err) {
|
|
continue
|
|
}
|
|
|
|
return err
|
|
}
|
|
if issue.IsPull && !access_model.CheckRepoUnitUser(ctx, issue.Repo, user, unit.TypePullRequests) {
|
|
continue
|
|
}
|
|
if !issue.IsPull && !access_model.CheckRepoUnitUser(ctx, issue.Repo, user, unit.TypeIssues) {
|
|
continue
|
|
}
|
|
|
|
if notificationExists(notifications, issue.ID, userID) {
|
|
if err = updateIssueNotification(ctx, userID, issue.ID, commentID); err != nil {
|
|
return err
|
|
}
|
|
continue
|
|
}
|
|
if err = createIssueNotification(ctx, userID, issue, commentID); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// NotificationList contains a list of notifications
|
|
type NotificationList []*Notification
|
|
|
|
// LoadAttributes load Repo Issue User and Comment if not loaded
|
|
func (nl NotificationList) LoadAttributes(ctx context.Context) error {
|
|
if _, _, err := nl.LoadRepos(ctx); err != nil {
|
|
return err
|
|
}
|
|
if _, err := nl.LoadIssues(ctx); err != nil {
|
|
return err
|
|
}
|
|
if _, err := nl.LoadUsers(ctx); err != nil {
|
|
return err
|
|
}
|
|
if _, err := nl.LoadComments(ctx); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// getPendingRepoIDs returns all the repositoty ids which haven't been loaded
|
|
func (nl NotificationList) getPendingRepoIDs() []int64 {
|
|
return container.FilterSlice(nl, func(n *Notification) (int64, bool) {
|
|
return n.RepoID, n.Repository == nil
|
|
})
|
|
}
|
|
|
|
// LoadRepos loads repositories from database
|
|
func (nl NotificationList) LoadRepos(ctx context.Context) (repo_model.RepositoryList, []int, error) {
|
|
if len(nl) == 0 {
|
|
return repo_model.RepositoryList{}, []int{}, nil
|
|
}
|
|
|
|
repoIDs := nl.getPendingRepoIDs()
|
|
repos, err := db.GetByIDs(ctx, "id", repoIDs, &repo_model.Repository{})
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
failed := []int{}
|
|
|
|
reposList := make(repo_model.RepositoryList, 0, len(repoIDs))
|
|
for i, notification := range nl {
|
|
if notification.Repository == nil {
|
|
notification.Repository = repos[notification.RepoID]
|
|
}
|
|
if notification.Repository == nil {
|
|
log.Error("Notification[%d]: RepoID: %d not found", notification.ID, notification.RepoID)
|
|
failed = append(failed, i)
|
|
continue
|
|
}
|
|
var found bool
|
|
for _, r := range reposList {
|
|
if r.ID == notification.RepoID {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
reposList = append(reposList, notification.Repository)
|
|
}
|
|
}
|
|
return reposList, failed, nil
|
|
}
|
|
|
|
func (nl NotificationList) getPendingIssueIDs() []int64 {
|
|
ids := make(container.Set[int64], len(nl))
|
|
for _, notification := range nl {
|
|
if notification.Issue != nil {
|
|
continue
|
|
}
|
|
ids.Add(notification.IssueID)
|
|
}
|
|
return ids.Values()
|
|
}
|
|
|
|
// LoadIssues loads issues from database
|
|
func (nl NotificationList) LoadIssues(ctx context.Context) ([]int, error) {
|
|
if len(nl) == 0 {
|
|
return []int{}, nil
|
|
}
|
|
|
|
issueIDs := nl.getPendingIssueIDs()
|
|
issues, err := db.GetByIDs(ctx, "id", issueIDs, &issues_model.Issue{})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
failures := []int{}
|
|
|
|
for i, notification := range nl {
|
|
if notification.Issue == nil {
|
|
notification.Issue = issues[notification.IssueID]
|
|
if notification.Issue == nil {
|
|
if notification.IssueID != 0 {
|
|
log.Error("Notification[%d]: IssueID: %d Not Found", notification.ID, notification.IssueID)
|
|
failures = append(failures, i)
|
|
}
|
|
continue
|
|
}
|
|
notification.Issue.Repo = notification.Repository
|
|
}
|
|
}
|
|
return failures, nil
|
|
}
|
|
|
|
// Without returns the notification list without the failures
|
|
func (nl NotificationList) Without(failures []int) NotificationList {
|
|
if len(failures) == 0 {
|
|
return nl
|
|
}
|
|
remaining := make([]*Notification, 0, len(nl))
|
|
last := -1
|
|
var i int
|
|
for _, i = range failures {
|
|
remaining = append(remaining, nl[last+1:i]...)
|
|
last = i
|
|
}
|
|
if len(nl) > i {
|
|
remaining = append(remaining, nl[i+1:]...)
|
|
}
|
|
return remaining
|
|
}
|
|
|
|
func (nl NotificationList) getPendingCommentIDs() []int64 {
|
|
ids := make(container.Set[int64], len(nl))
|
|
for _, notification := range nl {
|
|
if notification.CommentID == 0 || notification.Comment != nil {
|
|
continue
|
|
}
|
|
ids.Add(notification.CommentID)
|
|
}
|
|
return ids.Values()
|
|
}
|
|
|
|
func (nl NotificationList) getUserIDs() []int64 {
|
|
ids := make(container.Set[int64], len(nl))
|
|
for _, notification := range nl {
|
|
if notification.UserID == 0 || notification.User != nil {
|
|
continue
|
|
}
|
|
ids.Add(notification.UserID)
|
|
}
|
|
return ids.Values()
|
|
}
|
|
|
|
// LoadUsers loads users from database
|
|
func (nl NotificationList) LoadUsers(ctx context.Context) ([]int, error) {
|
|
if len(nl) == 0 {
|
|
return []int{}, nil
|
|
}
|
|
|
|
userIDs := nl.getUserIDs()
|
|
users, err := db.GetByIDs(ctx, "id", userIDs, &user_model.User{})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
failures := []int{}
|
|
for i, notification := range nl {
|
|
if notification.UserID > 0 && notification.User == nil && users[notification.UserID] != nil {
|
|
notification.User = users[notification.UserID]
|
|
if notification.User == nil {
|
|
log.Error("Notification[%d]: UserID[%d] failed to load", notification.ID, notification.UserID)
|
|
failures = append(failures, i)
|
|
continue
|
|
}
|
|
}
|
|
}
|
|
return failures, nil
|
|
}
|
|
|
|
// LoadComments loads comments from database
|
|
func (nl NotificationList) LoadComments(ctx context.Context) ([]int, error) {
|
|
if len(nl) == 0 {
|
|
return []int{}, nil
|
|
}
|
|
|
|
commentIDs := nl.getPendingCommentIDs()
|
|
comments, err := db.GetByIDs(ctx, "id", commentIDs, &issues_model.Comment{})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
failures := []int{}
|
|
for i, notification := range nl {
|
|
if notification.CommentID > 0 && notification.Comment == nil && comments[notification.CommentID] != nil {
|
|
notification.Comment = comments[notification.CommentID]
|
|
if notification.Comment == nil {
|
|
log.Error("Notification[%d]: CommentID[%d] failed to load", notification.ID, notification.CommentID)
|
|
failures = append(failures, i)
|
|
continue
|
|
}
|
|
notification.Comment.Issue = notification.Issue
|
|
}
|
|
}
|
|
return failures, nil
|
|
}
|