From dea9ef670683dd3957c35c89e7c3e3ecadaeaa63 Mon Sep 17 00:00:00 2001 From: Mathieu Fenniak Date: Tue, 18 Nov 2025 18:34:25 +0100 Subject: [PATCH] fix: realign indexes on the 'action' table (#10040) Fixes #9963. This realigns all the indexes on the `action` table to best match their intended usages. New: - `IDX_action_created_unix (created_unix)` - Intended for usage in `DeleteOldActions`. - `IDX_action_repo_id_created_unix (repo_id, created_unix)` - Intended for usage when fetching action feeds for a repo and a team, with the same logic as that described below in `IDX_action_user_id_created_unix`. - `IDX_action_repo_id_op_type (repo_id, op_type)` - Intended for `DeleteIssueActions` when it searches for CreateIssue & CreatePullRequest actions for cleanup. Could be optimized further with a denormalization of the issue identifier into a field, but there's no current evidence that this is required. Replaced: - `IDX_action_c_u (created_unix, user_id)` - Replaced with `IDX_action_user_id_created_unix (user_id, created_unix)`. When action feeds are created w/ `ORDER BY created_unix DESC LIMIT 20`, an index beginning with `created_unix` will have to index scan until it can satisfy 20 records; the `user_id` portion of the index is effectively useless until two records appear at the same time. By inverting the order, the database will be able to identify the records created by a user and then pop the most recent 20 from the index order. - At the scale of database I have access to, the performance difference is unmeasurable. This change is supported by theoretical grounds and the findings of #9963, but no experimental evidence. Removed: - `IDX_action_user_id (user_id)` - Redundant with the new `IDX_action_user_id_created_unix`. - `IDX_action_r_u (repo_id, user_id)` - No clear consumer for this index. Retained with no modification: - `IDX_action_comment_id (comment_id)` - Used in `DeleteIssueActions`. - `IDX_action_au_r_c_u (act_user_id, repo_id, created_unix, user_id)` - Heat map generation. ## Checklist The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org). ### Tests - I added test coverage for Go changes... - [ ] in their respective `*_test.go` for unit tests. - [ ] in the `tests/integration` directory if it involves interactions with a live Forgejo server. - I added test coverage for JavaScript changes... - [ ] in `web_src/js/*.test.js` if it can be unit tested. - [ ] in `tests/e2e/*.test.e2e.js` if it requires interactions with a live Forgejo server (see also the [developer guide for JavaScript testing](https://codeberg.org/forgejo/forgejo/src/branch/forgejo/tests/e2e/README.md#end-to-end-tests)). ### Documentation - [ ] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change. - [x] I did not document these changes and I do not expect someone else to do it. ### Release notes - [ ] I do not want this change to show in the release notes. - [x] I want the title to show in the release notes with a link to this pull request. - [ ] I want the content of the `release-notes/.md` to be be used for the release notes instead of the title. Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/10040 Reviewed-by: Gusted Co-authored-by: Mathieu Fenniak Co-committed-by: Mathieu Fenniak --- models/activities/action.go | 32 +++++--- models/forgejo_migrations/index_utils.go | 52 +++++++++++++ models/forgejo_migrations/index_utils_test.go | 62 ++++++++++++++++ .../v14b_action-reindexing.go | 73 +++++++++++++++++++ 4 files changed, 209 insertions(+), 10 deletions(-) create mode 100644 models/forgejo_migrations/index_utils.go create mode 100644 models/forgejo_migrations/index_utils_test.go create mode 100644 models/forgejo_migrations/v14b_action-reindexing.go diff --git a/models/activities/action.go b/models/activities/action.go index 7ac050d79f..8fd7709e81 100644 --- a/models/activities/action.go +++ b/models/activities/action.go @@ -145,37 +145,49 @@ func (at ActionType) InActions(actions ...string) bool { // used in template render. type Action struct { ID int64 `xorm:"pk autoincr"` - UserID int64 `xorm:"INDEX"` // Receiver user id. + UserID int64 // Receiver user id. OpType ActionType ActUserID int64 // Action user id. ActUser *user_model.User `xorm:"-"` RepoID int64 Repo *repo_model.Repository `xorm:"-"` - CommentID int64 `xorm:"INDEX"` + CommentID int64 `xorm:"INDEX"` // indexed to support `DeleteIssueActions` Comment *issues_model.Comment `xorm:"-"` Issue *issues_model.Issue `xorm:"-"` // get the issue id from content RefName string IsPrivate bool `xorm:"NOT NULL DEFAULT false"` Content string `xorm:"TEXT"` - CreatedUnix timeutil.TimeStamp `xorm:"created"` + CreatedUnix timeutil.TimeStamp `xorm:"created INDEX"` // indexed to support `DeleteOldActions` } func init() { db.RegisterModel(new(Action)) } -// TableIndices implements xorm's TableIndices interface +// TableIndices implements xorm's TableIndices interface. It is used here to ensure indexes with specified column order +// are created, which can't be created through xorm tags on the struct. func (a *Action) TableIndices() []*schemas.Index { - repoIndex := schemas.NewIndex("r_u", schemas.IndexType) - repoIndex.AddColumn("repo_id", "user_id") - + // Index to support getUserHeatmapData, which searches for data that is visible-to (user_id) and performed-by + // (act_user_id) a user, but only includes visible repos (repo_id). actUserIndex := schemas.NewIndex("au_r_c_u", schemas.IndexType) actUserIndex.AddColumn("act_user_id", "repo_id", "created_unix", "user_id") - cudIndex := schemas.NewIndex("c_u", schemas.IndexType) - cudIndex.AddColumn("created_unix", "user_id") + // GetFeeds is a common access point to Action and requires that all action feeds be queried based upon one of + // user_id (opts.RequestedUser), repo_id (opts.RequestedTeam... kinda), and/or repo_id (opts.RequestedRepo), and + // then the results are ordered by created_unix and paginated. The most efficient indexes to support those queries + // are: + requestedUser := schemas.NewIndex("user_id_created_unix", schemas.IndexType) + requestedUser.AddColumn("user_id", "created_unix") + requestedRepo := schemas.NewIndex("repo_id_created_unix", schemas.IndexType) + requestedRepo.AddColumn("repo_id", "created_unix") - indices := []*schemas.Index{actUserIndex, repoIndex, cudIndex} + // To support `DeleteIssueActions` search for createissue / createpullrequest actions; this isn't a great search + // because `DeleteIssueActions` searches by `content` as well, but it should be sufficient performance-wise for + // infrequent deleting of issues. + repoOpType := schemas.NewIndex("repo_id_op_type", schemas.IndexType) + repoOpType.AddColumn("repo_id", "op_type") + + indices := []*schemas.Index{actUserIndex, requestedUser, requestedRepo, repoOpType} return indices } diff --git a/models/forgejo_migrations/index_utils.go b/models/forgejo_migrations/index_utils.go new file mode 100644 index 0000000000..dda08e4018 --- /dev/null +++ b/models/forgejo_migrations/index_utils.go @@ -0,0 +1,52 @@ +// Copyright 2025 The Forgejo Authors. All rights reserved. +// SPDX-License-Identifier: GPL-3.0-or-later + +package forgejo_migrations + +import ( + "fmt" + "strings" + + "forgejo.org/modules/setting" + + "xorm.io/xorm" +) + +func dropIndexIfExists(x *xorm.Engine, tableName, indexName string) error { + switch { + case setting.Database.Type.IsSQLite3(), setting.Database.Type.IsPostgreSQL(): + if _, err := x.Exec(fmt.Sprintf("DROP INDEX IF EXISTS %s", x.Quote(indexName))); err != nil { + return err + } + + case setting.Database.Type.IsMySQL(): + exists, err := indexExists(x, tableName, indexName) + if err != nil { + return err + } + + if exists { + if _, err := x.Exec(fmt.Sprintf("DROP INDEX %s ON %s", x.Quote(indexName), x.Quote(tableName))); err != nil { + return err + } + } + default: + return fmt.Errorf("unsupported db dialect type %v", x.Dialect().URI().DBType) + } + + return nil +} + +func indexExists(x *xorm.Engine, tableName, indexName string) (bool, error) { + switch { + case setting.Database.Type.IsSQLite3(): + return x.SQL("SELECT name FROM sqlite_master WHERE type = 'index' and name = ?", indexName).Exist() + case setting.Database.Type.IsPostgreSQL(): + return x.SQL("SELECT indexname FROM pg_indexes WHERE schemaname = ? AND tablename = ? AND indexname = ?", setting.Database.Schema, tableName, indexName).Exist() + case setting.Database.Type.IsMySQL(): + databaseName := strings.SplitN(setting.Database.Name, "?", 2)[0] + return x.SQL("SELECT `INDEX_NAME` FROM `INFORMATION_SCHEMA`.`STATISTICS` WHERE `TABLE_SCHEMA` = ? AND `TABLE_NAME` = ? AND `INDEX_NAME` = ?", databaseName, tableName, indexName).Exist() + } + + return false, fmt.Errorf("unsupported db dialect type %v", x.Dialect().URI().DBType) +} diff --git a/models/forgejo_migrations/index_utils_test.go b/models/forgejo_migrations/index_utils_test.go new file mode 100644 index 0000000000..a7ae3700e5 --- /dev/null +++ b/models/forgejo_migrations/index_utils_test.go @@ -0,0 +1,62 @@ +// Copyright 2025 The Forgejo Authors. All rights reserved. +// SPDX-License-Identifier: GPL-3.0-or-later + +package forgejo_migrations + +import ( + "testing" + + migration_tests "forgejo.org/models/gitea_migrations/test" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDropIndexIfExists(t *testing.T) { + type Table struct { + ID int64 `xorm:"pk"` + DoerID int64 `xorm:"INDEX INDEX(s)"` + OwnerID int64 `xorm:"INDEX"` + RepoID int64 `xorm:"INDEX(s)"` + } + + x, deferable := migration_tests.PrepareTestEnv(t, 0, new(Table)) + x.ShowSQL(true) + defer deferable() + if x == nil || t.Failed() { + return + } + + exists, err := indexExists(x, "table", "IDX_table_doer_id") + require.NoError(t, err) + assert.True(t, exists) + + exists, err = indexExists(x, "table", "IDX_table_owner_id") + require.NoError(t, err) + assert.True(t, exists) + + exists, err = indexExists(x, "table", "IDX_table_repo_id") + require.NoError(t, err) + assert.False(t, exists) + + exists, err = indexExists(x, "table", "IDX_table_s") + require.NoError(t, err) + assert.True(t, exists) + + require.NoError(t, dropIndexIfExists(x, "table", "IDX_table_repo_id")) + + require.NoError(t, dropIndexIfExists(x, "table", "IDX_table_doer_id")) + exists, err = indexExists(x, "table", "IDX_table_doer_id") + require.NoError(t, err) + assert.False(t, exists) + + require.NoError(t, dropIndexIfExists(x, "table", "IDX_table_s")) + exists, err = indexExists(x, "table", "IDX_table_s") + require.NoError(t, err) + assert.False(t, exists) + + require.NoError(t, dropIndexIfExists(x, "table", "IDX_table_owner_id")) + exists, err = indexExists(x, "table", "IDX_table_owner_id") + require.NoError(t, err) + assert.False(t, exists) +} diff --git a/models/forgejo_migrations/v14b_action-reindexing.go b/models/forgejo_migrations/v14b_action-reindexing.go new file mode 100644 index 0000000000..6b5608a5d5 --- /dev/null +++ b/models/forgejo_migrations/v14b_action-reindexing.go @@ -0,0 +1,73 @@ +// Copyright 2025 The Forgejo Authors. All rights reserved. +// SPDX-License-Identifier: GPL-3.0-or-later + +package forgejo_migrations + +import ( + "xorm.io/xorm" + "xorm.io/xorm/schemas" +) + +func init() { + registerMigration(&Migration{ + Description: "rework indexes on table action", + Upgrade: reworkActionIndexes, + }) +} + +type v14bAction struct { + ID int64 `xorm:"pk autoincr"` + UserID int64 // Receiver user id. + OpType int + ActUserID int64 // Action user id. + RepoID int64 + CommentID int64 `xorm:"INDEX"` // indexed to support `DeleteIssueActions` + CreatedUnix int64 `xorm:"created INDEX"` // indexed to support `DeleteOldActions` +} + +// TableName sets the name of this table +func (a *v14bAction) TableName() string { + return "action" +} + +// TableIndices implements xorm's TableIndices interface. It is used here to ensure indexes with specified column order +// are created, which can't be created through xorm tags on the struct. +func (a *v14bAction) TableIndices() []*schemas.Index { + // Index to support getUserHeatmapData, which searches for data that is visible-to (user_id) and performed-by + // (act_user_id) a user, but only includes visible repos (repo_id). + actUserIndex := schemas.NewIndex("au_r_c_u", schemas.IndexType) + actUserIndex.AddColumn("act_user_id", "repo_id", "created_unix", "user_id") + + // GetFeeds is a common access point to Action and requires that all action feeds be queried based upon one of + // user_id (opts.RequestedUser), repo_id (opts.RequestedTeam... kinda), and/or repo_id (opts.RequestedRepo), and + // then the results are ordered by created_unix and paginated. The most efficient indexes to support those queries + // are: + requestedUser := schemas.NewIndex("user_id_created_unix", schemas.IndexType) + requestedUser.AddColumn("user_id", "created_unix") + requestedRepo := schemas.NewIndex("repo_id_created_unix", schemas.IndexType) + requestedRepo.AddColumn("repo_id", "created_unix") + + // To support `DeleteIssueActions` search for createissue / createpullrequest actions; this isn't a great search + // because `DeleteIssueActions` searches by `content` as well, but it should be sufficient performance-wise for + // infrequent deleting of issues. + repoOpType := schemas.NewIndex("repo_id_op_type", schemas.IndexType) + repoOpType.AddColumn("repo_id", "op_type") + + indices := []*schemas.Index{actUserIndex, requestedUser, requestedRepo, repoOpType} + + return indices +} + +func reworkActionIndexes(x *xorm.Engine) error { + if err := dropIndexIfExists(x, "action", "IDX_action_c_u"); err != nil { + return err + } + if err := dropIndexIfExists(x, "action", "IDX_action_r_u"); err != nil { + return err + } + if err := dropIndexIfExists(x, "action", "IDX_action_user_id"); err != nil { + return err + } + + return x.Sync(new(v14bAction)) +}