diff --git a/models/forgejo_migrations/v15c_add_mirror_remoteaddressauth.go b/models/forgejo_migrations/v15c_add_mirror_remoteaddressauth.go new file mode 100644 index 0000000000..b2f30235ad --- /dev/null +++ b/models/forgejo_migrations/v15c_add_mirror_remoteaddressauth.go @@ -0,0 +1,30 @@ +// Copyright 2026 The Forgejo Authors. All rights reserved. +// SPDX-License-Identifier: GPL-3.0-or-later + +package forgejo_migrations + +import ( + "xorm.io/xorm" +) + +func init() { + registerMigration(&Migration{ + Description: "replace remote_address with encrypted_remote_address in table mirror", + Upgrade: addMirrorRemoteAddressAuth, + }) +} + +func addMirrorRemoteAddressAuth(x *xorm.Engine) error { + type Mirror struct { + EncryptedRemoteAddress []byte `xorm:"BLOB NULL"` + } + if _, err := x.SyncWithOptions(xorm.SyncOptions{IgnoreDropIndices: true}, new(Mirror)); err != nil { + return err + } + // No data migration is necessary or desired. `remote_address` contains sanitized URLs which don't have + // credentials, so they can't be migrated to `encrypted_remote_address`. Instead, as this data is accessed, + // `DecryptOrRecoverRemoteAddress` will recover the fully credentialed contents of the remote address from the git + // repo's `origin` remote address. + _, err := x.Exec("ALTER TABLE `mirror` DROP COLUMN `remote_address`") + return err +} diff --git a/models/repo/mirror.go b/models/repo/mirror.go index 1fe9afd8e9..c64f9dc734 100644 --- a/models/repo/mirror.go +++ b/models/repo/mirror.go @@ -6,10 +6,14 @@ package repo import ( "context" + "errors" + "net/url" "time" "forgejo.org/models/db" + "forgejo.org/modules/keying" "forgejo.org/modules/log" + "forgejo.org/modules/optional" "forgejo.org/modules/timeutil" "forgejo.org/modules/util" ) @@ -31,7 +35,9 @@ type Mirror struct { LFS bool `xorm:"lfs_enabled NOT NULL DEFAULT false"` LFSEndpoint string `xorm:"lfs_endpoint TEXT"` - RemoteAddress string `xorm:"VARCHAR(2048)"` + // Encrypted remote address w/ credentials; can be NULL if a mirror has not performed a sync since this field was + // introduced, in which case the remote address exists only in the repo's configured git remote on disk. + EncryptedRemoteAddress []byte `xorm:"BLOB NULL"` } func init() { @@ -73,6 +79,71 @@ func (m *Mirror) ScheduleNextUpdate() { } } +// InsertMirror inserts a mirror to database. RemoteAddress must be provided so that it can be encrypted and stored +// during the insert process. +func (m *Mirror) InsertWithAddress(ctx context.Context, addr string) error { + return db.WithTx(ctx, func(ctx context.Context) error { + if _, err := db.GetEngine(ctx).Insert(m); err != nil { + return err + } + return m.UpdateRemoteAddress(ctx, addr) + }) +} + +// Stores a credential-free version of the address in `RemoteAddress`, encrypts the original into `RemoteAddressAuth`, +// and stores both in the database. The ID of the mirror must be known, so this must be done after the mirror is +// inserted. +func (m *Mirror) UpdateRemoteAddress(ctx context.Context, addr string) error { + if m.ID == 0 { + return errors.New("must persist mirror to database before using UpdateRemoteAddress") + } + + m.EncryptedRemoteAddress = keying.PullMirror.Encrypt( + []byte(addr), + keying.ColumnAndID("remote_address_auth", m.ID), + ) + _, err := db.GetEngine(ctx).ID(m.ID).Cols("encrypted_remote_address").Update(m) + return err +} + +// Retrieves the encrypted remote address and decrypts it. Note that this field is expected to be absent for mirrors +// created before the introduction of EncryptedRemoteAddress, in which case credentials are not known to Forgejo +// directly (but may be on-disk in the repository's config file) and None will be returned. +func (m *Mirror) DecryptRemoteAddress() (optional.Option[string], error) { + if m.EncryptedRemoteAddress == nil { + return optional.None[string](), nil + } + + contents, err := keying.PullMirror.Decrypt(m.EncryptedRemoteAddress, keying.ColumnAndID("remote_address_auth", m.ID)) + if err != nil { + return optional.None[string](), err + } + return optional.Some(string(contents)), nil +} + +// Retrieves the remote address but sanitizes it of sensitive credentials. May be absent for mirrors created before the +// introduction of EncryptedRemoteAddress. +func (m *Mirror) SanitizedRemoteAddress() (optional.Option[string], error) { + maybeAddr, err := m.DecryptRemoteAddress() + if err != nil { + return optional.None[string](), err + } else if has, addr := maybeAddr.Get(); has { + parsedURL, err := url.Parse(addr) + if err != nil { + return optional.None[string](), err + } + + // Remove the password if present. Retain the username for consistency with `AddAuthCredentialHelperForRemote` + // which retains the username for the `git clone` command line, which ends up as the remote URL in the mirror's + // git config. + if parsedURL.User != nil { + parsedURL.User = url.User(parsedURL.User.Username()) + } + return optional.Some(parsedURL.String()), nil + } + return optional.None[string](), nil +} + // GetMirrorByRepoID returns mirror information of a repository. func GetMirrorByRepoID(ctx context.Context, repoID int64) (*Mirror, error) { m := &Mirror{RepoID: repoID} @@ -115,9 +186,3 @@ func MirrorsIterate(ctx context.Context, limit int, f func(idx int, bean any) er } return sess.Iterate(new(Mirror), f) } - -// InsertMirror inserts a mirror to database -func InsertMirror(ctx context.Context, mirror *Mirror) error { - _, err := db.GetEngine(ctx).Insert(mirror) - return err -} diff --git a/modules/git/command.go b/modules/git/command.go index bf1d624dbf..4ab081418b 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io" + "net/url" "os" "os/exec" "runtime/trace" @@ -446,6 +447,54 @@ func (c *Command) RunStdBytes(opts *RunOpts) (stdout, stderr []byte, runErr RunS return stdoutBuf.Bytes(), stderr, nil } +// If `remoteURL` is a URL with a password in it, add parameters to the git command that will read that password from a +// credential store file, and return the URL that should be used in the command instead of the original, and a cleanup +// function to call to remove the credential file. If `remoteURL` doesn't have a password, then it is returned as-is. +// This function must be invoked on the the git command before the git sub-command -- eg. before the `clone` or `fetch` +// parameter is added to the command's args. +func (c *Command) AddAuthCredentialHelperForRemote(remoteURL string) (commandURL string, cleanup func(), err error) { + parsedFromURL, _ := url.Parse(remoteURL) + + // If the clone URL has credentials, build a credential file for usage by git-credential-store + // to prevent credential leak in the process list. + // https://git-scm.com/docs/git-credential-store#_storage_format + // credential.helper adjustment must be set before the git subcommand + if strings.Contains(remoteURL, "://") && strings.Contains(remoteURL, "@") && parsedFromURL != nil { + credentialsFile, err := os.CreateTemp("", "forgejo-clone-credentials-") + if err != nil { + return "", nil, err + } + credentialsPath := credentialsFile.Name() + + cleanup := func() { + _ = credentialsFile.Close() + if err := util.Remove(credentialsPath); err != nil { + log.Warn("Unable to remove temporary file %q: %v", credentialsPath, err) + } + } + _, err = credentialsFile.Write([]byte(parsedFromURL.String())) + if err != nil { + cleanup() + return "", nil, err + } + err = credentialsFile.Close() + if err != nil { + cleanup() + return "", nil, err + } + + c.AddArguments("-c").AddDynamicArguments("credential.helper=store --file=" + credentialsPath) + + // remove the password from the URL argument + parsedFromURL.User = url.User(parsedFromURL.User.Username()) + commandURL = parsedFromURL.String() + + return commandURL, cleanup, nil + } + + return remoteURL, func() {}, nil +} + // AllowLFSFiltersArgs return globalCommandArgs with lfs filter, it should only be used for tests func AllowLFSFiltersArgs() TrustedCmdArgs { // Now here we should explicitly allow lfs filters to run diff --git a/modules/git/repo.go b/modules/git/repo.go index 6bd03f8e3c..8f9b95f1f2 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -18,7 +18,6 @@ import ( "strings" "time" - "forgejo.org/modules/log" "forgejo.org/modules/proxy" "forgejo.org/modules/setting" "forgejo.org/modules/util" @@ -141,44 +140,13 @@ func CloneWithArgs(ctx context.Context, args TrustedCmdArgs, from, to string, op envs = proxy.EnvWithProxy(parsedFromURL) } - fromURL := from - sanitizedFrom := from + sanitizedFrom := util.SanitizeCredentialURLs(from) - // If the clone URL has credentials, build a credential file for usage by git-credential-store - // to prevent credential leak in the process list. - // https://git-scm.com/docs/git-credential-store#_storage_format - // credential.helper adjustment must be set before the git subcommand - if strings.Contains(from, "://") && strings.Contains(from, "@") { - sanitizedFrom = util.SanitizeCredentialURLs(from) - if parsedFromURL != nil { - credentialsFile, err := os.CreateTemp("", "forgejo-clone-credentials-") - if err != nil { - return err - } - credentialsPath := credentialsFile.Name() - - defer func() { - _ = credentialsFile.Close() - if err := util.Remove(credentialsPath); err != nil { - log.Warn("Unable to remove temporary file %q: %v", credentialsPath, err) - } - }() - _, err = credentialsFile.Write([]byte(parsedFromURL.String())) - if err != nil { - return err - } - err = credentialsFile.Close() - if err != nil { - return err - } - - cmd.AddArguments("-c").AddDynamicArguments("credential.helper=store --file=" + credentialsPath) - - // remove the password from the URL argument - parsedFromURL.User = url.User(parsedFromURL.User.Username()) - fromURL = parsedFromURL.String() - } + fromURL, cleanup, err := cmd.AddAuthCredentialHelperForRemote(from) + if err != nil { + return err } + defer cleanup() cmd.AddArguments("clone") diff --git a/modules/keying/keying.go b/modules/keying/keying.go index 751fd77521..14fbaaca98 100644 --- a/modules/keying/keying.go +++ b/modules/keying/keying.go @@ -41,6 +41,8 @@ var ( MigrateTask = deriveKey("migrate_repo_task") // Used for the `webhook` table. Webhook = deriveKey("webhook") + // Used for the `mirror` table. + PullMirror = deriveKey("pullmirror") ) var ( diff --git a/modules/templates/util_misc.go b/modules/templates/util_misc.go index 60f918be47..8e1ef71f5d 100644 --- a/modules/templates/util_misc.go +++ b/modules/templates/util_misc.go @@ -17,12 +17,11 @@ import ( asymkey_model "forgejo.org/models/asymkey" repo_model "forgejo.org/models/repo" user_model "forgejo.org/models/user" - "forgejo.org/modules/git" - giturl "forgejo.org/modules/git/url" "forgejo.org/modules/json" "forgejo.org/modules/log" "forgejo.org/modules/repository" "forgejo.org/modules/svg" + mirror_service "forgejo.org/services/mirror" "github.com/editorconfig/editorconfig-core-go/v2" ) @@ -164,17 +163,11 @@ type remoteAddress struct { Password string } -func mirrorRemoteAddress(ctx context.Context, m *repo_model.Repository, remoteName string) remoteAddress { +func mirrorRemoteAddress(ctx context.Context, mirror *repo_model.Mirror) remoteAddress { ret := remoteAddress{} - remoteURL, err := git.GetRemoteAddress(ctx, m.RepoPath(), remoteName) + u, err := mirror_service.DecryptOrRecoverRemoteAddress(ctx, mirror) if err != nil { - log.Error("GetRemoteURL %v", err) - return ret - } - - u, err := giturl.Parse(remoteURL) - if err != nil { - log.Error("giturl.Parse %v", err) + log.Error("DecryptOrRecoverRemoteAddress %v", err) return ret } diff --git a/routers/web/repo/setting/setting.go b/routers/web/repo/setting/setting.go index f7d0348634..b001cf961e 100644 --- a/routers/web/repo/setting/setting.go +++ b/routers/web/repo/setting/setting.go @@ -462,12 +462,12 @@ func SettingsPost(ctx *context.Context) { return } - u, err := git.GetRemoteURL(ctx, ctx.Repo.Repository.RepoPath(), pullMirror.GetRemoteName()) + u, err := mirror_service.DecryptOrRecoverRemoteAddress(ctx, pullMirror) if err != nil { - ctx.Data["Err_MirrorAddress"] = true - handleSettingRemoteAddrError(ctx, err, form) + ctx.ServerError("DecryptOrRecoverRemoteAddress", err) return } + if u.User != nil && form.MirrorPassword == "" && form.MirrorUsername == u.User.Username() { form.MirrorPassword, _ = u.User.Password() } @@ -482,17 +482,25 @@ func SettingsPost(ctx *context.Context) { return } - if err := mirror_service.UpdateAddress(ctx, pullMirror, address); err != nil { - ctx.ServerError("UpdateAddress", err) - return - } - remoteAddress, err := util.SanitizeURL(address) - if err != nil { + if err := pullMirror.UpdateRemoteAddress(ctx, address); err != nil { ctx.Data["Err_MirrorAddress"] = true handleSettingRemoteAddrError(ctx, err, form) return } - pullMirror.RemoteAddress = remoteAddress + + // Update the unencrypted address stored in the git config, so that future `git fetch` will access the right + // address. pullMirror.RemoteAddress is the sanitized no-creds version from UpdateRemoteAddress. + if maybeSanitizedURL, err := pullMirror.SanitizedRemoteAddress(); err != nil { + ctx.ServerError("SanitizedRemoteAddress", err) + return + } else if has, sanitizedURL := maybeSanitizedURL.Get(); !has { + // SanitizedRemoteAddress must be present after we just stored it + ctx.ServerError("SanitizedRemoteAddress", err) + return + } else if err := mirror_service.UpdateAddress(ctx, pullMirror, sanitizedURL); err != nil { + ctx.ServerError("UpdateAddress", err) + return + } form.LFS = form.LFS && setting.LFS.StartServer diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index d6e1ff6c51..c5adc6105f 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -31,55 +31,27 @@ const gitShortEmptySha = "0000000" // UpdateAddress writes new address to Git repository and database func UpdateAddress(ctx context.Context, m *repo_model.Mirror, addr string) error { - u, err := giturl.Parse(addr) - if err != nil { - return fmt.Errorf("invalid addr: %v", err) - } - remoteName := m.GetRemoteName() repoPath := m.GetRepository(ctx).RepoPath() - // Remove old remote - _, _, err = git.NewCommand(ctx, "remote", "rm").AddDynamicArguments(remoteName).RunStdString(&git.RunOpts{Dir: repoPath}) - if err != nil && !git.IsRemoteNotExistError(err) { - return err - } - - cmd := git.NewCommand(ctx, "remote", "add").AddDynamicArguments(remoteName).AddArguments("--mirror=fetch").AddDynamicArguments(addr) - if strings.Contains(addr, "://") && strings.Contains(addr, "@") { - cmd.SetDescription(fmt.Sprintf("remote add %s --mirror=fetch %s [repo_path: %s]", remoteName, util.SanitizeCredentialURLs(addr), repoPath)) - } else { - cmd.SetDescription(fmt.Sprintf("remote add %s --mirror=fetch %s [repo_path: %s]", remoteName, addr, repoPath)) - } - _, _, err = cmd.RunStdString(&git.RunOpts{Dir: repoPath}) - if err != nil && !git.IsRemoteNotExistError(err) { + _, _, err := git.NewCommand(ctx, "remote", "set-url"). + AddDynamicArguments(remoteName, addr). + RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { return err } if m.Repo.HasWiki() { wikiPath := m.Repo.WikiPath() wikiRemotePath := repo_module.WikiRemoteURL(ctx, addr) - // Remove old remote of wiki - _, _, err = git.NewCommand(ctx, "remote", "rm").AddDynamicArguments(remoteName).RunStdString(&git.RunOpts{Dir: wikiPath}) - if err != nil && !git.IsRemoteNotExistError(err) { - return err - } - - cmd = git.NewCommand(ctx, "remote", "add").AddDynamicArguments(remoteName).AddArguments("--mirror=fetch").AddDynamicArguments(wikiRemotePath) - if strings.Contains(wikiRemotePath, "://") && strings.Contains(wikiRemotePath, "@") { - cmd.SetDescription(fmt.Sprintf("remote add %s --mirror=fetch %s [repo_path: %s]", remoteName, util.SanitizeCredentialURLs(wikiRemotePath), wikiPath)) - } else { - cmd.SetDescription(fmt.Sprintf("remote add %s --mirror=fetch %s [repo_path: %s]", remoteName, wikiRemotePath, wikiPath)) - } - _, _, err = cmd.RunStdString(&git.RunOpts{Dir: wikiPath}) - if err != nil && !git.IsRemoteNotExistError(err) { + _, _, err = git.NewCommand(ctx, "remote", "set-url"). + AddDynamicArguments(remoteName, wikiRemotePath). + RunStdString(&git.RunOpts{Dir: wikiPath}) + if err != nil { return err } } - // erase authentication before storing in database - u.User = nil - m.Repo.OriginalURL = u.String() - return repo_model.UpdateRepositoryCols(ctx, m.Repo, "original_url") + return nil } // mirrorSyncResult contains information of a updated reference. @@ -262,6 +234,46 @@ func checkRecoverableSyncError(stderrMessage string) bool { } } +// Decrypt RemoteAddressAuth from the mirror. If absent on the mirror database table, fallback to the older method where +// credentials are stored in the git config file as the remote's address, encrypt those credentials and store them in +// the database, and wipe them from the git config file so that they're only stored in the one encrypted location in DB. +func DecryptOrRecoverRemoteAddress(ctx context.Context, m *repo_model.Mirror) (*giturl.GitURL, error) { + decryptedRemoteURL, err := m.DecryptRemoteAddress() + if err != nil { + return nil, fmt.Errorf("failed to decrypt remote address: %w", err) + } + + if has, url := decryptedRemoteURL.Get(); has { + remoteURL, err := giturl.Parse(url) + if err != nil { + return nil, fmt.Errorf("failed to parse decrypted remote address: %w", err) + } + return remoteURL, nil + } + + // fallback to reading remote URL from the git config file for repos that predate DecryptRemoteAddress + repoPath := m.GetRepository(ctx).RepoPath() + remoteURL, err := git.GetRemoteURL(ctx, repoPath, m.GetRemoteName()) + if err != nil { + return nil, fmt.Errorf("GetRemoteAddress error: %w", err) + } + + // Store the full address in the database + if err := m.UpdateRemoteAddress(ctx, remoteURL.URL.String()); err != nil { + return nil, fmt.Errorf("UpdateRemoteAddress error: %w", err) + } + + // Update the git config file to just contain the sanitized address + if maybeSanitizedURL, err := m.SanitizedRemoteAddress(); err != nil { + return nil, fmt.Errorf("SanitizedRemoteAddress error: %w", err) + } else if has, sanitizedURL := maybeSanitizedURL.Get(); !has { + return nil, fmt.Errorf("SanitizedRemoteAddress must be present after we just stored it, but had error: %w", err) + } else if err := UpdateAddress(ctx, m, sanitizedURL); err != nil { + return nil, fmt.Errorf("UpdateAddress error: %w", err) + } + return remoteURL, nil +} + // runSync returns true if sync finished without error. func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bool) { repoPath := m.Repo.RepoPath() @@ -270,19 +282,29 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo log.Trace("SyncMirrors [repo: %-v]: running git remote update...", m.Repo) + remoteURL, err := DecryptOrRecoverRemoteAddress(ctx, m) + if err != nil { + log.Error("SyncMirrors [repo: %-v]: failed to get remote address: %v", m.Repo, err) + return nil, false + } + + cmd := git.NewCommand(ctx) + + // Setup credential helper to authenticate the fetch; needs to occur before the `fetch` arg. + _, credCleanup, err := cmd.AddAuthCredentialHelperForRemote(remoteURL.URL.String()) + if err != nil { + log.Error("SyncMirrors [repo: %-v]: AddAuthCredentialHelperForRemote Error %v", m.Repo, err) + return nil, false + } + defer credCleanup() + // use fetch but not remote update because git fetch support --tags but remote update doesn't - cmd := git.NewCommand(ctx, "fetch") + cmd.AddArguments("fetch") if m.EnablePrune { cmd.AddArguments("--prune") } cmd.AddArguments("--tags").AddDynamicArguments(m.GetRemoteName()) - remoteURL, remoteErr := git.GetRemoteURL(ctx, repoPath, m.GetRemoteName()) - if remoteErr != nil { - log.Error("SyncMirrors [repo: %-v]: GetRemoteAddress Error %v", m.Repo, remoteErr) - return nil, false - } - envs := proxy.EnvWithProxy(remoteURL.URL) stdoutBuilder := strings.Builder{} diff --git a/services/repository/migrate.go b/services/repository/migrate.go index 80f5d68231..46b48d02d3 100644 --- a/services/repository/migrate.go +++ b/services/repository/migrate.go @@ -182,17 +182,12 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, defer committer.Close() if opts.Mirror { - remoteAddress, err := util.SanitizeURL(opts.CloneAddr) - if err != nil { - return repo, err - } mirrorModel := repo_model.Mirror{ RepoID: repo.ID, Interval: setting.Mirror.DefaultInterval, EnablePrune: true, NextUpdateUnix: timeutil.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval), LFS: opts.LFS, - RemoteAddress: remoteAddress, } if opts.LFS { mirrorModel.LFSEndpoint = opts.LFSEndpoint @@ -217,7 +212,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, } } - if err = repo_model.InsertMirror(ctx, &mirrorModel); err != nil { + if err = mirrorModel.InsertWithAddress(ctx, opts.CloneAddr); err != nil { return repo, fmt.Errorf("InsertOne: %w", err) } diff --git a/templates/repo/header.tmpl b/templates/repo/header.tmpl index 513758a149..44254aae82 100644 --- a/templates/repo/header.tmpl +++ b/templates/repo/header.tmpl @@ -77,9 +77,10 @@ {{end}} {{if $.PullMirror}} + {{$address := MirrorRemoteAddress $.Context $.PullMirror}}
{{ctx.Locale.Tr "repo.mirror_from"}} - {{$.PullMirror.RemoteAddress}} + {{$address.Address}} {{if $.PullMirror.UpdatedUnix}}{{ctx.Locale.Tr "repo.mirror_sync"}} {{DateUtils.TimeSince $.PullMirror.UpdatedUnix}}{{end}}
{{end}} diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index fa25f4630a..137be0f334 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -148,9 +148,10 @@ {{else if $isWorkingPullMirror}} + {{$address := MirrorRemoteAddress $.Context .PullMirror}} - {{.PullMirror.RemoteAddress}} + {{$address.Address}} {{ctx.Locale.Tr "repo.settings.mirror_settings.direction.pull"}} {{DateUtils.FullTime .PullMirror.UpdatedUnix}} @@ -176,7 +177,6 @@ - {{$address := MirrorRemoteAddress $.Context .Repository .PullMirror.GetRemoteName}}
diff --git a/tests/integration/mirror_pull_test.go b/tests/integration/mirror_pull_test.go index 03d4bdcf92..a0de586aaa 100644 --- a/tests/integration/mirror_pull_test.go +++ b/tests/integration/mirror_pull_test.go @@ -5,9 +5,16 @@ package integration import ( + "fmt" "net/http" + "net/url" + "os" + "path" + "strings" "testing" + "time" + "forgejo.org/models/auth" "forgejo.org/models/db" repo_model "forgejo.org/models/repo" "forgejo.org/models/unittest" @@ -15,94 +22,546 @@ import ( "forgejo.org/modules/git" "forgejo.org/modules/gitrepo" "forgejo.org/modules/migration" + "forgejo.org/modules/optional" + "forgejo.org/modules/process" + "forgejo.org/modules/setting" + "forgejo.org/modules/structs" app_context "forgejo.org/services/context" + "forgejo.org/services/forms" + "forgejo.org/services/migrations" mirror_service "forgejo.org/services/mirror" release_service "forgejo.org/services/release" repo_service "forgejo.org/services/repository" + files_service "forgejo.org/services/repository/files" "forgejo.org/tests" + "github.com/PuerkitoBio/goquery" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestMirrorPull(t *testing.T) { - defer tests.PrepareTestEnv(t)() + t.Run("Basic", func(t *testing.T) { + defer tests.PrepareTestEnv(t)() - user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) - repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) - repoPath := repo_model.RepoPath(user.Name, repo.Name) + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + repoPath := repo_model.RepoPath(user.Name, repo.Name) - opts := migration.MigrateOptions{ - RepoName: "test_mirror", - Description: "Test mirror", - Private: false, - Mirror: true, - CloneAddr: repoPath, - Wiki: true, - Releases: false, - } + opts := migration.MigrateOptions{ + RepoName: "test_mirror", + Description: "Test mirror", + Private: false, + Mirror: true, + CloneAddr: repoPath, + Wiki: true, + Releases: false, + } - mirrorRepo, err := repo_service.CreateRepositoryDirectly(db.DefaultContext, user, user, repo_service.CreateRepoOptions{ - Name: opts.RepoName, - Description: opts.Description, - IsPrivate: opts.Private, - IsMirror: opts.Mirror, - Status: repo_model.RepositoryBeingMigrated, + mirrorRepo, err := repo_service.CreateRepositoryDirectly(db.DefaultContext, user, user, repo_service.CreateRepoOptions{ + Name: opts.RepoName, + Description: opts.Description, + IsPrivate: opts.Private, + IsMirror: opts.Mirror, + Status: repo_model.RepositoryBeingMigrated, + }) + require.NoError(t, err) + assert.True(t, mirrorRepo.IsMirror, "expected pull-mirror repo to be marked as a mirror immediately after its creation") + + ctx := t.Context() + + mirror, err := repo_service.MigrateRepositoryGitData(ctx, user, mirrorRepo, opts, nil) + require.NoError(t, err) + + gitRepo, err := gitrepo.OpenRepository(git.DefaultContext, repo) + require.NoError(t, err) + defer gitRepo.Close() + + findOptions := repo_model.FindReleasesOptions{ + IncludeDrafts: true, + IncludeTags: true, + RepoID: mirror.ID, + } + initCount, err := db.Count[repo_model.Release](db.DefaultContext, findOptions) + require.NoError(t, err) + + require.NoError(t, release_service.CreateRelease(gitRepo, &repo_model.Release{ + RepoID: repo.ID, + Repo: repo, + PublisherID: user.ID, + Publisher: user, + TagName: "v0.2", + Target: "master", + Title: "v0.2 is released", + Note: "v0.2 is released", + IsDraft: false, + IsPrerelease: false, + IsTag: true, + }, "", []*release_service.AttachmentChange{})) + + _, err = repo_model.GetMirrorByRepoID(ctx, mirror.ID) + require.NoError(t, err) + + ok := mirror_service.SyncPullMirror(ctx, mirror.ID) + assert.True(t, ok) + + count, err := db.Count[repo_model.Release](db.DefaultContext, findOptions) + require.NoError(t, err) + assert.Equal(t, initCount+1, count) + + release, err := repo_model.GetRelease(db.DefaultContext, repo.ID, "v0.2") + require.NoError(t, err) + require.NoError(t, release_service.DeleteReleaseByID(ctx, repo, release, user, true)) + + ok = mirror_service.SyncPullMirror(ctx, mirror.ID) + assert.True(t, ok) + + count, err = db.Count[repo_model.Release](db.DefaultContext, findOptions) + require.NoError(t, err) + assert.Equal(t, initCount, count) }) - require.NoError(t, err) - assert.True(t, mirrorRepo.IsMirror, "expected pull-mirror repo to be marked as a mirror immediately after its creation") - ctx := t.Context() - - mirror, err := repo_service.MigrateRepositoryGitData(ctx, user, mirrorRepo, opts, nil) - require.NoError(t, err) - - gitRepo, err := gitrepo.OpenRepository(git.DefaultContext, repo) - require.NoError(t, err) - defer gitRepo.Close() - - findOptions := repo_model.FindReleasesOptions{ - IncludeDrafts: true, - IncludeTags: true, - RepoID: mirror.ID, + // How will we interact with the pull mirror during this test? + interactionMethod := []struct { + name string + useAPI bool + createPullMirror func(t *testing.T, sourceRepo *repo_model.Repository, authenticate bool) (repoName string) + verifyMirrorDetails func(t *testing.T, sourceRepo *repo_model.Repository, mirrorName string, authenticate bool) + triggerPullMirror func(t *testing.T, mirrorName string) + changePullMirrorCredentials func(t *testing.T, sourceRepo *repo_model.Repository, mirrorName string, authenticate bool) + changePullMirrorAddress func(t *testing.T, sourceRepo *repo_model.Repository, mirrorName string, authenticate bool) + }{ + { + name: "API", + useAPI: true, + createPullMirror: createPullMirrorViaAPI, + triggerPullMirror: triggerPullMirrorViaAPI, + verifyMirrorDetails: func(t *testing.T, sourceRepo *repo_model.Repository, mirrorName string, authenticate bool) { + // API provides no visibility into a repo's mirror settings right now + }, + }, + { + name: "Web", + useAPI: false, + createPullMirror: createPullMirrorViaWeb, + triggerPullMirror: triggerPullMirrorViaWeb, + verifyMirrorDetails: verifyPullMirrorViaWeb, + changePullMirrorCredentials: changePullMirrorCredentialsViaWeb, + changePullMirrorAddress: changePullMirrorCredentialsViaWeb, // one endpoint, so same as creds + }, } - initCount, err := db.Count[repo_model.Release](db.DefaultContext, findOptions) + + mirrorConfiguration := []struct { + name string + privateSource bool + }{ + { + name: "HTTP Without Auth", + }, + { + name: "HTTP With Auth", + privateSource: true, + }, + } + + // Not using MockVariableValue due to need to undo `migrations.Init()` + prev := setting.Migrations.AllowedDomains + setting.Migrations.AllowedDomains = "localhost" + migrations.Init() // reinitialize for changed allowList + defer func() { + setting.Migrations.AllowedDomains = prev + migrations.Init() // reinitialize for changed allowList + }() + + onApplicationRun(t, func(t *testing.T, u *url.URL) { + for _, im := range interactionMethod { + for _, mc := range mirrorConfiguration { + t.Run(fmt.Sprintf("%s/%s", im.name, mc.name), func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + + // Create the source repository that will be mirrored. + sourceRepo, sourceRepoSha, cleanupSource := tests.CreateDeclarativeRepoWithOptions(t, user2, + tests.DeclarativeRepoOptions{ + IsPrivate: optional.Some(mc.privateSource), + Files: optional.Some([]*files_service.ChangeRepoFile{ + { + Operation: "create", + TreePath: "docs.md", + ContentReader: strings.NewReader("hello, world"), + }, + }), + }, + ) + defer cleanupSource() + require.NotEmpty(t, sourceRepoSha) + + // Create pull mirror + mirror := im.createPullMirror(t, sourceRepo, mc.privateSource) + verifyPullMirrorContents(t, mirror, sourceRepoSha) + verifyPullMirrorConfig(t, mirror, sourceRepo, mc.privateSource) + im.verifyMirrorDetails(t, sourceRepo, mirror, mc.privateSource) + + // Push a change to the source and refresh the mirror + sourceRepoSha = changePullMirrorSource(t, sourceRepo, sourceRepoSha) + im.triggerPullMirror(t, mirror) + waitForPullMirror(t, mirror, sourceRepoSha) + + // Test changing the mirror's authentication method (if available) + if mc.privateSource && im.changePullMirrorCredentials != nil { + sourceRepoSha = changePullMirrorSource(t, sourceRepo, sourceRepoSha) + im.changePullMirrorCredentials(t, sourceRepo, mirror, mc.privateSource) + verifyPullMirrorConfig(t, mirror, sourceRepo, mc.privateSource) + im.verifyMirrorDetails(t, sourceRepo, mirror, mc.privateSource) + im.triggerPullMirror(t, mirror) + waitForPullMirror(t, mirror, sourceRepoSha) + } + + // Test changing the mirror's address (if available) + if im.changePullMirrorAddress != nil { + sourceRepo = renamePullMirrorSourceRepo(t, sourceRepo) + sourceRepoSha = changePullMirrorSource(t, sourceRepo, sourceRepoSha) + im.changePullMirrorAddress(t, sourceRepo, mirror, mc.privateSource) + verifyPullMirrorConfig(t, mirror, sourceRepo, mc.privateSource) + im.verifyMirrorDetails(t, sourceRepo, mirror, mc.privateSource) + im.triggerPullMirror(t, mirror) + waitForPullMirror(t, mirror, sourceRepoSha) + } + }) + } + } + }) + + t.Run("migrate from repo config credentials", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + + mirrorRepo, _, cleanupMirror := tests.CreateDeclarativeRepoWithOptions(t, user2, + tests.DeclarativeRepoOptions{}, + ) + defer cleanupMirror() + + // Write to the repo a config file that would have plausibly existed before EncryptedRemoteAddress was + // introduced: + repoPath := mirrorRepo.RepoPath() + err := os.WriteFile(path.Join(repoPath, "config"), []byte(` +[core] + repositoryformatversion = 0 + filemode = true + bare = true +[remote "origin"] + url = https://user:password@example.com/org/repo.git + tagOpt = --no-tags + fetch = +refs/*:refs/* + mirror = true + fetch = +refs/tags/*:refs/tags/* +`), 0o644) + require.NoError(t, err) + + // Create a Mirror record without an EncryptedRemoteAddress: + mirror := &repo_model.Mirror{ + RepoID: mirrorRepo.ID, + Interval: 8 * time.Hour, + EnablePrune: true, + } + _, err = db.GetEngine(t.Context()).Insert(mirror) + require.NoError(t, err) + require.Nil(t, mirror.EncryptedRemoteAddress) + + remoteURL, err := mirror_service.DecryptOrRecoverRemoteAddress(t.Context(), mirror) + require.NoError(t, err) + assert.Equal(t, "https://user:password@example.com/org/repo.git", remoteURL.URL.String()) + + // EncryptedRemoteAddress should now be populated from the recovery: + assert.NotNil(t, mirror.EncryptedRemoteAddress) + maybeDecryptedURL, err := mirror.DecryptRemoteAddress() + require.NoError(t, err) + has, decryptedURL := maybeDecryptedURL.Get() + require.True(t, has) + assert.Equal(t, "https://user:password@example.com/org/repo.git", decryptedURL) + + // SanitizedRemoteAddress can be fetched: + maybeSanitizedURL, err := mirror.SanitizedRemoteAddress() + require.NoError(t, err) + has, sanitizedURL := maybeSanitizedURL.Get() + require.True(t, has) + assert.Equal(t, "https://user@example.com/org/repo.git", sanitizedURL) + + // Database record is updated in the database: + refetchMirror := unittest.AssertExistsAndLoadBean(t, &repo_model.Mirror{RepoID: mirrorRepo.ID}) + assert.Equal(t, mirror.EncryptedRemoteAddress, refetchMirror.EncryptedRemoteAddress) + + // Config file is rewritten: + config, err := os.ReadFile(path.Join(repoPath, "config")) + require.NoError(t, err) + assert.Equal(t, ` +[core] + repositoryformatversion = 0 + filemode = true + bare = true +[remote "origin"] + url = https://user@example.com/org/repo.git + tagOpt = --no-tags + fetch = +refs/*:refs/* + mirror = true + fetch = +refs/tags/*:refs/tags/* +`, string(config)) + }) +} + +func createPullMirrorViaWeb(t *testing.T, sourceRepo *repo_model.Repository, authenticate bool) string { + session := loginUser(t, "user2") + + mirrorName := fmt.Sprintf("pullmirror-%s", sourceRepo.Name) + form := &forms.MigrateRepoForm{ + CloneAddr: sourceRepo.CloneLink().HTTPS, + Service: structs.PlainGitService, + UID: 2, + RepoName: mirrorName, + Mirror: true, + } + if authenticate { + form.AuthUsername = "user2" + form.AuthPassword = getTokenForLoggedInUser(t, session, auth.AccessTokenScopeReadRepository) + } + + resp := session.MakeRequest(t, + NewRequestWithJSON(t, "POST", "/repo/migrate", form), + http.StatusSeeOther) + location := resp.Header().Get("Location") + assert.Equal(t, fmt.Sprintf("/user2/pullmirror-%s", sourceRepo.Name), location) + + var lastBody string + if !assert.Eventuallyf(t, + func() bool { + resp := session.MakeRequest(t, + NewRequest(t, "GET", location), + http.StatusOK) + body := resp.Body.String() + lastBody = body + // Looking for the repo page to be fully populated indicating that the migration is complete: + // Check that the first commit message is present: + if !strings.Contains(body, "Initial commit") { + return false + } + // Check that the fork button is present: + if !strings.Contains(body, fmt.Sprintf("/user2/%s/fork", mirrorName)) { + return false + } + return true + }, + 15*time.Second, 1*time.Second, + "expected migration to complete and repo page to render") { + t.Logf("last received page body: %s", lastBody) + } + + return mirrorName +} + +func createPullMirrorViaAPI(t *testing.T, sourceRepo *repo_model.Repository, authenticate bool) string { + session := loginUser(t, "user2") + apiToken := getTokenForLoggedInUser(t, session, auth.AccessTokenScopeWriteRepository) + + mirrorName := fmt.Sprintf("pullmirror-%s", sourceRepo.Name) + form := &structs.MigrateRepoOptions{ + CloneAddr: sourceRepo.CloneLink().HTTPS, + Service: "git", + RepoOwner: "user2", + RepoName: mirrorName, + Mirror: true, + } + if authenticate { + form.AuthUsername = "user2" + form.AuthPassword = getTokenForLoggedInUser(t, session, auth.AccessTokenScopeReadRepository) + } + + resp := session.MakeRequest(t, + NewRequestWithJSON(t, "POST", "/api/v1/repos/migrate", form).AddTokenAuth(apiToken), + http.StatusCreated) + var repo structs.Repository + DecodeJSON(t, resp, &repo) + assert.NotNil(t, repo) + assert.True(t, repo.Mirror) + assert.False(t, repo.Empty) + + return mirrorName +} + +func verifyPullMirrorViaWeb(t *testing.T, sourceRepo *repo_model.Repository, mirrorName string, authenticate bool) { + session := loginUser(t, "user2") + resp := session.MakeRequest(t, + NewRequestf(t, "GET", "/user2/%s/settings", mirrorName), + http.StatusOK) + htmlDoc := NewHTMLParser(t, resp.Body) + htmlDoc.AssertAttrEqual(t, "#mirror_address", "value", sourceRepo.CloneLink().HTTPS) + if authenticate { + htmlDoc.AssertAttrEqual(t, "#mirror_username", "value", "user2") + htmlDoc.AssertAttrEqual(t, "#mirror_password", "value", "") + htmlDoc.AssertAttrEqual(t, "#mirror_password", "placeholder", "(Unchanged)") + } else { + htmlDoc.AssertAttrEqual(t, "#mirror_username", "value", "") + htmlDoc.AssertAttrEqual(t, "#mirror_password", "value", "") + htmlDoc.AssertAttrEqual(t, "#mirror_password", "placeholder", "(Unset)") + } + + resp = session.MakeRequest(t, + NewRequestf(t, "GET", "/user2/%s", mirrorName), + http.StatusOK) + htmlDoc = NewHTMLParser(t, resp.Body) + htmlDoc.AssertElementPredicate(t, ".fork-flag", func(selection *goquery.Selection) bool { + text := strings.TrimSpace(selection.Text()) + assert.Contains(t, text, "mirror of") + assert.Contains(t, text, sourceRepo.CloneLink().HTTPS) + return true + }) +} + +func triggerPullMirrorViaWeb(t *testing.T, mirrorName string) { + session := loginUser(t, "user2") + + resp := session.MakeRequest(t, + NewRequestWithValues(t, "POST", fmt.Sprintf("/user2/%s/settings", mirrorName), map[string]string{"action": "mirror-sync"}), + http.StatusSeeOther) + location := resp.Header().Get("Location") + assert.Equal(t, fmt.Sprintf("/user2/%s/settings", mirrorName), location) +} + +func triggerPullMirrorViaAPI(t *testing.T, mirrorName string) { + session := loginUser(t, "user2") + apiToken := getTokenForLoggedInUser(t, session, auth.AccessTokenScopeWriteRepository) + + // Trigger sync... + session.MakeRequest(t, + NewRequestf(t, "POST", "/api/v1/repos/user2/%s/mirror-sync", mirrorName).AddTokenAuth(apiToken), + http.StatusOK) +} + +func changePullMirrorCredentialsViaWeb(t *testing.T, sourceRepo *repo_model.Repository, mirrorName string, authenticate bool) { + session := loginUser(t, "user2") + + form := map[string]string{ + "action": "mirror", + "enable_prune": "on", + "interval": "8h0m0s", + "mirror_address": sourceRepo.CloneLink().HTTPS, + } + if authenticate { + form["mirror_username"] = "user2" + form["mirror_password"] = getTokenForLoggedInUser(t, session, auth.AccessTokenScopeReadRepository) + } + + resp := session.MakeRequest(t, + NewRequestWithValues(t, "POST", fmt.Sprintf("/user2/%s/settings", mirrorName), form), + http.StatusSeeOther) + location := resp.Header().Get("Location") + assert.Equal(t, fmt.Sprintf("/user2/%s/settings", mirrorName), location) +} + +func verifyPullMirrorContents(t *testing.T, mirrorName, expectedSha string) { + session := loginUser(t, "user2") + apiToken := getTokenForLoggedInUser(t, session, auth.AccessTokenScopeReadRepository) + resp := session.MakeRequest(t, + NewRequest(t, "GET", fmt.Sprintf("/api/v1/repos/user2/%s/commits?sha=main&limit=1", mirrorName)).AddTokenAuth(apiToken), + http.StatusOK) + var commits []*structs.Commit + DecodeJSON(t, resp, &commits) + require.Len(t, commits, 1) + assert.Equal(t, expectedSha, commits[0].SHA) +} + +func waitForPullMirror(t *testing.T, mirrorName, expectedSha string) { + session := loginUser(t, "user2") + apiToken := getTokenForLoggedInUser(t, session, auth.AccessTokenScopeReadRepository) + + var commits []*structs.Commit + if !assert.Eventually(t, + func() bool { + resp := session.MakeRequest(t, + NewRequest(t, "GET", fmt.Sprintf("/api/v1/repos/user2/%s/commits?sha=main&limit=1", mirrorName)).AddTokenAuth(apiToken), + http.StatusOK) + DecodeJSON(t, resp, &commits) + require.Len(t, commits, 1) + return commits[0].SHA == expectedSha + }, + 15*time.Second, 1*time.Second) { + t.Logf("sync was supposed to bring repo to commit %s, but observed commits = %#v", expectedSha, commits) + } +} + +func getGitConfig(t *testing.T, configFile, configPath string) string { + stdout, stderr, err := process.GetManager().Exec("getGitConfig", "git", "config", "get", "--file", configFile, configPath) + require.NoError(t, err, "fetch config %s failed: git stderr: %s", configPath, stderr) + return strings.TrimSpace(stdout) +} + +func verifyPullMirrorConfig(t *testing.T, mirrorName string, sourceRepo *repo_model.Repository, authenticate bool) { + mirrorRepo, err := repo_model.GetRepositoryByOwnerAndName(t.Context(), "user2", mirrorName) require.NoError(t, err) - require.NoError(t, release_service.CreateRelease(gitRepo, &repo_model.Release{ - RepoID: repo.ID, - Repo: repo, - PublisherID: user.ID, - Publisher: user, - TagName: "v0.2", - Target: "master", - Title: "v0.2 is released", - Note: "v0.2 is released", - IsDraft: false, - IsPrerelease: false, - IsTag: true, - }, "", []*release_service.AttachmentChange{})) + repoPath := mirrorRepo.RepoPath() + configPath := path.Join(repoPath, "config") - _, err = repo_model.GetMirrorByRepoID(ctx, mirror.ID) + expectedURL := sourceRepo.CloneLink().HTTPS + if authenticate { + expectedURL = strings.Replace(expectedURL, "http://", "http://user2@", 1) + } + assert.Equal(t, expectedURL, getGitConfig(t, configPath, "remote.origin.url")) + assert.Equal(t, "true", getGitConfig(t, configPath, "remote.origin.mirror")) + assert.Equal(t, "+refs/tags/*:refs/tags/*", getGitConfig(t, configPath, "remote.origin.fetch")) +} + +func changePullMirrorSource(t *testing.T, sourceRepo *repo_model.Repository, sourceRepoSha string) string { + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + resp, err := files_service.ChangeRepoFiles(git.DefaultContext, sourceRepo, user2, + &files_service.ChangeRepoFilesOptions{ + Files: []*files_service.ChangeRepoFile{ + { + Operation: "update", + TreePath: "docs.md", + ContentReader: strings.NewReader(uuid.NewString()), + }, + }, + Message: "add files", + OldBranch: "main", + NewBranch: "main", + Author: &files_service.IdentityOptions{ + Name: user2.Name, + Email: user2.Email, + }, + Committer: &files_service.IdentityOptions{ + Name: user2.Name, + Email: user2.Email, + }, + Dates: &files_service.CommitDateOptions{ + Author: time.Now(), + Committer: time.Now(), + }, + LastCommitID: sourceRepoSha, + }) require.NoError(t, err) + assert.NotEmpty(t, resp) + return resp.Commit.SHA +} - ok := mirror_service.SyncPullMirror(ctx, mirror.ID) - assert.True(t, ok) +func renamePullMirrorSourceRepo(t *testing.T, sourceRepo *repo_model.Repository) *repo_model.Repository { + session := loginUser(t, "user2") + apiToken := getTokenForLoggedInUser(t, session, auth.AccessTokenScopeWriteRepository) - count, err := db.Count[repo_model.Release](db.DefaultContext, findOptions) - require.NoError(t, err) - assert.Equal(t, initCount+1, count) + newName := uuid.NewString() + session.MakeRequest(t, + NewRequestWithJSON(t, "PATCH", fmt.Sprintf("/api/v1/repos/user2/%s", sourceRepo.Name), + &structs.EditRepoOption{ + Name: &newName, + }).AddTokenAuth(apiToken), + http.StatusOK) - release, err := repo_model.GetRelease(db.DefaultContext, repo.ID, "v0.2") - require.NoError(t, err) - require.NoError(t, release_service.DeleteReleaseByID(ctx, repo, release, user, true)) - - ok = mirror_service.SyncPullMirror(ctx, mirror.ID) - assert.True(t, ok) - - count, err = db.Count[repo_model.Release](db.DefaultContext, findOptions) - require.NoError(t, err) - assert.Equal(t, initCount, count) + newRepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: sourceRepo.ID}) + assert.Equal(t, newRepo.Name, newName) + assert.NotEqual(t, newRepo.CloneLink().HTTPS, sourceRepo.CloneLink().HTTPS) // should have changed to new name + return newRepo } func TestPullMirrorRedactCredentials(t *testing.T) {