[v15.0/forgejo] fix: store pull mirror creds encrypted with keying (#11984)

**Backport:** https://codeberg.org/forgejo/forgejo/pulls/11909

Fixes #9629.

New pull mirrors have credentials stored encrypted in the database, the same as push mirrors, rather than in the repository's `config` file.  `git fetch` on the pull mirror is updated to use the credential store.  Pull mirrors will have their credentials migrated to the encrypted storage in the database as they're synced or otherwise accessed via the web UI.

## Checklist

The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. All work and communication must conform to Forgejo's [AI Agreement](https://codeberg.org/forgejo/governance/src/branch/main/AIAgreement.md). There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org).

### Tests for Go changes

- I added test coverage for Go changes...
  - [ ] in their respective `*_test.go` for unit tests.
  - [x] in the `tests/integration` directory if it involves interactions with a live Forgejo server.
- I ran...
  - [x] `make pr-go` before pushing

### Documentation

- [ ] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change.
- [x] I did not document these changes and I do not expect someone else to do it.

### Release notes

- [x] This change will be noticed by a Forgejo user or admin (feature, bug fix, performance, etc.). I suggest to include a release note for this change.
- [ ] This change is not visible to a Forgejo user or admin (refactor, dependency upgrade, etc.). I think there is no need to add a release note for this change.

Co-authored-by: Mathieu Fenniak <mathieu@fenniak.net>
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/11984
Reviewed-by: Mathieu Fenniak <mfenniak@noreply.codeberg.org>
Co-authored-by: forgejo-backport-action <forgejo-backport-action@noreply.codeberg.org>
Co-committed-by: forgejo-backport-action <forgejo-backport-action@noreply.codeberg.org>

modules/git/command.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net/url"
 	"os"
 	"os/exec"
 	"runtime/trace"
@@ -446,6 +447,54 @@ func (c *Command) RunStdBytes(opts *RunOpts) (stdout, stderr []byte, runErr RunS
 	return stdoutBuf.Bytes(), stderr, nil
 }
 
+// If `remoteURL` is a URL with a password in it, add parameters to the git command that will read that password from a
+// credential store file, and return the URL that should be used in the command instead of the original, and a cleanup
+// function to call to remove the credential file. If `remoteURL` doesn't have a password, then it is returned as-is.
+// This function must be invoked on the the git command before the git sub-command -- eg. before the `clone` or `fetch`
+// parameter is added to the command's args.
+func (c *Command) AddAuthCredentialHelperForRemote(remoteURL string) (commandURL string, cleanup func(), err error) {
+	parsedFromURL, _ := url.Parse(remoteURL)
+
+	// If the clone URL has credentials, build a credential file for usage by git-credential-store
+	// to prevent credential leak in the process list.
+	// https://git-scm.com/docs/git-credential-store#_storage_format
+	// credential.helper adjustment must be set before the git subcommand
+	if strings.Contains(remoteURL, "://") && strings.Contains(remoteURL, "@") && parsedFromURL != nil {
+		credentialsFile, err := os.CreateTemp("", "forgejo-clone-credentials-")
+		if err != nil {
+			return "", nil, err
+		}
+		credentialsPath := credentialsFile.Name()
+
+		cleanup := func() {
+			_ = credentialsFile.Close()
+			if err := util.Remove(credentialsPath); err != nil {
+				log.Warn("Unable to remove temporary file %q: %v", credentialsPath, err)
+			}
+		}
+		_, err = credentialsFile.Write([]byte(parsedFromURL.String()))
+		if err != nil {
+			cleanup()
+			return "", nil, err
+		}
+		err = credentialsFile.Close()
+		if err != nil {
+			cleanup()
+			return "", nil, err
+		}
+
+		c.AddArguments("-c").AddDynamicArguments("credential.helper=store --file=" + credentialsPath)
+
+		// remove the password from the URL argument
+		parsedFromURL.User = url.User(parsedFromURL.User.Username())
+		commandURL = parsedFromURL.String()
+
+		return commandURL, cleanup, nil
+	}
+
+	return remoteURL, func() {}, nil
+}
+
 // AllowLFSFiltersArgs return globalCommandArgs with lfs filter, it should only be used for tests
 func AllowLFSFiltersArgs() TrustedCmdArgs {
 	// Now here we should explicitly allow lfs filters to run

modules/git/repo.go

@@ -18,7 +18,6 @@ import (
 	"strings"
 	"time"
 
-	"forgejo.org/modules/log"
 	"forgejo.org/modules/proxy"
 	"forgejo.org/modules/setting"
 	"forgejo.org/modules/util"
@@ -141,44 +140,13 @@ func CloneWithArgs(ctx context.Context, args TrustedCmdArgs, from, to string, op
 		envs = proxy.EnvWithProxy(parsedFromURL)
 	}
 
-	fromURL := from
-	sanitizedFrom := from
+	sanitizedFrom := util.SanitizeCredentialURLs(from)
 
-	// If the clone URL has credentials, build a credential file for usage by git-credential-store
-	// to prevent credential leak in the process list.
-	// https://git-scm.com/docs/git-credential-store#_storage_format
-	// credential.helper adjustment must be set before the git subcommand
-	if strings.Contains(from, "://") && strings.Contains(from, "@") {
-		sanitizedFrom = util.SanitizeCredentialURLs(from)
-		if parsedFromURL != nil {
-			credentialsFile, err := os.CreateTemp("", "forgejo-clone-credentials-")
-			if err != nil {
-				return err
-			}
-			credentialsPath := credentialsFile.Name()
-
-			defer func() {
-				_ = credentialsFile.Close()
-				if err := util.Remove(credentialsPath); err != nil {
-					log.Warn("Unable to remove temporary file %q: %v", credentialsPath, err)
-				}
-			}()
-			_, err = credentialsFile.Write([]byte(parsedFromURL.String()))
-			if err != nil {
-				return err
-			}
-			err = credentialsFile.Close()
-			if err != nil {
-				return err
-			}
-
-			cmd.AddArguments("-c").AddDynamicArguments("credential.helper=store --file=" + credentialsPath)
-
-			// remove the password from the URL argument
-			parsedFromURL.User = url.User(parsedFromURL.User.Username())
-			fromURL = parsedFromURL.String()
-		}
+	fromURL, cleanup, err := cmd.AddAuthCredentialHelperForRemote(from)
+	if err != nil {
+		return err
 	}
+	defer cleanup()
 
 	cmd.AddArguments("clone")
 

modules/keying/keying.go

@@ -41,6 +41,8 @@ var (
 	MigrateTask = deriveKey("migrate_repo_task")
 	// Used for the `webhook` table.
 	Webhook = deriveKey("webhook")
+	// Used for the `mirror` table.
+	PullMirror = deriveKey("pullmirror")
 )
 
 var (

modules/templates/util_misc.go

@@ -17,12 +17,11 @@ import (
 	asymkey_model "forgejo.org/models/asymkey"
 	repo_model "forgejo.org/models/repo"
 	user_model "forgejo.org/models/user"
-	"forgejo.org/modules/git"
-	giturl "forgejo.org/modules/git/url"
 	"forgejo.org/modules/json"
 	"forgejo.org/modules/log"
 	"forgejo.org/modules/repository"
 	"forgejo.org/modules/svg"
+	mirror_service "forgejo.org/services/mirror"
 
 	"github.com/editorconfig/editorconfig-core-go/v2"
 )
@@ -164,17 +163,11 @@ type remoteAddress struct {
 	Password string
 }
 
-func mirrorRemoteAddress(ctx context.Context, m *repo_model.Repository, remoteName string) remoteAddress {
+func mirrorRemoteAddress(ctx context.Context, mirror *repo_model.Mirror) remoteAddress {
 	ret := remoteAddress{}
-	remoteURL, err := git.GetRemoteAddress(ctx, m.RepoPath(), remoteName)
+	u, err := mirror_service.DecryptOrRecoverRemoteAddress(ctx, mirror)
 	if err != nil {
-		log.Error("GetRemoteURL %v", err)
-		return ret
-	}
-
-	u, err := giturl.Parse(remoteURL)
-	if err != nil {
-		log.Error("giturl.Parse %v", err)
+		log.Error("DecryptOrRecoverRemoteAddress %v", err)
 		return ret
 	}