peezy-tech/jojo
2
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo.git synced 2026-05-12 22:10:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: support reusable workflow expansion when with or strategy.matrix contains ${{ needs... }} (#10647)

Browse source 
This change allows the `with:` field of a reusable workflow to reference a previous job, such as `with: { some-input: "${{ needs.other-job.outputs.other-output }}" }`.  `strategy.matrix` can also reference `${{ needs... }}`.

When a job is parsed and encounters this situation, the outer job of the workflow is marked with a field `incomplete_with` (or `incomplete_matrix`), indicating to Forgejo that it can't be executed as-is and the other jobs in its `needs` list need to be completed first.  And then in `job_emitter.go` when one job is completed, it checks if other jobs had a `needs` reference to it and unblocks those jobs -- but if they're marked with `incomplete_with` then they can be sent back through the job parser, with the now-available job outputs, to be expanded into the correct definition of the job.

The core functionality for this already exists to allow `runs-on` and `strategy.matrix` to reference the outputs of other jobs, but it is expanded upon here to include `with` for reusable workflows.

There is one known defect in this implementation, but it has a limited scope -- if this code path is used to expand a nested reusable workflow, then the `${{ input.... }}` context will be incorrect.  This will require an update to the jobparser in runner version 12.4.0, and so it is out-of-scope of this PR.

## Checklist

The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org).

### Tests

- I added test coverage for Go changes...
  - [x] in their respective `*_test.go` for unit tests.
  - [ ] in the `tests/integration` directory if it involves interactions with a live Forgejo server.
- I added test coverage for JavaScript changes...
  - [ ] in `web_src/js/*.test.js` if it can be unit tested.
  - [ ] in `tests/e2e/*.test.e2e.js` if it requires interactions with a live Forgejo server (see also the [developer guide for JavaScript testing](https://codeberg.org/forgejo/forgejo/src/branch/forgejo/tests/e2e/README.md#end-to-end-tests)).
- **end-to-end test:** will require the noted "known defect" to be resolved, but tests are authored at https://code.forgejo.org/forgejo/end-to-end/compare/main...mfenniak:expand-reusable-workflows-needs

### Documentation

- [ ] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change.
- [x] I did not document these changes and I do not expect someone else to do it.

### Release notes

- [ ] I do not want this change to show in the release notes.
- [x] I want the title to show in the release notes with a link to this pull request.
- [ ] I want the content of the `release-notes/<pull request number>.md` to be be used for the release notes instead of the title.

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/10647
Reviewed-by: Andreas Ahlenstorf <aahlenst@noreply.codeberg.org>
Co-authored-by: Mathieu Fenniak <mathieu@fenniak.net>
Co-committed-by: Mathieu Fenniak <mathieu@fenniak.net>
This commit is contained in:
Mathieu Fenniak 2025-12-31 19:04:35 +01:00 • committed by Mathieu Fenniak
parent 0a6a5cb73e
commit 75cb38faa6
15 changed files with 971 additions and 65 deletions
Download patch file Download diff file Expand all files Collapse all files

12
models/actions/pre_execution_errors.go
View file

@ -26,6 +26,10 @@ const (
ErrorCodeIncompleteRunsOnMissingOutput
ErrorCodeIncompleteRunsOnMissingMatrixDimension
ErrorCodeIncompleteRunsOnUnknownCause
ErrorCodeIncompleteWithMissingJob
ErrorCodeIncompleteWithMissingOutput
ErrorCodeIncompleteWithMissingMatrixDimension
ErrorCodeIncompleteWithUnknownCause
)
func TranslatePreExecutionError(lang translation.Locale, run *ActionRun) string {
@ -57,6 +61,14 @@ func TranslatePreExecutionError(lang translation.Locale, run *ActionRun) string
return lang.TrString("actions.workflow.incomplete_runson_missing_matrix_dimension", run.PreExecutionErrorDetails...)
case ErrorCodeIncompleteRunsOnUnknownCause:
return lang.TrString("actions.workflow.incomplete_runson_unknown_cause", run.PreExecutionErrorDetails...)
case ErrorCodeIncompleteWithMissingJob:
return lang.TrString("actions.workflow.incomplete_with_missing_job", run.PreExecutionErrorDetails...)
case ErrorCodeIncompleteWithMissingOutput:
return lang.TrString("actions.workflow.incomplete_with_missing_output", run.PreExecutionErrorDetails...)
case ErrorCodeIncompleteWithMissingMatrixDimension:
return lang.TrString("actions.workflow.incomplete_with_missing_matrix_dimension", run.PreExecutionErrorDetails...)
case ErrorCodeIncompleteWithUnknownCause:
return lang.TrString("actions.workflow.incomplete_with_unknown_cause", run.PreExecutionErrorDetails...)
}
return fmt.Sprintf("<unsupported error: code=%v details=%#v", run.PreExecutionErrorCode, run.PreExecutionErrorDetails)
}

36
models/actions/pre_execution_errors_test.go
View file

@ -60,7 +60,7 @@ func TestTranslatePreExecutionError(t *testing.T) {
PreExecutionErrorCode: ErrorCodeIncompleteMatrixMissingOutput,
PreExecutionErrorDetails: []any{"blocked_job", "other_job", "some_output"},
},
expected: "Unable to evaluate `strategy.matrix` of job blocked_job: job other_job does not have an output some_output.",
expected: "Unable to evaluate `strategy.matrix` of job blocked_job: job other_job is missing output some_output.",
},
{
name: "ErrorCodeIncompleteMatrixMissingJob",
@ -84,7 +84,7 @@ func TestTranslatePreExecutionError(t *testing.T) {
PreExecutionErrorCode: ErrorCodeIncompleteRunsOnMissingOutput,
PreExecutionErrorDetails: []any{"blocked_job", "other_job", "some_output"},
},
expected: "Unable to evaluate `runs-on` of job blocked_job: job other_job does not have an output some_output.",
expected: "Unable to evaluate `runs-on` of job blocked_job: job other_job is missing output some_output.",
},
{
name: "ErrorCodeIncompleteRunsOnMissingJob",
@ -110,6 +110,38 @@ func TestTranslatePreExecutionError(t *testing.T) {
},
expected: "Unable to evaluate `runs-on` of job blocked_job: unknown error.",
},
{
name: "ErrorCodeIncompleteWithMissingOutput",
run: &ActionRun{
PreExecutionErrorCode: ErrorCodeIncompleteWithMissingOutput,
PreExecutionErrorDetails: []any{"blocked_job", "other_job", "some_output"},
},
expected: "Unable to evaluate `with` of job blocked_job: job other_job is missing output some_output.",
},
{
name: "ErrorCodeIncompleteWithMissingJob",
run: &ActionRun{
PreExecutionErrorCode: ErrorCodeIncompleteWithMissingJob,
PreExecutionErrorDetails: []any{"blocked_job", "other_job", "needs-1, needs-2"},
},
expected: "Unable to evaluate `with` of job blocked_job: job other_job is not in the `needs` list of job blocked_job (needs-1, needs-2).",
},
{
name: "ErrorCodeIncompleteWithMissingMatrixDimension",
run: &ActionRun{
PreExecutionErrorCode: ErrorCodeIncompleteWithMissingMatrixDimension,
PreExecutionErrorDetails: []any{"blocked_job", "platfurm"},
},
expected: "Unable to evaluate `with` of job blocked_job: matrix dimension platfurm does not exist.",
},
{
name: "ErrorCodeIncompleteWithUnknownCause",
run: &ActionRun{
PreExecutionErrorCode: ErrorCodeIncompleteWithUnknownCause,
PreExecutionErrorDetails: []any{"blocked_job"},
},
expected: "Unable to evaluate `with` of job blocked_job: unknown error.",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {

2
models/actions/run.go
View file

@ -366,7 +366,7 @@ func InsertRunJobs(ctx context.Context, run *ActionRun, jobs []*jobparser.Single
}
payload, _ = v.Marshal()
if len(needs) > 0 || run.NeedApproval || v.IncompleteMatrix || v.IncompleteRunsOn {
if len(needs) > 0 || run.NeedApproval || v.IncompleteMatrix || v.IncompleteRunsOn || v.IncompleteWith {
status = StatusBlocked
} else {
status = StatusWaiting

22
models/actions/run_job.go
View file

@ -260,7 +260,7 @@ func (job *ActionRunJob) ClearCachedWorkflowPayload() {
// Checks whether the target job is an `(incomplete matrix)` job that will be blocked until the matrix is complete, and
// then regenerated and deleted. If it is incomplete, and if the information is available, the specific job and/or
// output that causes it to be incomplete will be returned as well.
func (job *ActionRunJob) IsIncompleteMatrix() (bool, *jobparser.IncompleteNeeds, error) {
func (job *ActionRunJob) HasIncompleteMatrix() (bool, *jobparser.IncompleteNeeds, error) {
jobWorkflow, err := job.DecodeWorkflowPayload()
if err != nil {
return false, nil, fmt.Errorf("failure decoding workflow payload: %w", err)
@ -270,7 +270,7 @@ func (job *ActionRunJob) IsIncompleteMatrix() (bool, *jobparser.IncompleteNeeds,
// Checks whether the target job has a `runs-on` field with an expression that requires an input from another job. The
// job will be blocked until the other job is complete, and then regenerated and deleted.
func (job *ActionRunJob) IsIncompleteRunsOn() (bool, *jobparser.IncompleteNeeds, *jobparser.IncompleteMatrix, error) {
func (job *ActionRunJob) HasIncompleteRunsOn() (bool, *jobparser.IncompleteNeeds, *jobparser.IncompleteMatrix, error) {
jobWorkflow, err := job.DecodeWorkflowPayload()
if err != nil {
return false, nil, nil, fmt.Errorf("failure decoding workflow payload: %w", err)
@ -278,6 +278,15 @@ func (job *ActionRunJob) IsIncompleteRunsOn() (bool, *jobparser.IncompleteNeeds,
return jobWorkflow.IncompleteRunsOn, jobWorkflow.IncompleteRunsOnNeeds, jobWorkflow.IncompleteRunsOnMatrix, nil
}
// Check whether the target job was generated as a result of expanding a reusable workflow.
func (job *ActionRunJob) IsWorkflowCallInnerJob() (bool, error) {
jobWorkflow, err := job.DecodeWorkflowPayload()
if err != nil {
return false, fmt.Errorf("failure decoding workflow payload: %w", err)
}
return jobWorkflow.Metadata.WorkflowCallParent != "", nil
}
// Check whether this job is a caller of a reusable workflow -- in other words, the real work done in this job is in
// spawned child jobs, not this job.
func (job *ActionRunJob) IsWorkflowCallOuterJob() (bool, error) {
@ -288,11 +297,12 @@ func (job *ActionRunJob) IsWorkflowCallOuterJob() (bool, error) {
return jobWorkflow.Metadata.WorkflowCallID != "", nil
}
// Check whether the target job was generated as a result of expanding a reusable workflow.
func (job *ActionRunJob) IsWorkflowCallInnerJob() (bool, error) {
// Checks whether the target job has a `with` field with an expression that requires an input from another job. The job
// will be blocked until the other job is complete, and then regenerated and deleted.
func (job *ActionRunJob) HasIncompleteWith() (bool, *jobparser.IncompleteNeeds, *jobparser.IncompleteMatrix, error) {
jobWorkflow, err := job.DecodeWorkflowPayload()
if err != nil {
return false, fmt.Errorf("failure decoding workflow payload: %w", err)
return false, nil, nil, fmt.Errorf("failure decoding workflow payload: %w", err)
}
return jobWorkflow.Metadata.WorkflowCallParent != "", nil
return jobWorkflow.IncompleteWith, jobWorkflow.IncompleteWithNeeds, jobWorkflow.IncompleteWithMatrix, nil
}

56
models/actions/run_job_test.go
View file

@ -72,7 +72,7 @@ func TestActionRunJob_HTMLURL(t *testing.T) {
}
}
func TestActionRunJob_IsIncompleteMatrix(t *testing.T) {
func TestActionRunJob_HasIncompleteMatrix(t *testing.T) {
tests := []struct {
name string
job ActionRunJob
@ -100,7 +100,7 @@ func TestActionRunJob_IsIncompleteMatrix(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
isIncomplete, needs, err := tt.job.IsIncompleteMatrix()
isIncomplete, needs, err := tt.job.HasIncompleteMatrix()
if tt.errContains != "" {
assert.ErrorContains(t, err, tt.errContains)
} else {
@ -112,7 +112,7 @@ func TestActionRunJob_IsIncompleteMatrix(t *testing.T) {
}
}
func TestActionRunJob_IsIncompleteRunsOn(t *testing.T) {
func TestActionRunJob_HasIncompleteRunsOn(t *testing.T) {
tests := []struct {
name string
job ActionRunJob
@ -147,7 +147,7 @@ func TestActionRunJob_IsIncompleteRunsOn(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
isIncomplete, needs, matrix, err := tt.job.IsIncompleteRunsOn()
isIncomplete, needs, matrix, err := tt.job.HasIncompleteRunsOn()
if tt.errContains != "" {
assert.ErrorContains(t, err, tt.errContains)
} else {
@ -233,3 +233,51 @@ func TestActionRunJob_IsWorkflowCallInnerJob(t *testing.T) {
})
}
}
func TestActionRunJob_HasIncompleteWith(t *testing.T) {
tests := []struct {
name string
job ActionRunJob
isIncomplete bool
needs *jobparser.IncompleteNeeds
matrix *jobparser.IncompleteMatrix
errContains string
}{
{
name: "normal workflow",
job: ActionRunJob{WorkflowPayload: []byte("name: workflow")},
isIncomplete: false,
},
{
name: "incomplete_with workflow",
job: ActionRunJob{WorkflowPayload: []byte("name: workflow\nincomplete_with: true\nincomplete_with_needs: { job: abc }")},
needs: &jobparser.IncompleteNeeds{Job: "abc"},
isIncomplete: true,
},
{
name: "incomplete_with workflow",
job: ActionRunJob{WorkflowPayload: []byte("name: workflow\nincomplete_with: true\nincomplete_with_matrix: { dimension: abc }")},
matrix: &jobparser.IncompleteMatrix{Dimension: "abc"},
isIncomplete: true,
},
{
name: "unparseable workflow",
job: ActionRunJob{WorkflowPayload: []byte("name: []\nincomplete_with: true")},
errContains: "failure unmarshaling WorkflowPayload to SingleWorkflow: yaml: unmarshal errors",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
isIncomplete, needs, matrix, err := tt.job.HasIncompleteWith()
if tt.errContains != "" {
assert.ErrorContains(t, err, tt.errContains)
} else {
require.NoError(t, err)
assert.Equal(t, tt.isIncomplete, isIncomplete)
assert.Equal(t, tt.needs, needs)
assert.Equal(t, tt.matrix, matrix)
}
})
}
}

48
models/actions/run_test.go
View file

@ -327,3 +327,51 @@ jobs:
})
}
}
func TestActionRun_IncompleteWith(t *testing.T) {
require.NoError(t, unittest.PrepareTestDatabase())
pullRequestPosterID := int64(4)
repoID := int64(10)
pullRequestID := int64(2)
runDoesNotNeedApproval := &ActionRun{
RepoID: repoID,
PullRequestID: pullRequestID,
PullRequestPosterID: pullRequestPosterID,
}
workflowRaw := []byte(`
jobs:
outer-job:
with:
some_input: ${{ needs.other-job.outputs.some-output }}
uses: ./.forgejo/workflows/reusable.yml
`)
workflows, err := jobparser.Parse(workflowRaw, false,
jobparser.WithJobOutputs(map[string]map[string]string{}),
jobparser.ExpandLocalReusableWorkflows(func(job *jobparser.Job, path string) ([]byte, error) {
return []byte(`
on:
workflow_call:
inputs:
some_input:
type: string
jobs:
inner-job:
runs-on: debian
steps: []
`), nil
}))
require.NoError(t, err)
require.True(t, workflows[0].IncompleteWith) // must be set for this test scenario to be valid
require.NoError(t, InsertRun(t.Context(), runDoesNotNeedApproval, workflows))
jobs, err := db.Find[ActionRunJob](t.Context(), FindRunJobOptions{RunID: runDoesNotNeedApproval.ID})
require.NoError(t, err)
require.Len(t, jobs, 1)
job := jobs[0]
// Expect job with an incomplete with to be StatusBlocked:
assert.Equal(t, StatusBlocked, job.Status)
}