peezy-tech/jojo
2
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo.git synced 2026-05-12 22:10:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(actions): support referencing ${{ needs... }} variables in strategy.matrix (#10244)

Browse source 
https://code.forgejo.org/forgejo/forgejo-actions-feature-requests/issues/71 requires partial implementation in runner, and partial in Forgejo; this is the Forgejo implementation.

Allows for the definition of dynamic job matrixes in Forgejo Actions, where an earlier job provides and output that is used in `strategy.matrix` for a later job that requires it.  For example, adapted from the GitHub Actions example for this feature:

```yaml
name: shared matrix
on:
  push:
  workflow_dispatch:

jobs:
  define-matrix:
    runs-on: docker

    outputs:
      colors: ${{ steps.colors.outputs.colors }}

    steps:
      - name: Define Colors
        id: colors
        run: |
          echo 'colors=["red", "green", "blue"]' >> "$GITHUB_OUTPUT"

  produce-artifacts:
    runs-on: docker
    needs: define-matrix
    strategy:
      matrix:
        color: ${{ fromJSON(needs.define-matrix.outputs.colors) }}

    steps:
      - name: Define Color
        env:
          color: ${{ matrix.color }}
        run: |
          echo "$color" > color
      - name: Produce Artifact
        uses: https://data.forgejo.org/forgejo/upload-artifact@v4
        with:
          name: ${{ matrix.color }}
          path: color

  consume-artifacts:
    runs-on: docker
    needs:
    - define-matrix
    - produce-artifacts
    strategy:
      matrix:
        color: ${{ fromJSON(needs.define-matrix.outputs.colors) }}

    steps:
    - name: Retrieve Artifact
      uses: https://data.forgejo.org/forgejo/download-artifact@v4
      with:
        name: ${{ matrix.color }}

    - name: Report Color
      run: |
        cat color
```

## Checklist

The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org).

### Tests

- I added test coverage for Go changes...
  - [x] in their respective `*_test.go` for unit tests.
  - [x] in the `tests/integration` directory if it involves interactions with a live Forgejo server.
- I added test coverage for JavaScript changes...
  - [ ] in `web_src/js/*.test.js` if it can be unit tested.
  - [ ] in `tests/e2e/*.test.e2e.js` if it requires interactions with a live Forgejo server (see also the [developer guide for JavaScript testing](https://codeberg.org/forgejo/forgejo/src/branch/forgejo/tests/e2e/README.md#end-to-end-tests)).

### Documentation

- [x] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change.
    - https://codeberg.org/forgejo/docs/pulls/1607
- [ ] I did not document these changes and I do not expect someone else to do it.

### Release notes

- [ ] I do not want this change to show in the release notes.
- [x] I want the title to show in the release notes with a link to this pull request.
- [ ] I want the content of the `release-notes/<pull request number>.md` to be be used for the release notes instead of the title.

<!--start release-notes-assistant-->

## Release notes
<!--URL:https://codeberg.org/forgejo/forgejo-->
- Features
  - [PR](https://codeberg.org/forgejo/forgejo/pulls/10244): <!--number 10244 --><!--line 0 --><!--description ZmVhdChhY3Rpb25zKTogc3VwcG9ydCByZWZlcmVuY2luZyAke3sgbmVlZHMuLi4gfX0gdmFyaWFibGVzIGluIGBzdHJhdGVneS5tYXRyaXhg-->feat(actions): support referencing ${{ needs... }} variables in `strategy.matrix`<!--description-->
<!--end release-notes-assistant-->

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/10244
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
Co-authored-by: Mathieu Fenniak <mathieu@fenniak.net>
Co-committed-by: Mathieu Fenniak <mathieu@fenniak.net>
This commit is contained in:
Mathieu Fenniak 2025-11-29 17:49:04 +01:00 • committed by Mathieu Fenniak
parent 20dd01908c
commit 482ba3a4e5
17 changed files with 1084 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

152
services/actions/Test_tryHandleIncompleteMatrix/action_run.yml Normal file
View file

@ -0,0 +1,152 @@
-
id: 900
title: "running workflow_dispatch run"
repo_id: 63
owner_id: 2
workflow_id: "running.yaml"
index: 4
trigger_user_id: 2
ref: "refs/heads/main"
commit_sha: "97f29ee599c373c729132a5c46a046978311e0ee"
trigger_event: "workflow_dispatch"
is_fork_pull_request: 0
status: 6 # running
started: 1683636528
created: 1683636108
updated: 1683636626
need_approval: 0
approved_by: 0
concurrency_group: abc123
-
id: 901
title: "running workflow_dispatch run"
repo_id: 63
owner_id: 2
workflow_id: "running.yaml"
index: 5
trigger_user_id: 2
ref: "refs/heads/main"
commit_sha: "97f29ee599c373c729132a5c46a046978311e0ee"
trigger_event: "workflow_dispatch"
is_fork_pull_request: 0
status: 6 # running
started: 1683636528
created: 1683636108
updated: 1683636626
need_approval: 0
approved_by: 0
concurrency_group: abc123
-
id: 902
title: "running workflow_dispatch run"
repo_id: 63
owner_id: 2
workflow_id: "running.yaml"
index: 6
trigger_user_id: 2
ref: "refs/heads/main"
commit_sha: "97f29ee599c373c729132a5c46a046978311e0ee"
trigger_event: "workflow_dispatch"
is_fork_pull_request: 0
status: 6 # running
started: 1683636528
created: 1683636108
updated: 1683636626
need_approval: 0
approved_by: 0
concurrency_group: abc123
-
id: 903
title: "running workflow_dispatch run"
repo_id: 63
owner_id: 2
workflow_id: "running.yaml"
index: 7
trigger_user_id: 2
ref: "refs/heads/main"
commit_sha: "97f29ee599c373c729132a5c46a046978311e0ee"
trigger_event: "workflow_dispatch"
is_fork_pull_request: 0
status: 6 # running
started: 1683636528
created: 1683636108
updated: 1683636626
need_approval: 0
approved_by: 0
concurrency_group: abc123
-
id: 904
title: "running workflow_dispatch run"
repo_id: 63
owner_id: 2
workflow_id: "running.yaml"
index: 8
trigger_user_id: 2
ref: "refs/heads/main"
commit_sha: "97f29ee599c373c729132a5c46a046978311e0ee"
trigger_event: "workflow_dispatch"
is_fork_pull_request: 0
status: 6 # running
started: 1683636528
created: 1683636108
updated: 1683636626
need_approval: 0
approved_by: 0
concurrency_group: abc123
-
id: 905
title: "running workflow_dispatch run"
repo_id: 63
owner_id: 2
workflow_id: "running.yaml"
index: 9
trigger_user_id: 2
ref: "refs/heads/main"
commit_sha: "97f29ee599c373c729132a5c46a046978311e0ee"
trigger_event: "workflow_dispatch"
is_fork_pull_request: 0
status: 6 # running
started: 1683636528
created: 1683636108
updated: 1683636626
need_approval: 0
approved_by: 0
concurrency_group: abc123
-
id: 906
title: "running workflow_dispatch run"
repo_id: 63
owner_id: 2
workflow_id: "running.yaml"
index: 10
trigger_user_id: 2
ref: "refs/heads/main"
commit_sha: "97f29ee599c373c729132a5c46a046978311e0ee"
trigger_event: "workflow_dispatch"
is_fork_pull_request: 0
status: 6 # running
started: 1683636528
created: 1683636108
updated: 1683636626
need_approval: 0
approved_by: 0
concurrency_group: abc123
-
id: 907
title: "running workflow_dispatch run"
repo_id: 63
owner_id: 2
workflow_id: "running.yaml"
index: 11
trigger_user_id: 2
ref: "refs/heads/main"
commit_sha: "97f29ee599c373c729132a5c46a046978311e0ee"
trigger_event: "workflow_dispatch"
is_fork_pull_request: 0
status: 6 # running
started: 1683636528
created: 1683636108
updated: 1683636626
need_approval: 0
approved_by: 0
concurrency_group: abc123

320
services/actions/Test_tryHandleIncompleteMatrix/action_run_job.yml Normal file
View file

@ -0,0 +1,320 @@
-
id: 600
run_id: 900
repo_id: 63
owner_id: 2
commit_sha: 97f29ee599c373c729132a5c46a046978311e0ee
is_fork_pull_request: 0
name: job_1
attempt: 0
job_id: job_1
task_id: 0
status: 7 # blocked
runs_on: '["fedora"]'
started: 1683636528
needs: '["job1", "job2"]'
workflow_payload: |
"on":
push:
jobs:
produce-artifacts:
name: produce-artifacts
runs-on: docker
steps:
- run: echo "OK!"
strategy:
matrix:
color: red
-
id: 601
run_id: 901
repo_id: 63
owner_id: 2
commit_sha: 97f29ee599c373c729132a5c46a046978311e0ee
is_fork_pull_request: 0
name: job_1
attempt: 0
job_id: job_1
task_id: 0
status: 7 # blocked
runs_on: '["fedora"]'
needs: '["define-matrix"]'
workflow_payload: |
"on":
push:
jobs:
produce-artifacts:
name: produce-artifacts (incomplete matrix)
runs-on: docker
steps:
- run: echo "OK!"
strategy:
matrix:
color: ${{ fromJSON(needs.define-matrix.outputs.colors) }}
incomplete_matrix: true
-
id: 602
run_id: 901
repo_id: 63
owner_id: 2
commit_sha: 97f29ee599c373c729132a5c46a046978311e0ee
is_fork_pull_request: 0
name: define-matrix
attempt: 0
job_id: define-matrix
task_id: 100
status: 1 # success
runs_on: '["fedora"]'
-
id: 603
run_id: 902
repo_id: 63
owner_id: 2
commit_sha: 97f29ee599c373c729132a5c46a046978311e0ee
is_fork_pull_request: 0
name: job_1
attempt: 0
job_id: job_1
task_id: 0
status: 7 # blocked
runs_on: '["fedora"]'
needs: '["define-matrix"]'
workflow_payload: |
"on":
push:
jobs:
produce-artifacts:
name: produce-artifacts (incomplete matrix)
runs-on: docker
steps:
- run: echo "OK!"
strategy:
matrix:
color: ${{ fromJSON(needs.define-matrix.outputs.colors) }}
incomplete_matrix: true
-
id: 604
run_id: 902
repo_id: 63
owner_id: 2
commit_sha: 97f29ee599c373c729132a5c46a046978311e0ee
is_fork_pull_request: 0
name: define-matrix
attempt: 0
job_id: define-matrix
task_id: 100
status: 7 # blocked
runs_on: '["fedora"]'
-
id: 605
run_id: 903
repo_id: 63
owner_id: 2
commit_sha: 97f29ee599c373c729132a5c46a046978311e0ee
is_fork_pull_request: 0
name: job_1
attempt: 0
job_id: job_1
task_id: 0
status: 7 # blocked
runs_on: '["fedora"]'
needs: '["define-matrix-1"]'
workflow_payload: |
"on":
push:
jobs:
produce-artifacts:
name: produce-artifacts (incomplete matrix)
runs-on: docker
steps:
- run: echo "OK!"
strategy:
matrix:
color: ${{ fromJSON(needs.define-matrix-1.outputs.colors) }}
brightness: ${{ fromJSON(needs.define-matrix-2.outputs.colors) }}
incomplete_matrix: true
-
id: 606
run_id: 903
repo_id: 63
owner_id: 2
commit_sha: 97f29ee599c373c729132a5c46a046978311e0ee
is_fork_pull_request: 0
name: define-matrix-1
attempt: 0
job_id: define-matrix-1
task_id: 100
status: 1 # success
runs_on: '["fedora"]'
-
id: 607
run_id: 904
repo_id: 63
owner_id: 2
commit_sha: 97f29ee599c373c729132a5c46a046978311e0ee
is_fork_pull_request: 0
name: job_1
attempt: 0
job_id: job_1
task_id: 0
status: 7 # blocked
runs_on: '["fedora"]'
needs: '["define-matrix"]'
workflow_payload: |
"on":
push:
jobs:
produce-artifacts:
name: produce-artifacts (incomplete matrix)
runs-on: docker
steps:
- run: echo "OK!"
strategy:
matrix:
color: ${{ fromJSON(needs.define-matrix.outputs.colors) }}
incomplete_matrix: true
-
id: 608
run_id: 904
repo_id: 63
owner_id: 2
commit_sha: 97f29ee599c373c729132a5c46a046978311e0ee
is_fork_pull_request: 0
name: define-matrix
attempt: 0
job_id: define-matrix
task_id: 101
status: 1 # success
runs_on: '["fedora"]'
-
id: 609
run_id: 905
repo_id: 63
owner_id: 2
commit_sha: 97f29ee599c373c729132a5c46a046978311e0ee
is_fork_pull_request: 0
name: job_1
attempt: 0
job_id: job_1
task_id: 0
status: 7 # blocked
runs_on: '["fedora"]'
needs: '["define-matrix"]'
workflow_payload: |
"on":
push:
jobs:
run-tests:
name: run-tests (incomplete matrix)
runs-on: docker
steps:
- run: echo "OK!"
strategy:
matrix:
datacenter: ${{ fromJSON(needs.define-matrix.outputs.datacenters) }}
node: ${{ fromJSON(needs.define-matrix.outputs.node-versions) }}
pg: ${{ fromJSON(needs.define-matrix.outputs.pg_version) }}
incomplete_matrix: true
-
id: 610
run_id: 905
repo_id: 63
owner_id: 2
commit_sha: 97f29ee599c373c729132a5c46a046978311e0ee
is_fork_pull_request: 0
name: define-matrix
attempt: 0
job_id: define-matrix
task_id: 102
status: 1 # success
runs_on: '["fedora"]'
-
id: 611
run_id: 906
repo_id: 63
owner_id: 2
commit_sha: 97f29ee599c373c729132a5c46a046978311e0ee
is_fork_pull_request: 0
name: job_1
attempt: 0
job_id: job_1
task_id: 0
status: 7 # blocked
runs_on: '["fedora"]'
needs: '["define-matrix"]'
workflow_payload: |
"on":
push:
jobs:
run-tests:
name: run-tests (incomplete matrix)
runs-on: docker
steps:
- run: echo "OK!"
strategy:
matrix: ${{ fromJSON(needs.define-matrix.outputs.entire-matrix) }}
incomplete_matrix: true
-
id: 612
run_id: 906
repo_id: 63
owner_id: 2
commit_sha: 97f29ee599c373c729132a5c46a046978311e0ee
is_fork_pull_request: 0
name: define-matrix
attempt: 0
job_id: define-matrix
task_id: 103
status: 1 # success
runs_on: '["fedora"]'
-
id: 613
run_id: 907
repo_id: 63
owner_id: 2
commit_sha: 97f29ee599c373c729132a5c46a046978311e0ee
is_fork_pull_request: 0
name: job_1
attempt: 0
job_id: job_1
task_id: 0
status: 7 # blocked
runs_on: '["fedora"]'
needs: '["define-matrix"]'
workflow_payload: |
"on": [push]
jobs:
scalar-job:
name: scalar-job (incomplete matrix)
runs-on: docker
steps:
- run: |
set -x
[ "${{ matrix.scalar }}" = "scalar value" ] || [ "${{ matrix.scalar }}" = "hard-coded value" ] || exit 1
strategy:
matrix:
scalar:
- "${{ needs.define-matrix.outputs.scalar-value }}"
- hard-coded value
incomplete_matrix: true
-
id: 614
run_id: 907
repo_id: 63
owner_id: 2
commit_sha: 97f29ee599c373c729132a5c46a046978311e0ee
is_fork_pull_request: 0
name: define-matrix
attempt: 0
job_id: define-matrix
task_id: 104
status: 1 # success
runs_on: '["fedora"]'

35
services/actions/Test_tryHandleIncompleteMatrix/action_task_output.yml Normal file
View file

@ -0,0 +1,35 @@
-
id: 100
task_id: 100
output_key: colors
output_value: '["red", "blue", "green"]'
-
id: 101
task_id: 101
output_key: colors
output_value: '[]'
-
id: 102
task_id: 102
output_key: datacenters
output_value: '["site-a", "site-b"]'
-
id: 103
task_id: 102
output_key: node-versions
output_value: '["12.x", "14.x"]'
-
id: 104
task_id: 102
output_key: pg_version
output_value: '[17, 18]'
-
id: 105
task_id: 103
output_key: entire-matrix
output_value: '{"datacenter": ["site-a", "site-b"], "node": ["12.x", "14.x"], "pg": [17, 18]}'
-
id: 106
task_id: 104
output_key: scalar-value
output_value: just some value

7
services/actions/commit_status.go
View file

@ -33,6 +33,13 @@ func CreateCommitStatus(ctx context.Context, jobs ...*actions_model.ActionRunJob
}
func createCommitStatus(ctx context.Context, job *actions_model.ActionRunJob) error {
if incompleteMatrix, err := job.IsIncompleteMatrix(); err != nil {
return fmt.Errorf("job IsIncompleteMatrix: %w", err)
} else if incompleteMatrix {
// Don't create commit statuses for incomplete matrix jobs because they are never completed.
return nil
}
if err := job.LoadAttributes(ctx); err != nil {
return fmt.Errorf("load run: %w", err)
}

37
services/actions/commit_status_test.go Normal file
View file

@ -0,0 +1,37 @@
// Copyright 2025 The Forgejo Authors. All rights reserved.
// SPDX-License-Identifier: GPL-3.0-or-later
package actions
import (
"testing"
actions_model "forgejo.org/models/actions"
"forgejo.org/models/unittest"
"github.com/stretchr/testify/require"
)
func TestCreateCommitStatus_IncompleteMatrix(t *testing.T) {
require.NoError(t, unittest.PrepareTestDatabase())
job := unittest.AssertExistsAndLoadBean(t, &actions_model.ActionRunJob{ID: 192})
// Normally this job will attempt to create a commit status on a commit that doesn't exist in the test repo,
// resulting in an error due to the test fixture data not matching the related repos. But it tried.
err := createCommitStatus(t.Context(), job)
require.ErrorContains(t, err, "object does not exist [id: 7a3858dc7f059543a8807a8b551304b7e362a7ef")
// Transition from IsIncompleteMatrix()=false to true...
isIncomplete, err := job.IsIncompleteMatrix()
require.NoError(t, err)
require.False(t, isIncomplete)
job.WorkflowPayload = append(job.WorkflowPayload, "\nincomplete_matrix: true\n"...)
isIncomplete, err = job.IsIncompleteMatrix()
require.NoError(t, err)
require.True(t, isIncomplete)
// Now there should be no error since createCommitStatus will exit early due to the IsIncompleteMatrix() flag.
err = createCommitStatus(t.Context(), job)
require.NoError(t, err)
}

134
services/actions/job_emitter.go
View file

@ -1,5 +1,6 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
// Copyright 2025 The Forgejo Authors. All rights reserved.
// SPDX-License-Identifier: MIT AND GPL-3.0-or-later
package actions
@ -7,17 +8,23 @@ import (
"context"
"errors"
"fmt"
"slices"
"strings"
actions_model "forgejo.org/models/actions"
"forgejo.org/models/db"
"forgejo.org/modules/graceful"
"forgejo.org/modules/log"
"forgejo.org/modules/queue"
"code.forgejo.org/forgejo/runner/v12/act/jobparser"
"xorm.io/builder"
)
var jobEmitterQueue *queue.WorkerPoolQueue[*jobUpdate]
var (
logger = log.GetManager().GetLogger(log.DEFAULT)
jobEmitterQueue *queue.WorkerPoolQueue[*jobUpdate]
)
type jobUpdate struct {
RunID int64
@ -38,6 +45,7 @@ func jobEmitterQueueHandler(items ...*jobUpdate) []*jobUpdate {
var ret []*jobUpdate
for _, update := range items {
if err := checkJobsOfRun(ctx, update.RunID); err != nil {
logger.Error("checkJobsOfRun failed for RunID = %d: %v", update.RunID, err)
ret = append(ret, update)
}
}
@ -59,6 +67,16 @@ func checkJobsOfRun(ctx context.Context, runID int64) error {
for _, job := range jobs {
if status, ok := updates[job.ID]; ok {
job.Status = status
if status == actions_model.StatusWaiting {
ignore, err := tryHandleIncompleteMatrix(ctx, job, jobs)
if err != nil {
return fmt.Errorf("error in tryHandleIncompleteMatrix: %w", err)
} else if ignore {
continue
}
}
if n, err := UpdateRunJob(ctx, job, builder.Eq{"status": actions_model.StatusBlocked}, "status"); err != nil {
return err
} else if n != 1 {
@ -160,3 +178,115 @@ func (r *jobStatusResolver) resolve() map[int64]actions_model.Status {
}
return ret
}
// Invoked once a job has all its `needs` parameters met and is ready to transition to waiting, this may expand the
// job's `strategy.matrix` into multiple new jobs.
func tryHandleIncompleteMatrix(ctx context.Context, blockedJob *actions_model.ActionRunJob, jobsInRun []*actions_model.ActionRunJob) (bool, error) {
if incompleteMatrix, err := blockedJob.IsIncompleteMatrix(); err != nil {
return false, fmt.Errorf("job IsIncompleteMatrix: %w", err)
} else if !incompleteMatrix {
// Not relevant to attempt expansion if it wasn't marked IncompleteMatrix previously.
return false, nil
}
if err := blockedJob.LoadRun(ctx); err != nil {
return false, fmt.Errorf("failure LoadRun in tryHandleIncompleteMatrix: %w", err)
}
// Compute jobOutputs for all the other jobs required as needed by this job:
jobOutputs := make(map[string]map[string]string, len(jobsInRun))
for _, job := range jobsInRun {
if !slices.Contains(blockedJob.Needs, job.JobID) {
// Only include jobs that are in the `needs` of the blocked job.
continue
} else if !job.Status.IsDone() {
// Unexpected: `job` is needed by `blockedJob` but it isn't done; `jobStatusResolver` shouldn't be calling
// `tryHandleIncompleteMatrix` in this case.
return false, fmt.Errorf(
"jobStatusResolver attempted to tryHandleIncompleteMatrix for a job (id=%d) with an incomplete 'needs' job (id=%d)", blockedJob.ID, job.ID)
}
outputs, err := actions_model.FindTaskOutputByTaskID(ctx, job.TaskID)
if err != nil {
return false, fmt.Errorf("failed loading task outputs: %w", err)
}
outputsMap := make(map[string]string, len(outputs))
for _, v := range outputs {
outputsMap[v.OutputKey] = v.OutputValue
}
jobOutputs[job.JobID] = outputsMap
}
// Re-parse the blocked job, providing all the other completed jobs' outputs, to turn this incomplete matrix into
// one-or-more new jobs:
newJobWorkflows, err := jobparser.Parse(blockedJob.WorkflowPayload, false,
jobparser.WithJobOutputs(jobOutputs),
jobparser.WithWorkflowNeeds(blockedJob.Needs),
)
if err != nil {
return false, fmt.Errorf("failure re-parsing SingleWorkflow: %w", err)
}
// Sanity check that the expanded jobs are !IncompleteMatrix:
for _, swf := range newJobWorkflows {
if swf.IncompleteMatrix {
// Even though every job in the `needs` list is done, this job came back as `IncompleteMatrix`. This could
// happen if the job referenced `needs.some-job` in the `strategy.matrix`, but the job didn't have `needs:
// some-job`, or it could happen if it references an output that doesn't exist on that job. We don't have
// enough information from the jobparser to determine what failed specifically.
//
// This is an error that needs to be reported back to the user for them to correct their workflow, so we
// slip this notification into PreExecutionError.
// This string isn't translated because PreExecutionError needs a design update -- it only stores a single
// string that is displayed to all users irrespective of their language. We could guess at the the language
// based upon the triggering user of the workflow, but it would just be a rough guess, and it would expose a
// user's personal configuration (their language).
run := blockedJob.Run
run.PreExecutionError = fmt.Sprintf(
"Unable to evaluate `strategy.matrix` of job %[1]s due to a `needs` expression that was invalid. It may reference a job that is not in it's 'needs' list (%[2]s), or an output that doesn't exist on one of those jobs.",
blockedJob.JobID,
strings.Join(blockedJob.Needs, ", "),
)
run.Status = actions_model.StatusFailure
err = actions_model.UpdateRunWithoutNotification(ctx, run, "pre_execution_error", "status")
if err != nil {
return false, fmt.Errorf("failure updating PreExecutionError: %w", err)
}
// Mark the job as failed as well so that it doesn't remain sitting "blocked" in the UI
blockedJob.Status = actions_model.StatusFailure
affected, err := UpdateRunJob(ctx, blockedJob, nil, "status")
if err != nil {
return false, fmt.Errorf("failure updating blockedJob.Status=StatusFailure: %w", err)
} else if affected != 1 {
return false, fmt.Errorf("expected 1 row to be updated setting blockedJob.Status=StatusFailure, but was %d", affected)
}
// Return `true` to skip running this job in this invalid state
return true, nil
}
}
err = db.WithTx(ctx, func(ctx context.Context) error {
err := actions_model.InsertRunJobs(ctx, blockedJob.Run, newJobWorkflows)
if err != nil {
return fmt.Errorf("failure in InsertRunJobs: %w", err)
}
// Delete the blocked job which has been expanded into `newJobWorkflows`.
count, err := db.DeleteByID[actions_model.ActionRunJob](ctx, blockedJob.ID)
if err != nil {
return err
} else if count != 1 {
return fmt.Errorf("unexpected record count in delete incomplete_matrix=true job with ID %d; count = %d", blockedJob.ID, count)
}
return nil
})
if err != nil {
return false, err
}
return true, nil
}

143
services/actions/job_emitter_test.go
View file

@ -4,11 +4,15 @@
package actions
import (
"slices"
"testing"
actions_model "forgejo.org/models/actions"
"forgejo.org/models/db"
"forgejo.org/models/unittest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_jobStatusResolver_Resolve(t *testing.T) {
@ -134,3 +138,142 @@ jobs:
})
}
}
func Test_tryHandleIncompleteMatrix(t *testing.T) {
tests := []struct {
name string
runJobID int64
errContains string
consumed bool
runJobNames []string
preExecutionError string
}{
{
name: "not incomplete_matrix",
runJobID: 600,
},
{
name: "matrix expanded to 3 new jobs",
runJobID: 601,
consumed: true,
runJobNames: []string{"define-matrix", "produce-artifacts (blue)", "produce-artifacts (green)", "produce-artifacts (red)"},
},
{
name: "needs an incomplete job",
runJobID: 603,
errContains: "jobStatusResolver attempted to tryHandleIncompleteMatrix for a job (id=603) with an incomplete 'needs' job (id=604)",
},
{
name: "missing needs for strategy.matrix evaluation",
runJobID: 605,
preExecutionError: "Unable to evaluate `strategy.matrix` of job job_1 due to a `needs` expression that was invalid. It may reference a job that is not in it's 'needs' list (define-matrix-1), or an output that doesn't exist on one of those jobs.",
},
{
name: "matrix expanded to 0 jobs",
runJobID: 607,
consumed: true,
runJobNames: []string{"define-matrix"},
},
{
name: "matrix multiple dimensions from separate outputs",
runJobID: 609,
consumed: true,
runJobNames: []string{
"define-matrix",
"run-tests (site-a, 12.x, 17)",
"run-tests (site-a, 12.x, 18)",
"run-tests (site-a, 14.x, 17)",
"run-tests (site-a, 14.x, 18)",
"run-tests (site-b, 12.x, 17)",
"run-tests (site-b, 12.x, 18)",
"run-tests (site-b, 14.x, 17)",
"run-tests (site-b, 14.x, 18)",
},
},
{
name: "matrix multiple dimensions from one output",
runJobID: 611,
consumed: true,
runJobNames: []string{
"define-matrix",
"run-tests (site-a, 12.x, 17)",
"run-tests (site-a, 12.x, 18)",
"run-tests (site-a, 14.x, 17)",
"run-tests (site-a, 14.x, 18)",
"run-tests (site-b, 12.x, 17)",
"run-tests (site-b, 12.x, 18)",
"run-tests (site-b, 14.x, 17)",
"run-tests (site-b, 14.x, 18)",
},
},
{
// This test case also includes `on: [push]` in the workflow_payload, which appears to trigger a regression
// in go.yaml.in/yaml/v4 v4.0.0-rc.2 (which I had accidentally referenced in job_emitter.go), and so serves
// as a regression prevention test for this case...
//
// unmarshal WorkflowPayload to SingleWorkflow failed: yaml: unmarshal errors: line 1: cannot unmarshal
// !!seq into yaml.Node
name: "scalar expansion into matrix",
runJobID: 613,
consumed: true,
runJobNames: []string{
"define-matrix",
"scalar-job (hard-coded value)",
"scalar-job (just some value)",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
defer unittest.OverrideFixtures("services/actions/Test_tryHandleIncompleteMatrix")()
require.NoError(t, unittest.PrepareTestDatabase())
blockedJob := unittest.AssertExistsAndLoadBean(t, &actions_model.ActionRunJob{ID: tt.runJobID})
jobsInRun, err := db.Find[actions_model.ActionRunJob](t.Context(), actions_model.FindRunJobOptions{RunID: blockedJob.RunID})
require.NoError(t, err)
skip, err := tryHandleIncompleteMatrix(t.Context(), blockedJob, jobsInRun)
if tt.errContains != "" {
require.ErrorContains(t, err, tt.errContains)
} else {
require.NoError(t, err)
if tt.consumed {
assert.True(t, skip, "skip flag")
// blockedJob should no longer exist in the database
unittest.AssertNotExistsBean(t, &actions_model.ActionRunJob{ID: tt.runJobID})
// expectations are that the ActionRun has an empty PreExecutionError
actionRun := unittest.AssertExistsAndLoadBean(t, &actions_model.ActionRun{ID: blockedJob.RunID})
assert.Empty(t, actionRun.PreExecutionError)
// compare jobs that exist with `runJobNames` to ensure new jobs are inserted:
allJobsInRun, err := db.Find[actions_model.ActionRunJob](t.Context(), actions_model.FindRunJobOptions{RunID: blockedJob.RunID})
require.NoError(t, err)
allJobNames := []string{}
for _, j := range allJobsInRun {
allJobNames = append(allJobNames, j.Name)
}
slices.Sort(allJobNames)
assert.Equal(t, tt.runJobNames, allJobNames)
} else if tt.preExecutionError != "" {
// expectations are that the ActionRun has a populated PreExecutionError, is marked as failed
actionRun := unittest.AssertExistsAndLoadBean(t, &actions_model.ActionRun{ID: blockedJob.RunID})
assert.Equal(t, tt.preExecutionError, actionRun.PreExecutionError)
assert.Equal(t, actions_model.StatusFailure, actionRun.Status)
// ActionRunJob is marked as failed
blockedJobReloaded := unittest.AssertExistsAndLoadBean(t, &actions_model.ActionRunJob{ID: tt.runJobID})
assert.Equal(t, actions_model.StatusFailure, blockedJobReloaded.Status)
// skip is set to true
assert.True(t, skip, "skip flag")
} else {
assert.False(t, skip, "skip flag")
}
}
})
}
}

7
services/actions/notifier_helper.go
View file

@ -412,7 +412,12 @@ func handleWorkflows(
Name: dwf.EntryName,
}}
} else {
jobs, err = actions_module.JobParser(dwf.Content, jobparser.WithVars(vars))
jobs, err = actions_module.JobParser(dwf.Content,
jobparser.WithVars(vars),
// We don't have any job outputs yet, but `WithJobOutputs(...)` triggers JobParser to supporting its
// `IncompleteMatrix` tagging for any jobs that require the inputs of other jobs.
jobparser.WithJobOutputs(map[string]map[string]string{}),
)
if err != nil {
log.Info("jobparser.Parse: invalid workflow, setting job status to failed: %v", err)
tr := translation.NewLocale(input.Doer.Language)

28
services/actions/notifier_helper_test.go
View file

@ -265,3 +265,31 @@ func TestActionsNotifier_handleWorkflows_setRunTrustForPullRequest(t *testing.T)
assert.Equal(t, pr.ID, run.PullRequestID)
assert.True(t, run.NeedApproval)
}
func TestActionsNotifier_DynamicMatrix(t *testing.T) {
require.NoError(t, unittest.PrepareTestDatabase())
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 10})
pr := unittest.AssertExistsAndLoadBean(t, &issues_model.PullRequest{ID: 3})
dw := &actions_module.DetectedWorkflow{
Content: []byte("{ on: pull_request, jobs: { j1: { strategy: { matrix: { dim1: \"${{ fromJSON(needs.other-job.outputs.some-output) }}\" } } } } }"),
}
testActionsNotifierPullRequest(t, repo, pr, dw, webhook_module.HookEventPullRequestSync)
runs, err := db.Find[actions_model.ActionRun](db.DefaultContext, actions_model.FindRunOptions{
RepoID: repo.ID,
})
require.NoError(t, err)
require.Len(t, runs, 1)
run := runs[0]
jobs, err := db.Find[actions_model.ActionRunJob](t.Context(), actions_model.FindRunJobOptions{RunID: run.ID})
require.NoError(t, err)
require.Len(t, jobs, 1)
job := jobs[0]
// With a matrix that contains ${{ needs ... }} references, the only requirement to work is that when the job is
// first inserted it is tagged w/ incomplete_matrix
assert.Contains(t, string(job.WorkflowPayload), "incomplete_matrix: true")
}

7
services/actions/schedule_tasks.go
View file

@ -169,7 +169,12 @@ func CreateScheduleTask(ctx context.Context, cron *actions_model.ActionSchedule)
}
// Parse the workflow specification from the cron schedule
workflows, err := actions_module.JobParser(cron.Content, jobparser.WithVars(vars))
workflows, err := actions_module.JobParser(cron.Content,
jobparser.WithVars(vars),
// We don't have any job outputs yet, but `WithJobOutputs(...)` triggers JobParser to supporting its
// `IncompleteMatrix` tagging for any jobs that require the inputs of other jobs.
jobparser.WithJobOutputs(map[string]map[string]string{}),
)
if err != nil {
return err
}

62
services/actions/schedule_tasks_test.go
View file

@ -264,3 +264,65 @@ func TestCancelPreviousWithConcurrencyGroup(t *testing.T) {
})
}
}
func TestServiceActions_DynamicMatrix(t *testing.T) {
defer unittest.OverrideFixtures("services/actions/TestServiceActions_startTask")()
require.NoError(t, unittest.PrepareTestDatabase())
// Load fixtures that are corrupted and create one valid scheduled workflow
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 4})
workflowID := "some.yml"
schedules := []*actions_model.ActionSchedule{
{
Title: "scheduletitle1",
RepoID: repo.ID,
OwnerID: repo.OwnerID,
WorkflowID: workflowID,
TriggerUserID: repo.OwnerID,
Ref: "branch",
CommitSHA: "fakeSHA",
Event: webhook_module.HookEventSchedule,
EventPayload: "fakepayload",
Specs: []string{"* * * * *"},
Content: []byte(
`
jobs:
job2:
runs-on: ubuntu-latest
strategy:
matrix:
dim1: "${{ fromJSON(needs.other-job.outputs.some-output) }}"
steps:
- run: true
`),
},
}
require.Equal(t, 2, unittest.GetCount(t, actions_model.ActionScheduleSpec{}))
require.NoError(t, actions_model.CreateScheduleTask(t.Context(), schedules))
require.Equal(t, 3, unittest.GetCount(t, actions_model.ActionScheduleSpec{}))
_, err := db.GetEngine(db.DefaultContext).Exec("UPDATE `action_schedule_spec` SET next = 1")
require.NoError(t, err)
// After running startTasks an ActionRun row is created for the valid scheduled workflow
require.Empty(t, unittest.GetCount(t, actions_model.ActionRun{WorkflowID: workflowID}))
require.NoError(t, startTasks(t.Context()))
require.NotEmpty(t, unittest.GetCount(t, actions_model.ActionRun{WorkflowID: workflowID}))
runs, err := db.Find[actions_model.ActionRun](db.DefaultContext, actions_model.FindRunOptions{
WorkflowID: workflowID,
})
require.NoError(t, err)
require.Len(t, runs, 1)
run := runs[0]
jobs, err := db.Find[actions_model.ActionRunJob](t.Context(), actions_model.FindRunJobOptions{RunID: run.ID})
require.NoError(t, err)
require.Len(t, jobs, 1)
job := jobs[0]
// With a matrix that contains ${{ needs ... }} references, the only requirement to work is that when the job is
// first inserted it is tagged w/ incomplete_matrix
assert.Contains(t, string(job.WorkflowPayload), "incomplete_matrix: true")
}

8
services/actions/workflows.go
View file

@ -168,7 +168,13 @@ func (entry *Workflow) Dispatch(ctx context.Context, inputGetter InputValueGette
}
}
jobs, err := actions.JobParser(content, jobparser.WithVars(vars), jobparser.WithInputs(inputsAny))
jobs, err := actions.JobParser(content,
jobparser.WithVars(vars),
jobparser.WithInputs(inputsAny),
// We don't have any job outputs yet, but `WithJobOutputs(...)` triggers JobParser to supporting its
// `IncompleteMatrix` tagging for any jobs that require the inputs of other jobs.
jobparser.WithJobOutputs(map[string]map[string]string{}),
)
if err != nil {
return nil, nil, err
}