✨ (ci): Materialize workflow runs on push
Changes
7 files changed, +465 -7
MODIFY
docs/maintainers/ci-runners.md
+15 -7
@@ -13,7 +13,8 @@
13
13
| Connect endpoint | `ci/ConnectRunnerResource.java` | JAX-RS resource serving the Connect unary RPCs under `/api/actions`. |
14
14
| Registration/presence | `ci/RunnerRegistrationService.java` | Token issue, runner register/declare/authenticate, list/delete. |
15
15
| Entities | `model/CiRunner.java`, `model/CiRunnerRegistrationToken.java` | Runner state (migration `V19`). |
16
-| Run entities | `model/ActionRun.java`, `model/ActionTask.java`, `model/ActionLog.java` | Run/job/log-row persistence (migration `V23`). Not yet wired to a run loop. |
16
+| Run entities | `model/ActionRun.java`, `model/ActionTask.java`, `model/ActionLog.java` | Run/job/log-row persistence (migration `V23`). |
17
+| Workflow ingest | `ci/WorkflowIngestService.java`, `ci/WorkflowRunFactory.java` | Post-receive hook: parse `.forgejo`/`.gitea` workflows at the pushed head, evaluate `on: push`, persist a run + its tasks. Rows are created but not yet dispatched. |
17
18
| Admin UI | `ci/AdminRunnerResource.java` + `templates/AdminRunnerResource/` | Token generation, runner list, deletion. |
18
19
| Admin gate | `account/AdminAccess.java` | Config-driven instance-admin check. |
19
20
@@ -50,18 +51,25 @@
50
51
`/admin/*` authenticated policy.
51
52
- **Run-persistence tables:** `action_run`, `action_task`, `action_log` (migration `V23`) with their
52
53
Panache entities. `action_task.log_length` is the durable log-row count that doubles as the
53
- UpdateLog resume/ack offset; `action_task.deadline` is the zombie-timeout anchor. Schema and
54
- entities only — nothing writes to them yet.
54
+ UpdateLog resume/ack offset; `action_task.deadline` is the zombie-timeout anchor.
55
+- **Workflow ingest on push:** the post-receive hooks (HTTP + SSH) call `WorkflowIngestService`,
56
+ which reads `.forgejo/workflows/*.{yml,yaml}` and `.gitea/workflows/*` at the new commit of each
57
+ updated branch, parses them (Jackson `YAMLMapper`), and for those triggered by `push` persists one
58
+ `action_run` (per-repo `number`, PENDING) with one PENDING `action_task` per job via
59
+ `WorkflowRunFactory` (`@Transactional`). Handles the YAML-1.1 `on:`→boolean-`true` key coercion.
55
60
- Tests: `RunnerRegistrationServiceTest` (service), `ConnectRunnerResourceTest` (protobuf-over-HTTP
56
61
round-trip for Ping/Register/Declare + auth failures), `AdminAccessTest` (admin gate),
57
- `ActionRunPersistenceTest` (run/task/log persistence, per-repo run numbering, pending-task lookup).
62
+ `ActionRunPersistenceTest` (run/task/log persistence, per-repo run numbering, pending-task lookup),
63
+ `WorkflowIngestServiceTest` (push → run/task creation, non-push trigger and no-workflow are no-ops).
58
64
59
65
## What still needs to be implemented
60
66
61
67
- **Run loop:** `FetchTask` (long-poll with `tasks_version`), `UpdateTask`, `UpdateLog` (offset /
62
- `ack_index` resume). Not served yet — the tables above are the foundation for it.
63
-- **Workflow pipeline:** parse `.forgejo/workflows/*.yml` at the pushed head, evaluate `on:` triggers
64
- (push only for the MVP), expand a single job into a Task payload. No `needs`/`matrix` yet.
68
+ `ack_index` resume). Not served yet — the ingested PENDING tasks are the queue it will drain.
69
+- **Per-job payload expansion:** ingest stores the raw workflow YAML in `action_task.payload`;
70
+ `Task.workflow_payload` needs the single job isolated/expanded. No `needs`/`matrix` yet.
71
+- **Trigger refinement:** only bare `on: push` is honored; branch/tag/path filters and other events
72
+ (tag push, `pull_request`) are not evaluated.
65
73
- **Run UI:** per-repository run list + run detail with live per-step status and logs.
66
74
- **Task state machine:** timeout / zombie handling when a runner vanishes mid-task.
67
75
- **Real-runner integration test:** protocol round-trip against an actual `forgejo-runner` container
MODIFY
pom.xml
+5 -0
@@ -64,6 +64,11 @@
64
64
<artifactId>quarkus-rest-jackson</artifactId>
65
65
</dependency>
66
66
<dependency>
67
+ <!-- Parsing .forgejo/.gitea workflow YAML (CI ingest); otherwise only transitive via OpenAPI. -->
68
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
69
+ <artifactId>jackson-dataformat-yaml</artifactId>
70
+ </dependency>
71
+ <dependency>
67
72
<groupId>io.quarkiverse.mcp</groupId>
68
73
<artifactId>quarkus-mcp-server-http</artifactId>
69
74
<version>${quarkus-mcp-server.version}</version>
ADD
src/main/java/de/workaround/ci/WorkflowIngestService.java
+234 -0
@@ -0,0 +1,234 @@
1
+package de.workaround.ci;
2
+
3
+import java.nio.charset.StandardCharsets;
4
+import java.util.ArrayList;
5
+import java.util.Iterator;
6
+import java.util.List;
7
+import java.util.UUID;
8
+
9
+import org.eclipse.jgit.errors.LargeObjectException;
10
+import org.eclipse.jgit.lib.Constants;
11
+import org.eclipse.jgit.lib.ObjectId;
12
+import org.eclipse.jgit.revwalk.RevCommit;
13
+import org.eclipse.jgit.revwalk.RevWalk;
14
+import org.eclipse.jgit.transport.ReceiveCommand;
15
+import org.eclipse.jgit.treewalk.TreeWalk;
16
+import org.eclipse.jgit.treewalk.filter.PathFilter;
17
+import org.jboss.logging.Logger;
18
+
19
+import com.fasterxml.jackson.databind.JsonNode;
20
+import com.fasterxml.jackson.dataformat.yaml.YAMLMapper;
21
+
22
+import de.workaround.git.GitRepositoryService;
23
+import de.workaround.model.Repository;
24
+import io.quarkus.arc.Arc;
25
+import jakarta.enterprise.context.ApplicationScoped;
26
+import jakarta.inject.Inject;
27
+
28
+/**
29
+ * Detects workflow files pushed to a repository and materializes CI runs (issue #2, phase 1). For
30
+ * every branch update, workflow files under {@code .forgejo/workflows/} and {@code .gitea/workflows/}
31
+ * at the new commit are parsed; those triggered by {@code push} produce one {@link
32
+ * de.workaround.model.ActionRun} with one {@link de.workaround.model.ActionTask} per job.
33
+ *
34
+ * <p>Phase-1 scope: {@code on: push} only (branch/tag/path filters, other events, {@code needs} and
35
+ * {@code matrix} are phase 2). Invoked from the transports' post-receive hooks on a Git worker thread
36
+ * with no CDI request context, so it activates one and never throws into the Git path.
37
+ */
38
+@ApplicationScoped
39
+public class WorkflowIngestService
40
+{
41
+ private static final Logger LOG = Logger.getLogger(WorkflowIngestService.class);
42
+
43
+ private static final List<String> WORKFLOW_DIRS = List.of(".forgejo/workflows", ".gitea/workflows");
44
+
45
+ private static final int MAX_WORKFLOW_BYTES = 512 * 1024;
46
+
47
+ private static final YAMLMapper YAML = new YAMLMapper();
48
+
49
+ @Inject
50
+ GitRepositoryService repositories;
51
+
52
+ @Inject
53
+ WorkflowRunFactory factory;
54
+
55
+ /** Entry point from the transports' post-receive hooks. */
56
+ public void onPush(String ownerName, String repoName, UUID pusherUserId,
57
+ org.eclipse.jgit.lib.Repository db, java.util.Collection<ReceiveCommand> commands)
58
+ {
59
+ var requestContext = Arc.container().requestContext();
60
+ boolean activated = !requestContext.isActive();
61
+ if (activated)
62
+ {
63
+ requestContext.activate();
64
+ }
65
+ try
66
+ {
67
+ ingest(ownerName, repoName, pusherUserId, db, commands);
68
+ }
69
+ catch (RuntimeException e)
70
+ {
71
+ LOG.warnf(e, "Failed to ingest workflows from pushed commits for %s/%s", ownerName, repoName);
72
+ }
73
+ finally
74
+ {
75
+ if (activated)
76
+ {
77
+ requestContext.terminate();
78
+ }
79
+ }
80
+ }
81
+
82
+ private void ingest(String ownerName, String repoName, UUID pusherUserId,
83
+ org.eclipse.jgit.lib.Repository db, java.util.Collection<ReceiveCommand> commands)
84
+ {
85
+ Repository repo = repositories.find(ownerName, repoName).orElse(null);
86
+ if (repo == null)
87
+ {
88
+ return;
89
+ }
90
+ for (ReceiveCommand command : commands)
91
+ {
92
+ if (command.getResult() != ReceiveCommand.Result.OK
93
+ || !command.getRefName().startsWith("refs/heads/")
94
+ || command.getType() == ReceiveCommand.Type.DELETE)
95
+ {
96
+ continue;
97
+ }
98
+ for (WorkflowFile workflow : readWorkflows(db, command.getNewId()))
99
+ {
100
+ JsonNode root = parse(workflow.content());
101
+ if (root == null || !triggeredByPush(root))
102
+ {
103
+ continue;
104
+ }
105
+ List<String> jobs = jobNames(root);
106
+ if (jobs.isEmpty())
107
+ {
108
+ continue;
109
+ }
110
+ factory.create(repo, pusherUserId, command.getRefName(), command.getNewId().name(),
111
+ workflowName(root, workflow.path()), workflow.path(), jobs, workflow.content());
112
+ }
113
+ }
114
+ }
115
+
116
+ private static List<WorkflowFile> readWorkflows(org.eclipse.jgit.lib.Repository db, ObjectId commitId)
117
+ {
118
+ List<WorkflowFile> out = new ArrayList<>();
119
+ try (RevWalk walk = new RevWalk(db))
120
+ {
121
+ RevCommit commit = walk.parseCommit(commitId);
122
+ for (String dir : WORKFLOW_DIRS)
123
+ {
124
+ collect(db, commit, dir, out);
125
+ }
126
+ }
127
+ catch (Exception e)
128
+ {
129
+ // best-effort: reading the pushed tree failed, so no runs are created from this push
130
+ LOG.debugf(e, "Could not read workflow files at %s", commitId);
131
+ }
132
+ return out;
133
+ }
134
+
135
+ private static void collect(org.eclipse.jgit.lib.Repository db, RevCommit commit, String dir,
136
+ List<WorkflowFile> out) throws Exception
137
+ {
138
+ try (TreeWalk walk = new TreeWalk(db))
139
+ {
140
+ walk.addTree(commit.getTree());
141
+ walk.setRecursive(true);
142
+ walk.setFilter(PathFilter.create(dir));
143
+ while (walk.next())
144
+ {
145
+ String path = walk.getPathString();
146
+ if (!path.endsWith(".yml") && !path.endsWith(".yaml"))
147
+ {
148
+ continue;
149
+ }
150
+ try
151
+ {
152
+ byte[] content = db.open(walk.getObjectId(0), Constants.OBJ_BLOB).getCachedBytes(MAX_WORKFLOW_BYTES);
153
+ out.add(new WorkflowFile(path, new String(content, StandardCharsets.UTF_8)));
154
+ }
155
+ catch (LargeObjectException tooBig)
156
+ {
157
+ LOG.debugf("Skipping oversized workflow file %s", path);
158
+ }
159
+ }
160
+ }
161
+ }
162
+
163
+ private static JsonNode parse(String yaml)
164
+ {
165
+ try
166
+ {
167
+ return YAML.readTree(yaml);
168
+ }
169
+ catch (Exception malformed)
170
+ {
171
+ LOG.debugf(malformed, "Skipping unparseable workflow file");
172
+ return null;
173
+ }
174
+ }
175
+
176
+ /**
177
+ * Reads the {@code on:} trigger. YAML 1.1 coerces the bare key {@code on} to boolean true, so
178
+ * SnakeYAML/Jackson may surface it under the field name {@code "true"} — both keys are checked.
179
+ */
180
+ static boolean triggeredByPush(JsonNode root)
181
+ {
182
+ JsonNode on = root.has("on") ? root.get("on") : root.get("true");
183
+ if (on == null)
184
+ {
185
+ return false;
186
+ }
187
+ if (on.isTextual())
188
+ {
189
+ return "push".equals(on.asText());
190
+ }
191
+ if (on.isArray())
192
+ {
193
+ for (JsonNode event : on)
194
+ {
195
+ if (event.isTextual() && "push".equals(event.asText()))
196
+ {
197
+ return true;
198
+ }
199
+ }
200
+ return false;
201
+ }
202
+ return on.isObject() && on.has("push");
203
+ }
204
+
205
+ private static List<String> jobNames(JsonNode root)
206
+ {
207
+ List<String> names = new ArrayList<>();
208
+ JsonNode jobs = root.get("jobs");
209
+ if (jobs != null && jobs.isObject())
210
+ {
211
+ for (Iterator<String> it = jobs.fieldNames(); it.hasNext();)
212
+ {
213
+ names.add(it.next());
214
+ }
215
+ }
216
+ return names;
217
+ }
218
+
219
+ private static String workflowName(JsonNode root, String path)
220
+ {
221
+ JsonNode name = root.get("name");
222
+ if (name != null && name.isTextual() && !name.asText().isBlank())
223
+ {
224
+ return name.asText();
225
+ }
226
+ int slash = path.lastIndexOf('/');
227
+ return slash >= 0 ? path.substring(slash + 1) : path;
228
+ }
229
+
230
+ private record WorkflowFile(String path, String content)
231
+ {
232
+ }
233
+
234
+}
ADD
src/main/java/de/workaround/ci/WorkflowRunFactory.java
+64 -0
@@ -0,0 +1,64 @@
1
+package de.workaround.ci;
2
+
3
+import java.util.List;
4
+import java.util.UUID;
5
+
6
+import de.workaround.model.ActionRun;
7
+import de.workaround.model.ActionTask;
8
+import de.workaround.model.Repository;
9
+import de.workaround.model.User;
10
+import jakarta.enterprise.context.ApplicationScoped;
11
+import jakarta.inject.Inject;
12
+import jakarta.transaction.Transactional;
13
+
14
+/**
15
+ * Persists a workflow run and its jobs in a single transaction (issue #2, phase 1). Split out from
16
+ * {@link WorkflowIngestService} so the transaction boundary is a proper CDI-proxied call: the ingest
17
+ * orchestration (git reads, trigger evaluation, error-swallowing) stays outside any transaction, and
18
+ * each run is committed atomically here. The per-repository run {@link ActionRun#number} is allocated
19
+ * as {@code max(number) + 1} inside the transaction, mirroring {@link de.workaround.model.Issue}.
20
+ */
21
+@ApplicationScoped
22
+public class WorkflowRunFactory
23
+{
24
+ @Inject
25
+ Repository.Repo repositories;
26
+
27
+ @Inject
28
+ User.Repo users;
29
+
30
+ @Inject
31
+ ActionRun.Repo runs;
32
+
33
+ @Inject
34
+ ActionTask.Repo tasks;
35
+
36
+ @Transactional
37
+ public ActionRun create(Repository repository, UUID pusherUserId, String ref, String commitSha,
38
+ String workflowName, String workflowFile, List<String> jobNames, String payload)
39
+ {
40
+ Repository repo = repositories.findById(repository.id);
41
+
42
+ ActionRun run = new ActionRun();
43
+ run.repository = repo;
44
+ run.number = runs.maxNumber(repo) + 1;
45
+ run.workflowName = workflowName;
46
+ run.workflowFile = workflowFile;
47
+ run.event = "push";
48
+ run.ref = ref;
49
+ run.commitSha = commitSha;
50
+ run.triggeredBy = pusherUserId == null ? null : users.findById(pusherUserId);
51
+ run.persist();
52
+
53
+ for (String jobName : jobNames)
54
+ {
55
+ ActionTask task = new ActionTask();
56
+ task.run = run;
57
+ task.name = jobName;
58
+ task.payload = payload;
59
+ task.persist();
60
+ }
61
+ return run;
62
+ }
63
+
64
+}
MODIFY
src/main/java/de/workaround/http/GitHttpServlet.java
+4 -0
@@ -47,6 +47,9 @@
47
47
IssueCommitCloser issueCloser;
48
48
49
49
@Inject
50
+ de.workaround.ci.WorkflowIngestService workflowIngest;
51
+
52
+ @Inject
50
53
de.workaround.mirror.MirrorService mirrorService;
51
54
52
55
@Override
@@ -115,6 +118,7 @@
115
118
receivePack.setPostReceiveHook((rp, commands) -> {
116
119
pushService.onPush(ownerName, repoName, pusherId, rp.getRepository(), commands);
117
120
issueCloser.onPush(ownerName, repoName, pusherId, rp.getRepository(), commands);
121
+ workflowIngest.onPush(ownerName, repoName, pusherId, rp.getRepository(), commands);
118
122
mirrorService.onPush(ownerName, repoName, commands);
119
123
});
120
124
return receivePack;
MODIFY
src/main/java/de/workaround/ssh/GitSshCommandFactory.java
+4 -0
@@ -37,6 +37,9 @@
37
37
de.workaround.git.IssueCommitCloser issueCloser;
38
38
39
39
@Inject
40
+ de.workaround.ci.WorkflowIngestService workflowIngest;
41
+
42
+ @Inject
40
43
de.workaround.mirror.MirrorService mirrorService;
41
44
42
45
@Override
@@ -151,6 +154,7 @@
151
154
receivePack.setPostReceiveHook((rp, commands) -> {
152
155
pushService.onPush(ownerName, repoName, userId, rp.getRepository(), commands);
153
156
issueCloser.onPush(ownerName, repoName, userId, rp.getRepository(), commands);
157
+ workflowIngest.onPush(ownerName, repoName, userId, rp.getRepository(), commands);
154
158
mirrorService.onPush(ownerName, repoName, commands);
155
159
});
156
160
}
ADD
src/test/java/de/workaround/ci/WorkflowIngestServiceTest.java
+139 -0
@@ -0,0 +1,139 @@
1
+package de.workaround.ci;
2
+
3
+import java.nio.charset.StandardCharsets;
4
+import java.nio.file.Path;
5
+import java.util.List;
6
+import java.util.Map;
7
+import java.util.UUID;
8
+
9
+import org.eclipse.jgit.lib.ObjectId;
10
+import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
11
+import org.eclipse.jgit.transport.ReceiveCommand;
12
+import org.junit.jupiter.api.Test;
13
+
14
+import de.workaround.git.GitRepositoryService;
15
+import de.workaround.git.GitTestSeeder;
16
+import de.workaround.model.ActionRun;
17
+import de.workaround.model.ActionTask;
18
+import de.workaround.model.Repository;
19
+import de.workaround.model.User;
20
+import io.quarkus.test.junit.QuarkusTest;
21
+import jakarta.inject.Inject;
22
+import jakarta.transaction.Transactional;
23
+
24
+import static org.junit.jupiter.api.Assertions.assertEquals;
25
+import static org.junit.jupiter.api.Assertions.assertTrue;
26
+
27
+/**
28
+ * Workflow ingestion on push (issue #2, phase 1): a push carrying a workflow with an {@code on: push}
29
+ * trigger materializes one {@link ActionRun} and its {@link ActionTask}s; unrelated triggers and
30
+ * pushes without workflows create nothing.
31
+ */
32
+@QuarkusTest
33
+class WorkflowIngestServiceTest
34
+{
35
+ @Inject
36
+ WorkflowIngestService ingest;
37
+
38
+ @Inject
39
+ GitRepositoryService repositories;
40
+
41
+ @Inject
42
+ ActionRun.Repo runs;
43
+
44
+ @Inject
45
+ ActionTask.Repo tasks;
46
+
47
+ @Test
48
+ void pushWithOnPushWorkflowCreatesRunAndTask() throws Exception
49
+ {
50
+ User owner = persistUser("wf-alice-" + UUID.randomUUID().toString().substring(0, 8));
51
+ Repository repo = repositories.create(owner, "wf", Repository.Visibility.PUBLIC, null);
52
+
53
+ String yaml = """
54
+ name: CI
55
+ on: push
56
+ jobs:
57
+ build:
58
+ runs-on: ubuntu-latest
59
+ steps:
60
+ - run: echo hi
61
+ """;
62
+ ObjectId head = pushWorkflows(repo, Map.of(".forgejo/workflows/ci.yml", yaml));
63
+
64
+ List<ActionRun> created = runs.findByRepository(repo);
65
+ assertEquals(1, created.size());
66
+ ActionRun run = created.get(0);
67
+ assertEquals(1, run.number);
68
+ assertEquals("push", run.event);
69
+ assertEquals("refs/heads/main", run.ref);
70
+ assertEquals(head.name(), run.commitSha);
71
+ assertEquals("CI", run.workflowName);
72
+ assertEquals(".forgejo/workflows/ci.yml", run.workflowFile);
73
+ assertEquals(ActionRun.Status.PENDING, run.status);
74
+ assertEquals(owner.id, run.triggeredBy.id);
75
+
76
+ List<ActionTask> jobs = tasks.findByRun(run);
77
+ assertEquals(1, jobs.size());
78
+ assertEquals("build", jobs.get(0).name);
79
+ assertEquals(ActionRun.Status.PENDING, jobs.get(0).status);
80
+ }
81
+
82
+ @Test
83
+ void pushWithNonPushTriggerCreatesNothing() throws Exception
84
+ {
85
+ User owner = persistUser("wf-bob-" + UUID.randomUUID().toString().substring(0, 8));
86
+ Repository repo = repositories.create(owner, "wf", Repository.Visibility.PUBLIC, null);
87
+
88
+ String yaml = """
89
+ name: PR only
90
+ on: pull_request
91
+ jobs:
92
+ test:
93
+ runs-on: ubuntu-latest
94
+ """;
95
+ pushWorkflows(repo, Map.of(".forgejo/workflows/pr.yml", yaml));
96
+
97
+ assertTrue(runs.findByRepository(repo).isEmpty());
98
+ }
99
+
100
+ @Test
101
+ void pushWithoutWorkflowsCreatesNothing() throws Exception
102
+ {
103
+ User owner = persistUser("wf-carol-" + UUID.randomUUID().toString().substring(0, 8));
104
+ Repository repo = repositories.create(owner, "wf", Repository.Visibility.PUBLIC, null);
105
+
106
+ pushWorkflows(repo, Map.of("README.md", "# hi\n"));
107
+
108
+ assertTrue(runs.findByRepository(repo).isEmpty());
109
+ }
110
+
111
+ private ObjectId pushWorkflows(Repository repo, Map<String, String> files) throws Exception
112
+ {
113
+ Path bare = repositories.repositoryPath(repo);
114
+ Map<String, byte[]> bytes = files.entrySet().stream()
115
+ .collect(java.util.stream.Collectors.toMap(Map.Entry::getKey,
116
+ e -> e.getValue().getBytes(StandardCharsets.UTF_8)));
117
+ GitTestSeeder.seed(bare, bytes);
118
+
119
+ try (org.eclipse.jgit.lib.Repository db = new FileRepositoryBuilder().setGitDir(bare.toFile()).build())
120
+ {
121
+ ObjectId head = db.resolve("refs/heads/main");
122
+ ReceiveCommand command = new ReceiveCommand(ObjectId.zeroId(), head, "refs/heads/main");
123
+ command.setResult(ReceiveCommand.Result.OK);
124
+ ingest.onPush(repo.ownerHandle(), repo.name, repo.ownerUser.id, db, List.of(command));
125
+ return head;
126
+ }
127
+ }
128
+
129
+ @Transactional
130
+ User persistUser(String name)
131
+ {
132
+ User user = new User();
133
+ user.oidcSub = name;
134
+ user.username = name;
135
+ user.persist();
136
+ return user;
137
+ }
138
+
139
+}