feat: expose lastRetry.exitCodes (all previous attempt exit codes)

.features/pending/last-retry-exit-codes.md

@@ -0,0 +1,8 @@
+Description: Add lastRetry.exitCodes variable exposing all previous attempt exit codes
+Authors: [Liron Shabtai](https://github.com/lirons-legit)
+Component: General
+Issues: 12849
+
+`lastRetry.exitCodes` is a comma-separated list of the exit codes of all previous retry attempts, oldest first, and is empty on the first attempt.
+Unlike `lastRetry.exitCode`, which reports only the immediately previous attempt, it lets an expression accumulate a resource across retries conditionally.
+For example, a `podSpecPatch` memory request of `{{= 2 + len(filter(split(lastRetry.exitCodes, ','), {# == '137'})) }}Gi` grows a pod's memory once per previous OOM (exit code 137) and holds it across non-OOM (eviction) retries, which the previous single-attempt variable could not express.

docs/variable-flow/variables.md

@@ -1,6 +1,6 @@
 # Workflow variables catalog
 
-Auto-generated from `util/variables` via `GenerateMarkdown()`. 83 variables registered.
+Auto-generated from `util/variables` via `GenerateMarkdown()`. 84 variables registered.
 
 **Skipped and omitted nodes:** when a step or task is skipped (its `when` evaluates false) or omitted (its dependencies never ran), it produces no real outputs. Its `outputs.parameters.<name>`, `outputs.result` and `outputs.artifacts.<name>` variables are still populated with empty placeholder values, so downstream references resolve to empty rather than leaving the workflow stuck on an unresolvable variable.
 
@@ -29,6 +29,7 @@ A runnable [variables-showcase.yaml](https://raw.githubusercontent.com/argoproj/
 | `item.<key>`                              | item          | string         | inside-loop                                                | Accessor into a map-typed loop iteration value                                                                                                                                                                                                                                                                                                                    |
 | `lastRetry.duration`                      | retry         | string         | inside-retry                                               | Duration of the previous attempt in seconds                                                                                                                                                                                                                                                                                                                       |
 | `lastRetry.exitCode`                      | retry         | string         | inside-retry                                               | Exit code of the previous attempt (or 0 on first attempt)                                                                                                                                                                                                                                                                                                         |
+| `lastRetry.exitCodes`                     | retry         | string         | inside-retry                                               | Comma-separated exit codes of all previous attempts, oldest first (empty on the first attempt)                                                                                                                                                                                                                                                                    |
 | `lastRetry.message`                       | retry         | string         | inside-retry                                               | Message of the previous attempt                                                                                                                                                                                                                                                                                                                                   |
 | `lastRetry.status`                        | retry         | string         | inside-retry                                               | Phase of the previous attempt (or empty on first)                                                                                                                                                                                                                                                                                                                 |
 | `node.name`                               | node-ctx      | string         | pre-dispatch, during-execute                               | Full node name                                                                                                                                                                                                                                                                                                                                                    |
@@ -187,13 +188,14 @@ A runnable [variables-showcase.yaml](https://raw.githubusercontent.com/argoproj/
 
 ### Retry
 
-|         Key          |  Type  | Availability |                        Description                        |
-|----------------------|--------|--------------|-----------------------------------------------------------|
-| `lastRetry.duration` | string | inside-retry | Duration of the previous attempt in seconds               |
-| `lastRetry.exitCode` | string | inside-retry | Exit code of the previous attempt (or 0 on first attempt) |
-| `lastRetry.message`  | string | inside-retry | Message of the previous attempt                           |
-| `lastRetry.status`   | string | inside-retry | Phase of the previous attempt (or empty on first)         |
-| `retries`            | string | inside-retry | 0-based retry attempt index                               |
+|          Key          |  Type  | Availability |                                          Description                                           |
+|-----------------------|--------|--------------|------------------------------------------------------------------------------------------------|
+| `lastRetry.duration`  | string | inside-retry | Duration of the previous attempt in seconds                                                    |
+| `lastRetry.exitCode`  | string | inside-retry | Exit code of the previous attempt (or 0 on first attempt)                                      |
+| `lastRetry.exitCodes` | string | inside-retry | Comma-separated exit codes of all previous attempts, oldest first (empty on the first attempt) |
+| `lastRetry.message`   | string | inside-retry | Message of the previous attempt                                                                |
+| `lastRetry.status`    | string | inside-retry | Phase of the previous attempt (or empty on first)                                              |
+| `retries`             | string | inside-retry | 0-based retry attempt index                                                                    |
 
 ### Node-ctx
 
@@ -254,6 +256,7 @@ Which variables are in scope for each template type. `•` = in scope, blank = n
 | `item.<key>`                              | •   | •         | •             | •      | •        | •     | •   | •    | •       | •    | •      |              |               |
 | `lastRetry.duration`                      |     | •         | •             | •      | •        | •     | •   | •    |         | •    | •      |              |               |
 | `lastRetry.exitCode`                      |     | •         | •             | •      | •        | •     | •   | •    |         | •    | •      |              |               |
+| `lastRetry.exitCodes`                     |     | •         | •             | •      | •        | •     | •   | •    |         | •    | •      |              |               |
 | `lastRetry.message`                       |     | •         | •             | •      | •        | •     | •   | •    |         | •    | •      |              |               |
 | `lastRetry.status`                        |     | •         | •             | •      | •        | •     | •   | •    |         | •    | •      |              |               |
 | `node.name`                               | •   | •         | •             | •      | •        | •     | •   | •    | •       | •    | •      | •            |               |
@@ -439,15 +442,16 @@ Which variables are in scope for each template type. `•` = in scope, blank = n
 | `item`       | item | string or json |
 | `item.<key>` | item | string         |
 
-### inside-retry (5 variables)
+### inside-retry (6 variables)
 
-|         Key          | Kind  |  Type  |
-|----------------------|-------|--------|
-| `lastRetry.duration` | retry | string |
-| `lastRetry.exitCode` | retry | string |
-| `lastRetry.message`  | retry | string |
-| `lastRetry.status`   | retry | string |
-| `retries`            | retry | string |
+|          Key          | Kind  |  Type  |
+|-----------------------|-------|--------|
+| `lastRetry.duration`  | retry | string |
+| `lastRetry.exitCode`  | retry | string |
+| `lastRetry.exitCodes` | retry | string |
+| `lastRetry.message`   | retry | string |
+| `lastRetry.status`    | retry | string |
+| `retries`             | retry | string |
 
 ### after-node-init (6 variables)
 

docs/variables.md

@@ -240,9 +240,22 @@ When using the `expression` field within `retryStrategy`, special variables are
 | `lastRetry.status` | Status of the last retry |
 | `lastRetry.duration` | Duration in seconds of the last retry |
 | `lastRetry.message` | Message output from the last retry (available from version 3.5) |
+| `lastRetry.exitCodes` | Comma-separated exit codes of all previous attempts, oldest first (empty on the first attempt) |
 
 Note: These variables evaluate to a string type. If using advanced expressions, either cast them to int values (`expression: "{{=asInt(lastRetry.exitCode) >= 2}}"`) or compare them to string values (`expression: "{{=lastRetry.exitCode != '2'}}"`).
 
+`lastRetry.exitCodes` carries the full history so an expression can escalate a resource cumulatively per failure type — e.g. grow memory once per prior OOM and hold it across non-OOM (eviction) retries, which `lastRetry.exitCode` (the last attempt alone) cannot express:
+
+```yaml
+podSpecPatch: |
+  containers:
+    - name: main
+      resources:
+        requests:
+          # 2Gi base + 1Gi per previous OOM (exit 137); held across non-OOM retries
+          memory: "{{=2 + (len(filter(split(lastRetry.exitCodes, ','), {# == '137'})))}}Gi"
+```
+
 ### Container/Script Templates
 
 | Variable | Description|

util/template/expression_template.go

@@ -35,6 +35,7 @@ var (
 		varkeys.RetriesLastStatus.Template(),
 		varkeys.RetriesLastDuration.Template(),
 		varkeys.RetriesLastMessage.Template(),
+		varkeys.RetriesExitCodes.Template(),
 		varkeys.WorkflowStatus.Template(),
 		varkeys.WorkflowFailures.Template(),
 	}

util/variables/keys/retries.go

@@ -39,4 +39,5 @@ var (
 	RetriesLastStatus   = retry("lastRetry.status", "Phase of the previous attempt (or empty on first)")
 	RetriesLastDuration = retry("lastRetry.duration", "Duration of the previous attempt in seconds")
 	RetriesLastMessage  = retry("lastRetry.message", "Message of the previous attempt")
+	RetriesExitCodes    = retry("lastRetry.exitCodes", "Comma-separated exit codes of all previous attempts, oldest first (empty on the first attempt)")
 )

workflow/controller/operator.go

@@ -2140,6 +2140,10 @@ func getRetryNodeChildrenIds(node *wfv1.NodeStatus, nodes wfv1.Nodes) []string {
 	return childrenIds
 }
 
+// buildRetryStrategyLocalScope returns the lastRetry.* variables for a retry node — the retries index,
+// the last retried child's exit code, status, duration, and message, and the exit-code history of all
+// previous attempts — for use in retryStrategy.expression. It returns an empty scope when the node has
+// no retried children yet.
 func buildRetryStrategyLocalScope(node *wfv1.NodeStatus, nodes wfv1.Nodes) map[string]any {
 	localScope := make(map[string]any)
 
@@ -2160,6 +2164,20 @@ func buildRetryStrategyLocalScope(node *wfv1.NodeStatus, nodes wfv1.Nodes) map[s
 	localScope[varkeys.RetriesLastDuration.Template()] = fmt.Sprint(lastChildNode.GetDuration().Seconds())
 	localScope[varkeys.RetriesLastMessage.Template()] = lastChildNode.Message
 
+	// Mirror the template-substitution scope: expose the exit codes of ALL previous
+	// attempts (oldest first, comma-separated) so retryStrategy.expression can reference
+	// lastRetry.exitCodes as well. Validation allows it in both scopes, so without this
+	// an expression using it passes validation but resolves to nothing at runtime.
+	lastRetryExitCodes := make([]string, 0, len(childNodeIds))
+	for _, childID := range childNodeIds {
+		if childNode, err := nodes.Get(childID); err == nil {
+			if childNode.Outputs != nil && childNode.Outputs.ExitCode != nil {
+				lastRetryExitCodes = append(lastRetryExitCodes, *childNode.Outputs.ExitCode)
+			}
+		}
+	}
+	localScope[varkeys.RetriesExitCodes.Template()] = strings.Join(lastRetryExitCodes, ",")
+
 	return localScope
 }
 
@@ -2530,6 +2548,21 @@ func (woc *wfOperationCtx) executeTemplate(ctx context.Context, nodeName string,
 		localParams[varkeys.RetriesLastDuration.Template()] = lastRetryDuration
 		localParams[varkeys.RetriesLastStatus.Template()] = lastRetryStatus
 		localParams[varkeys.RetriesLastMessage.Template()] = lastRetryMessage
+
+		// Inject the exit codes of ALL previous attempts (oldest first, comma-separated) so an
+		// expression can escalate resources cumulatively per failure type — e.g. grow memory once
+		// per prior OOM and hold it across non-OOM retries — which lastRetry.exitCode (the last
+		// attempt alone) cannot express. Empty on the first attempt.
+		lastRetryExitCodes := make([]string, 0, len(childNodeIDs))
+		for _, childID := range childNodeIDs {
+			if childNode, getErr := woc.wf.Status.Nodes.Get(childID); getErr == nil {
+				if childNode.Outputs != nil && childNode.Outputs.ExitCode != nil {
+					lastRetryExitCodes = append(lastRetryExitCodes, *childNode.Outputs.ExitCode)
+				}
+			}
+		}
+		localParams[varkeys.RetriesExitCodes.Template()] = strings.Join(lastRetryExitCodes, ",")
+
 		processedTmpl, err = common.SubstituteParams(ctx, processedTmpl, woc.globalParams(), localParams)
 		if errorsutil.IsTransientErr(ctx, err) {
 			return node, err

workflow/controller/operator_test.go

@@ -1635,6 +1635,66 @@ func TestLastRetryVariableInPodSpecPatch(t *testing.T) {
 	assert.ElementsMatch(t, actual, expected)
 }
 
+var lastRetryExitCodesInPodSpecPatchTemplate = `
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  name: whalesay
+spec:
+  entrypoint: whalesay
+  templates:
+  - name: whalesay
+    retryStrategy:
+      limit: 10
+    podSpecPatch: |
+      containers:
+        - name: main
+          resources:
+            limits:
+              memory: "{{= (1 + len(filter(split(lastRetry.exitCodes, ','), {# == '137'}))) * 100}}Mi"
+    container:
+      image: python:alpine3.23
+      command: ["python", -c]
+      args: ["import sys; sys.exit(1)"]
+`
+
+// TestLastRetryExitCodesInPodSpecPatch verifies that lastRetry.exitCodes exposes the full history
+// of previous attempts' exit codes, so a podSpecPatch can grow memory once per prior OOM (137) and
+// HOLD it across non-OOM (here exit 1) retries — behavior lastRetry.exitCode (the last attempt
+// alone) cannot express.
+func TestLastRetryExitCodesInPodSpecPatch(t *testing.T) {
+	wf := wfv1.MustUnmarshalWorkflow(lastRetryExitCodesInPodSpecPatchTemplate)
+	cancel, controller := newController(logging.TestContext(t.Context()), wf)
+	defer cancel()
+	ctx := logging.TestContext(t.Context())
+	// Iteration i fails the (single, still-Pending) attempt created at iteration i-1.
+	// Applied exit codes per attempt: 137, 1, 137, 1 (the 5th attempt stays Pending).
+	exitCodeByIteration := map[int]int32{2: 137, 3: 1, 4: 137, 5: 1}
+	iterations := 5
+	var woc *wfOperationCtx
+	for i := 1; i <= iterations; i++ {
+		woc = newWorkflowOperationCtx(ctx, wf, controller)
+		if code, ok := exitCodeByIteration[i]; ok {
+			makePodsPhase(ctx, woc, apiv1.PodFailed, withExitCode(code))
+		}
+		woc.operate(ctx)
+		wf = woc.wf
+	}
+
+	pods, err := listPods(ctx, woc)
+	require.NoError(t, err)
+	assert.Len(t, pods.Items, iterations)
+	// count of prior 137s by attempt: 0,1,1,2,2 -> memory (1+count)*100Mi. The two "held" values
+	// (200Mi after the exit-1 retry, 300Mi after the second exit-1 retry) are the point of the test.
+	expected := []string{"100Mi", "200Mi", "200Mi", "300Mi", "300Mi"}
+	actual := []string{}
+	for i := range iterations {
+		actual = append(actual, pods.Items[i].Spec.Containers[1].Resources.Limits.Memory().String())
+	}
+	// ordering not preserved
+	assert.ElementsMatch(t, actual, expected)
+}
+
 var stepsRetriesVariableTemplate = `
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
@@ -10135,19 +10195,23 @@ func TestOperatorRetryExpression(t *testing.T) {
 	assert.Equal(t, "retryStrategy.expression evaluated to false", retryNode.Message)
 }
 
+// TestBuildRetryStrategyLocalScope verifies buildRetryStrategyLocalScope populates the lastRetry.*
+// variables from a retry node's child attempts.
 func TestBuildRetryStrategyLocalScope(t *testing.T) {
 	wf := wfv1.MustUnmarshalWorkflow(operatorRetryExpression)
 	retryNode, err := wf.GetNodeByName("retry-script-9z9pv[1].retry")
 	require.NoError(t, err)
 
 	localScope := buildRetryStrategyLocalScope(retryNode, wf.Status.Nodes)
 
-	assert.Len(t, localScope, 5)
+	assert.Len(t, localScope, 6)
 	assert.Equal(t, "1", localScope[varkeys.Retries.Template()])
 	assert.Equal(t, "1", localScope[varkeys.RetriesLastExitCode.Template()])
 	assert.Equal(t, string(wfv1.NodeFailed), localScope[varkeys.RetriesLastStatus.Template()])
 	assert.Equal(t, "6", localScope[varkeys.RetriesLastDuration.Template()])
 	assert.Equal(t, "Error (exit code 1)", localScope[varkeys.RetriesLastMessage.Template()])
+	// All previous attempts, oldest first: retry(0) exited 2, retry(1) exited 1.
+	assert.Equal(t, "2,1", localScope[varkeys.RetriesExitCodes.Template()])
 }
 
 const operatorRetryExpressionError = `

workflow/validate/validate.go

@@ -523,11 +523,13 @@ func (tctx *templateValidationCtx) validateTemplate(ctx context.Context, tmpl *w
 		localParams[varkeys.RetriesLastStatus.Template()] = placeholderGenerator.NextPlaceholder()
 		localParams[varkeys.RetriesLastDuration.Template()] = placeholderGenerator.NextPlaceholder()
 		localParams[varkeys.RetriesLastMessage.Template()] = placeholderGenerator.NextPlaceholder()
+		localParams[varkeys.RetriesExitCodes.Template()] = placeholderGenerator.NextPlaceholder()
 		scope[varkeys.Retries.Template()] = placeholderGenerator.NextPlaceholder()
 		scope[varkeys.RetriesLastExitCode.Template()] = placeholderGenerator.NextPlaceholder()
 		scope[varkeys.RetriesLastStatus.Template()] = placeholderGenerator.NextPlaceholder()
 		scope[varkeys.RetriesLastDuration.Template()] = placeholderGenerator.NextPlaceholder()
 		scope[varkeys.RetriesLastMessage.Template()] = placeholderGenerator.NextPlaceholder()
+		scope[varkeys.RetriesExitCodes.Template()] = placeholderGenerator.NextPlaceholder()
 	}
 	if tmpl.IsLeaf() {
 		for _, art := range tmpl.Outputs.Artifacts {