acoustic-carpenter-78188
09/30/2025, 4:21 PM{"json":{"exec_id":"axtjrfrj8xck879d2xn8","ns":"flyte-staging","res_ver":"5768777206","routine":"worker-2","wf":"flyte:staging:car_early_claim_flyte.workflows.train.train_workflow"},"level":"error","msg":"panic when reconciling workflow, Stack: [goroutine 622 [running]:\nruntime/debug.Stack()\n\t/usr/local/go/src/runtime/debug/stack.go:26 +0x5e\<http://ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller.(*Propeller).TryMutateWorkflow.func2.1()|ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller.(*Propeller).TryMutateWorkflow.func2.1()>\n\t/go/src/github.com/flyteorg/flytepropeller/pkg/controller/handler.go:137 +0x6e\npanic({0x26a9c40?, 0x47d2a90?})\n\t/usr/local/go/src/runtime/panic.go:791 +0x132\<http://ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes/task.(*Handler).IsCacheable(0xc00070c000|ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes/task.(*Handler).IsCacheable(0xc00070c000>, {0x30be230, 0xc001fa8780}, {0x30d5f70, 0xc001785980})\n\t/go/src/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/cache.go:46 +0x1c9\<http://ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes.(*nodeExecutor).handleNotYetStartedNode(0xc001386000|ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes.(*nodeExecutor).handleNotYetStartedNode(0xc001386000>, {0x30be230, 0xc001fa8780}, {0x30a1990, 0xc0007d6f08}, {0x30d5f70, 0xc001785980}, {0x30c3280, 0xc000594280})\n\t/go/src/github.com/flyteorg/flytepropeller/pkg/controller/nodes/executor.go:981 +0x674\<http://ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes.(*nodeExecutor).HandleNode(0xc001386000|ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes.(*nodeExecutor).HandleNode(0xc001386000>, {0x30be230?, 0xc001fa8390?}, {0x30a1990, 0xc0007d6f08}, {0x30d5f70, 0xc001785980}, {0x30c3280, 0xc000594280})\n\t/go/src/github.com/flyteorg/flytepropeller/pkg/controller/nodes/executor.go:1365 +0xefb\<http://ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes.(*recursiveNodeExecutor).RecursiveNodeHandler(0xc001388000|ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes.(*recursiveNodeExecutor).RecursiveNodeHandler(0xc001388000>, {0x30be230, 0xc001f50cc0}, {0x30e0010, 0xc0031b11d0}, {0x30a1990, 0xc0007d6f08}, {0x30bef98, 0xc0007d6f08}, {0x30da360, ...})\n\t/go/src/github.com/flyteorg/flytepropeller/pkg/controller/nodes/executor.go:234 +0x8fc\<http://ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes.(*recursiveNodeExecutor).handleDownstream(0xc001388000|ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes.(*recursiveNodeExecutor).handleDownstream(0xc001388000>, {0x30be230, 0xc001f50cc0}, {0x30e0010, 0xc0031b11d0}, {0x30a1990, 0xc0007d6f08}, {0x30bef98, 0xc0007d6f08}, {0x30da360, ...})\n\t/go/src/github.com/flyteorg/flytepropeller/pkg/controller/nodes/executor.go:300 +0x47a\<http://ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes.(*recursiveNodeExecutor).RecursiveNodeHandler(0xc001388000|ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes.(*recursiveNodeExecutor).RecursiveNodeHandler(0xc001388000>, {0x30be230, 0xc001f50cc0}, {0x30e0010, 0xc0031b11d0}, {0x30a1990, 0xc0007d6f08}, {0x30bef98, 0xc0007d6f08}, {0x30da360, ...})\n\t/go/src/github.com/flyteorg/flytepropeller/pkg/controller/nodes/executor.go:241 +0xb19\<http://ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller/workflow.(*workflowExecutor).handleRunningWorkflow(0xc00020b080|ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller/workflow.(*workflowExecutor).handleRunningWorkflow(0xc00020b080>, {0x30be230, 0xc001f50cc0}, 0xc0007d6f08)\n\t/go/src/github.com/flyteorg/flytepropeller/pkg/controller/workflow/executor.go:175 +0x1bb\<http://ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller/workflow.(*workflowExecutor).HandleFlyteWorkflow(0xc00020b080|ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller/workflow.(*workflowExecutor).HandleFlyteWorkflow(0xc00020b080>, {0x30be230, 0xc001f50cc0}, 0xc0007d6f08)\n\t/go/src/github.com/flyteorg/flytepropeller/pkg/controller/workflow/executor.go:432 +0x3af\<http://ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller.(*Propeller).TryMutateWorkflow.func2(0xc000687500|ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller.(*Propeller).TryMutateWorkflow.func2(0xc000687500>, {0x30be230, 0xc001f50cc0}, 0xc001083820, 0xc0007d6f08)\n\t/go/src/github.com/flyteorg/flytepropeller/pkg/controller/handler.go:143 +0x15d\<http://ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller.(*Propeller).TryMutateWorkflow(0xc000687500|ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller.(*Propeller).TryMutateWorkflow(0xc000687500>, {0x30be230, 0xc001f50180}, 0xc0007d7908)\n\t/go/src/github.com/flyteorg/flytepropeller/pkg/controller/handler.go:144 +0x3be\<http://ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller.(*Propeller).streak(0xc000687500|ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller.(*Propeller).streak(0xc000687500>, {0x30be230?, 0xc001f50030?}, 0xc0007d7908, 0x0)\n\t/go/src/github.com/flyteorg/flytepropeller/pkg/controller/handler.go:301 +0x271\<http://ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller.(*Propeller).Handle(0xc000687500|ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller.(*Propeller).Handle(0xc000687500>, {0x30be230?, 0xc0017c3f50?}, {0xc00266d620, 0xd}, {0xc00266d62e, 0x14})\n\t/go/src/github.com/flyteorg/flytepropeller/pkg/controller/handler.go:244 +0x951\<http://ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller.(*WorkerPool).processNextWorkItem.func1(0xc0010b0240|ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller.(*WorkerPool).processNextWorkItem.func1(0xc0010b0240>, 0xc001083f30, {0x24f49c0, 0xc00189cd40})\n\t/go/src/github.com/flyteorg/flytepropeller/pkg/controller/workers.go:89 +0x4c3\<http://ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller.(*WorkerPool).processNextWorkItem(0xc0010b0240|ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller.(*WorkerPool).processNextWorkItem(0xc0010b0240>, {0x30be230, 0xc0017c3f50})\n\t/go/src/github.com/flyteorg/flytepropeller/pkg/controller/workers.go:100 +0xce\<http://ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller.(*WorkerPool).runWorker(0xc0010b0240|ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller.(*WorkerPool).runWorker(0xc0010b0240>, {0x30be230, 0xc0008206f0})\n\t/go/src/github.com/flyteorg/flytepropeller/pkg/controller/workers.go:116 +0x9a\<http://ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller.(*WorkerPool).Run.func1()|ngithub.com/flyteorg/flyte/flytepropeller/pkg/controller.(*WorkerPool).Run.func1()>\n\t/go/src/github.com/flyteorg/flytepropeller/pkg/controller/workers.go:151 +0x4f\ncreated by <http://github.com/flyteorg/flyte/flytepropeller/pkg/controller.(*WorkerPool).Run|github.com/flyteorg/flyte/flytepropeller/pkg/controller.(*WorkerPool).Run> in goroutine 32\n\t/go/src/github.com/flyteorg/flytepropeller/pkg/controller/workers.go:148 +0x272\n]","ts":"2025-09-30T09:53:15Z"}
### Expected behavior
Propeller should not panic
### Additional context to reproduce
No response
### Screenshots
No response
### Are you sure this issue hasn't been raised already?
• Yes
### Have you read the Code of Conduct?
• Yes
flyteorg/flyteacoustic-carpenter-78188
10/01/2025, 3:47 PM