acoustic-carpenter-78188
10/16/2023, 8:27 AMrayJob.Status.StartTime is nil when the status is JobStatusPending
Updates:
• Convert the ray cluster phase to the flyte task phase if ray cluster is not running
• Convert the ray job phase to the flyte task phase if ray cluster is running
Error:
```
Workflow[flytesnacksdevelopment.flytegen.task.ray_task] failed. RuntimeExecutionError: max number of system retry attempts [11/10] exhausted. Last known status message: failed at Node[taskraytask]. RuntimeExecutionError: failed during plugin execution, caused by: failed to execute handle for plugin [ray]: panic when executing a plugin [ray]. Stack: [goroutine 2818 [running]:
runtime/debug.Stack()
/opt/homebrew/opt/go/libexec/src/runtime/debug/stack.go:24 +0x64
github.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes/task.Handler.invokePlugin.func1.1()
/Users/jeev/Workspace/repos/flyte/flyte/flytepropeller/pkg/controller/nodes/task/handler.go:382 +0xb4
panic({0x103e54b40?, 0x105c06500?})
/opt/homebrew/opt/go/libexec/src/runtime/panic.go:914 +0x218
github.com/flyteorg/flyte/flyteplugins/go/tasks/plugins/k8s/ray.rayJobResourceHandler.GetTaskPhase({}, {0x140063a83c0?, 0x20?}, {0x1043e5ef0, 0x140063a83c0}, {0x1043fdc88?, 0x14005235180})
/Users/jeev/Workspace/repos/flyte/flyte/flyteplugins/go/tasks/plugins/k8s/ray/ray.go:405 +0x268
github.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes/task/k8s.(*PluginManager).checkResourcePhase(0x140063b6a68, {0x1043d9df8, 0x140063a6d80}, {0x1043ee780, 0x140063ac0c0}, {0x1043fdc88?, 0x14005235180?}, 0x140063a2588)
/Users/jeev/Workspace/repos/flyte/flyte/flytepropeller/pkg/controller/nodes/task/k8s/plugin_manager.go:282 +0x5f8
github.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes/task/k8s.PluginManager.Handle({{0x102b18ce4, 0x3}, {0x1043da3a8, 0x105d39fa0}, {0x1043b9dc0, 0x14000778b00}, {0x14b9be8a8, 0x1400106b520}, {{0x1043fa058, 0x1400108acd0}, ...}, ...}, ...)
/Users/jeev/Workspace/repos/flyte/flyte/flytepropeller/pkg/controller/nodes/task/k8s/plugin_manager.go:344 +0x3a8
github.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes/task.Handler.invokePlugin.func1(0x140063b6f88?, {0x1043d9df8, 0x140063a6a80}, {0x1043de2c0?, 0x14002214160?}, 0x9?)
/Users/jeev/Workspace/repos/flyte/flyte/flytepropeller/pkg/controller/nodes/task/handler.go:389 +0x110
github.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes/task.Handler.invokePlugin({{0x1043dd4c0, 0x14000134cf0}, {0x1043bd970, 0x140001a8a00}, 0x14000d13140, 0x14000d13170, 0x14000d131a0, {0x1043de2c0, 0x14002718580}, 0x14001224d20, ...}, ...)
/Users/jeev/Workspace/repos/flyte/flyte/flytepropeller/pkg/controller/nodes/task/handler.go:391 +0x60
github.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes/task.Handler.Handle({{0x1043dd4c0, 0x14000134cf0}, {0x1043bd970, 0x140001a8a00}, 0x14000d13140, 0x14000d13170, 0x14000d131a0, {0x1043de2c0, 0x14002718580}, 0x14001224d20, ...}, ...)
/Users/jeev/Workspace/repos/flyte/flyte/flytepropeller/pkg/controller/nodes/task/handler.go:572 +0x630
github.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes/dynamic.dynamicNodeTaskNodeHandler.handleParentNode({{0x1043e76b8, 0x14001290680}, {{0x14001289840, {{...}, 0x0}, {0x140008bb130, 0xb, 0xb}}, {0x14001289860, {{...}, ...}, ...}, ...}, ...}, ...)
/Users/jeev/Workspace/repos/flyte/flyte/flytepropeller/pkg/controller/nodes/dynamic/handler.go:67 +0xa0
github.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes/dynamic.dynamicNodeTaskNodeHandler.Handle({{0x1043e76b8, 0x14001290680}, {{0x14001289840, {{...}, 0x0}, {0x140008bb130, 0xb, 0xb}}, {0x14001289860, {{...}, ...}, ...}, ...}, ...}, ...)
/Users/jeev/Workspace/repos/flyte/flyte/flytepropeller/pkg/controller/nodes/dynamic/handler.go:224 +0x888
github.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes.(*nodeExecutor).execute(0x140012900d0, {0x1043d9df8, 0x14006383d40}, {0x1043dd580, 0x14000790780}, {0x1043f1cf0, 0x140063ac000}, {0x1044002e0?, 0x14006394460?})
/Users/jeev/Workspace/repos/flyte/flyte/flytepropeller/pkg/controller/nodes/executor.go:822 +0xd8
github.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes.(*nodeExecutor).handleQueuedOrRunningNode(0x140012900d0, {0x1043d9df8, 0x14006383d40}, {0x1043f1cf0?, 0x140063ac000?}, {0x1043dd580, 0x14000790780})
/Users/jeev/Workspace/repos/flyte/flyte/flytepropeller/pkg/controller/nodes/executor.go:1116 +0x838
github.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes.(*nodeExecutor).HandleNode(0x140012900d0, {0x1043d9df8, 0x14006383d40}, {0x1043b69e0, 0x14005713900}, {0x1043f1cf0, 0x140063ac000?}, {0x1043dd580?, 0x14000790780})
/Users/jeev/Workspace/repos/flyte/flyte/flytepropeller/pkg/controller/nodes/executor.go:1384 +0x228
github.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes.(*recursiveNodeExecutor).RecursiveNodeHandler(0x14000b37e00, {0x1043d9df8, 0x140063836e0}, {0x1043f6a48, 0x1400624eaf0}, {0x1043b69e0, 0x14005713900}, {0x1043dab18?, 0x14005713900?}, {0x1043f38e0, ...})
/Users/jeev/Workspace/repos/flyte/flyte/flytepropeller/pkg/controller/nodes/executor.go:229 +0x51c
github.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes.(*recursiveNodeExecutor).handleDownstream(0x1043d9df8?, {0x1043d9df8, 0x140063836e0}, {0x1043f6a48, 0x1400624eaf0}, {0x1043b69e0, 0x14005713900?}, {0x1043dab18?, 0x14005713900}, {0x1043f38e0, ...})
/Users/jeev/Workspace/repos/flyte/flyte/flytepropeller/pkg/controller/nodes/executor.go:294 +0x300
github.com/flyteorg/flyte/flytepropeller/pkg/controller/nodes.(*recursiveNodeExecutor).RecursiveNodeHandler(0x14000b37e00, {0x1043d9df8, 0x140063836e0}, {0x1043f6a48, 0x1400624eaf0}, {0x1043b69e0, 0x14005713900}, {0x1043dab18?, 0x14005713900?}, {0x1043f38e0, ...})
/Users/jeev/Workspace/repos/flyte/flyte/flytepropeller/pkg/controller/nodes/executor.go:236 +0x698
<http://github.com/flyteorg/flyte/flytepropeller/pkg/controller/workflow.(*workflowExecutor).handleRunningWorkflow(0x140000ddd50|github.com/flyteorg/flyte/flytepropeller/pkg/controller/workflow.(*workflowExe…
flyteorg/flyte
GitHub Actions: Build & Push Image
GitHub Actions: Bump git tags
✅ 28 other checks have passed
28/30 successful checksacoustic-carpenter-78188
10/16/2023, 9:59 PM