Felix Ruess
01/17/2023, 4:49 PM{"json":{"exec_id":"af89l2cwzfjfnjpbxqfh","node":"trainaae","ns":"cadmatch-training-development","res_ver":"44393559","routine":"worker-0","wf":"cadmatch-training:development:.flytegen.train_aae"},"level":"warning","msg":"Failed to record taskEvent, error [EventAlreadyInTerminalStateError: conflicting events; destination: ABORTED, caused by [rpc error: code = FailedPrecondition desc = invalid phase change from FAILED to ABORTED for task execution {resource_type:TASK project:\"cadmatch-training\" domain:\"development\" name:\"train_aae\" version:\"2fbd809\" node_id:\"trainaae\" execution_id:\u003cproject:\"cadmatch-training\" domain:\"development\" name:\"af89l2cwzfjfnjpbxqfh\" \u003e 0 {} [] 0}]]. Trying to record state: ABORTED. Ignoring this error!","ts":"2023-01-17T14:26:57Z"}
{"json":{"exec_id":"af89l2cwzfjfnjpbxqfh","node":"trainaae","ns":"cadmatch-training-development","res_ver":"44393628","routine":"worker-1","wf":"cadmatch-training:development:.flytegen.train_aae"},"level":"warning","msg":"Failed to record taskEvent, error [EventAlreadyInTerminalStateError: conflicting events; destination: ABORTED, caused by [rpc error: code = FailedPrecondition desc = invalid phase change from FAILED to ABORTED for task execution {resource_type:TASK project:\"cadmatch-training\" domain:\"development\" name:\"train_aae\" version:\"2fbd809\" node_id:\"trainaae\" execution_id:\u003cproject:\"cadmatch-training\" domain:\"development\" name:\"af89l2cwzfjfnjpbxqfh\" \u003e 1 {} [] 0}]]. Trying to record state: ABORTED. Ignoring this error!","ts":"2023-01-17T14:27:02Z"}
{"json":{"exec_id":"af89l2cwzfjfnjpbxqfh","node":"trainaae","ns":"cadmatch-training-development","res_ver":"44393690","routine":"worker-0","wf":"cadmatch-training:development:.flytegen.train_aae"},"level":"warning","msg":"Failed to record taskEvent, error [EventAlreadyInTerminalStateError: conflicting events; destination: ABORTED, caused by [rpc error: code = FailedPrecondition desc = invalid phase change from FAILED to ABORTED for task execution {resource_type:TASK project:\"cadmatch-training\" domain:\"development\" name:\"train_aae\" version:\"2fbd809\" node_id:\"trainaae\" execution_id:\u003cproject:\"cadmatch-training\" domain:\"development\" name:\"af89l2cwzfjfnjpbxqfh\" \u003e 2 {} [] 0}]]. Trying to record state: ABORTED. Ignoring this error!","ts":"2023-01-17T14:27:08Z"}
{"json":{"exec_id":"af89l2cwzfjfnjpbxqfh","node":"trainaae","ns":"cadmatch-training-development","res_ver":"44393750","routine":"worker-1","wf":"cadmatch-training:development:.flytegen.train_aae"},"level":"warning","msg":"Trying to abort a node in state [Failed]","ts":"2023-01-17T14:27:14Z"}
{"json":{"exec_id":"af89l2cwzfjfnjpbxqfh","ns":"cadmatch-training-development","routine":"worker-0"},"level":"warning","msg":"Workflow namespace[cadmatch-training-development]/name[af89l2cwzfjfnjpbxqfh] has already been terminated.","ts":"2023-01-17T14:27:48Z"}
{"json":{"exec_id":"af89l2cwzfjfnjpbxqfh","ns":"cadmatch-training-development","routine":"worker-1"},"level":"warning","msg":"Workflow namespace[cadmatch-training-development]/name[af89l2cwzfjfnjpbxqfh] has already been terminated.","ts":"2023-01-17T14:28:02Z"}
Björn
01/17/2023, 7:32 PMFelix Ruess
01/17/2023, 7:38 PMBjörn
01/17/2023, 7:40 PMFelix Ruess
01/17/2023, 7:44 PMDan Rammer (hamersaw)
01/17/2023, 8:14 PM