#4046 [BUG] Flyte propeller fatal error: concurrent map writes Issue created by pablocasares Describe the bug Flyte propeller crashes with a concurrent map writes fatal error in version v1.1.116. We had several goroutines running while the crash happened but I think this is the one that caused it as it's the only one with `panic.go`: `goroutine 1374 [runnable]: fmt.(*pp).printArg(0xc1a07836c0?, {0x23de800?, 0xc1a0785b90?}, 0x73?) /usr/local/go/src/fmt/print.go:681 +0x7b6 fmt.(*pp).doPrintf(0xc1a07836c0, {0x29a90d2, 0x2c}, {0xc1a0794eb0?, 0x1, 0x1}) /usr/local/go/src/fmt/print.go:1077 +0x387 fmt.Errorf({0x29a90d2, 0x2c}, {0xc1a0794eb0?, 0x1, 0x1}) /usr/local/go/src/fmt/errors.go:25 +0x97 github.com/flyteorg/flytepropeller/pkg/controller.(*Propeller).TryMutateWorkflow.func2.1() /go/pkg/mod/github.com/flyteorg/flytepropeller@v1.1.116/pkg/controller/handler.go:137 +0xba panic({0x23de800, 0xc1a0785260}) /usr/local/go/src/runtime/panic.go:884 +0x213 github.com/flyteorg/flytestdlib/promutils.panicIfError(...) /go/pkg/mod/github.com/flyteorg/flytestdlib@v1.0.22/promutils/scope.go:22 github.com/flyteorg/flytestdlib/promutils.metricsScope.MustNewCounter({{0xc0019af520?, 0x1a?}}, {0xc1a07a2030?, 0x1a?}, {0xc1a07a2000?, 0xa?}) /go/pkg/mod/github.com/flyteorg/flytestdlib@v1.0.22/promutils/scope.go:327 +0x74 github.com/flyteorg/flytestdlib/promutils/labeled.NewCounter({0xc19def7240?, 0x293cfb1?}, {0xc1a07a2000, 0x2d}, {0x2ea5290, 0xc0007f9520}, {0xc1a07952f8, 0x1, 0x0?}) /go/pkg/mod/github.com/flyteorg/flytestdlib@v1.0.22/promutils/labeled/counter.go:62 +0x163 github.com/flyteorg/flytepropeller/pkg/controller/nodes/task.Handler.fetchPluginTaskMetrics({{0x2e8b600, 0xc00154a120}, {0x2e74910, 0xc00084cb60}, 0xc001e01f20, 0xc001e01f50, 0xc001e01f80, {0x2e8b640, 0xc19a792500}, 0xc002032000, ...}, ...) /go/pkg/mod/github.com/flyteorg/flytepropeller@v1.1.116/pkg/controller/nodes/task/handler.go:373 +0x146 github.com/flyteorg/flytepropeller/pkg/controller/nodes/task.Handler.invokePlugin({{0x2e8b600, 0xc00154a120}, {0x2e74910, 0xc00084cb60}, 0xc001e01f20, 0xc001e01f50, 0xc001e01f80, {0x2e8b640, 0xc19a792500}, 0xc002032000, ...}, ...) /go/pkg/mod/github.com/flyteorg/flytepropeller@v1.1.116/pkg/controller/nodes/task/handler.go:453 +0xca8 github.com/flyteorg/flytepropeller/pkg/controller/nodes/task.Handler.Handle({{0x2e8b600, 0xc00154a120}, {0x2e74910, 0xc00084cb60}, 0xc001e01f20, 0xc001e01f50, 0xc001e01f80, {0x2e8b640, 0xc19a792500}, 0xc002032000, ...}, ...) /go/pkg/mod/github.com/flyteorg/flytepropeller@v1.1.116/pkg/controller/nodes/task/handler.go:674 +0x1c65 github.com/flyteorg/flytepropeller/pkg/controller/nodes/dynamic.dynamicNodeTaskNodeHandler.handleParentNode({{0x2e8d920, 0xc000001500}, {{0xc0007f9790, {{...}, 0x0}, {0xc00138e000, 0x3, 0x3}}, {0xc0007f97b0, {{...}, ...}, ...}, ...}, ...}, ...) /go/pkg/mod/github.com/flyteorg/flytepropeller@v1.1.116/pkg/controller/nodes/dynamic/handler.go:68 +0xe8 github.com/flyteorg/flytepropeller/pkg/controller/nodes/dynamic.dynamicNodeTaskNodeHandler.Handle({{0x2e8d920, 0xc000001500}, {{0xc0007f9790, {{...}, 0x0}, {0xc00138e000, 0x3, 0x3}}, {0xc0007f97b0, {{...}, ...}, ...}, ...}, ...}, ...) /go/pkg/mod/github.com/flyteorg/flytepropeller@v1.1.116/pkg/controller/nodes/dynamic/handler.go:229 +0xb4b github.com/flyteorg/flytepropeller/pkg/controller/nodes.(*nodeExecutor).execute(0xc00056c840, {0x2e89100, 0xc1a01eb410}, {0x2e8b440, 0xc000fba140}, {0x2e9daf8, 0xc19c52c240}, {0x2eac098?, 0xc19f0de8c0?}) /go/pkg/mod/github.com/flyteorg/flytepropeller@v1.1.116/pkg/controller/nodes/executor.go:837 +0x164 github.com/flyteorg/flytepropeller/pkg/controller/nodes.(*nodeExecutor).handleQueuedOrRunningNode(0xc00056c840, {0x2e89100, 0xc1a01eb410}, {0x2e9daf8?, 0xc19c52c240?}, {0x2e8b440, 0xc000fba140}) /go/pkg/mod/github.com/flyteorg/flytepropeller@v1.1.116/pkg/controller/nodes/executor.go:1011 +0x20a github.com/flyteorg/flytepropeller/pkg/controller/nodes.(*nodeExecutor).HandleNode(0xc00056c840, {0x2e89100, 0xc1a01eb410}, {0x2e74a50, 0xc1a01eb3e0}, {0x2e9daf8, 0xc19c52c240?}, {0x2e8b440?, 0xc000fba140}) /go/pkg/mod/github.com/flyteorg/flytepropeller@v1.1.116/pkg/controller/nodes/executor.go:1240 +0x338 github.com/flyteorg/flytepropeller/pkg/controller/nodes.(*recursiveNodeExecutor).RecursiveNodeHandler(0xc0015dfa90, {0x2e89100, 0xc1a01eab40}, {0x2ea2d18, 0xc1a01b8cd0}, {0x2e74a50, 0xc1a01eb3e0}, {0x2e89368?, 0xc19f0dc500?}, {0x2e9e648, ...}) /go/pkg/mod/github.com/flyteorg/flytepropeller@v1.1.116/pkg/controller/nodes/executor.go:216 +0x6a7 github.com/flyteorg/flytepropeller/pkg/controller/nodes/branch.(*branchHandler).recurseDownstream(0xc001992450, {0x2e89100, 0xc1a01eab40}, {0x2e9daf8, 0xc19dcf2d80}, {0x2eac098, 0xc19f0de700}, {0x2e9e648?, 0xc1a01d0140?}) /go/pkg/mod/github.com/flyteorg/flytepropeller@v1.1.116/pkg/controller/nodes/branch/handler.go:150 +0x408 github.com/flyteorg/flytepropeller/pkg/controller/nodes/branch.(*branchHandler).HandleBranchNode(0xc1a01c9b40?, {0x2e89100, 0xc1a01eab40}, {0x2e89330, 0xc1a01ccac0}, {0x2e9daf8?, 0xc19dcf2d80?}, {0x2e89368, 0xc19f0dc500}) /go/pkg/mod/github.com/flyteorg/flytepropeller@v1.1.116/pkg/controller/nodes/branch/handler.go:105 +0x889 github.com/flyteorg/flytepropeller/pkg/controller/nodes/branch.(*branchHandler).Handle(0x294cb4e?, {0x2e89100, 0xc1a01eab40}, {0x2e9daf8?, 0xc19dcf2d80?}) /go/pkg/mod/github.com/flyteorg/flytepropeller@v1.1.116/pkg/controller/nodes/branch/handler.go:117 +0x137 github.com/flyteorg/flytepropeller/pkg/controller/nodes.(*nodeExecutor).execute(0xc00056c840, {0x2e89100, 0xc1a01eab40}, {0x2e8b400, 0xc001992450}, {0x2e9daf8, 0xc19dcf2d80}, {0x2eac098?, 0xc19f0de700?}) /go/pkg/mod/github.com/flyteorg/flytepropeller@v1.1.116/pkg/controller/nodes/executor.go:837 +0x164 github.com/flyteorg/flytepropeller/pkg/controller/nodes.(*nodeExecutor).handleQueuedOrRunningNode(0xc00056c840, {0x2e89100, 0xc1a01eab40}, {0x2e9daf8?, 0xc19dcf2d80?}, {0x2e8b400, 0xc001992450}) /go/pkg/mod/github.com/flyteorg/flytepropeller@v1.1.116/pkg/controller/nodes/executor.go:1011 +0x20a <http://github.com/flyteorg/flytepropeller/pkg/controller/nodes.(*nodeExecutor).HandleNode(0xc00056c840|github.com/fly… flyteorg/flyte