Memory leak in flyte-binary helm setup

configuration:

  database:
    username: postgres
    password: password
    host: postgresql-flyte-binary-hl
    dbname: flyteadmin

  storage:
    provider: s3

  propeller:
    createCRDs: false

  logging:
    level: 1

  inline:

    plugins:
      k8s:
        inject-finalizer: true
        update-backoff-retries: 50
        
      logs:
        templates:
          displayName: Download Logs
          showWhilePending: false
          templateUris:
            - 'some-url'
          messageFormat: 0

    storage:
      # cache:
      #   max_size_mbs: 10
      #   target_gc_percent: 100
      limits:
        maxDownloadMBs: 4

    task_resources:
      defaults:
        cpu: 0.2
        memory: 400Mi
      limits:
        cpu: 2
        memory: 4Gi


    propeller:
      max-output-size-bytes: 52428800
      max-streak-length: '12'
      max-workflow-retries: '100'
      workers: '40'

    server:
      grpc:
        maxMessageSizeBytes: 33554432


deployment:
  resources:
    requests:
      cpu: "1"
      memory: 1Gi
    limits:
      cpu: "2"
      memory: 4Gi


flyte-core-components:
  admin:
    disableClusterResourceManager: false



# rbac Configure Kubernetes RBAC for Flyte
rbac:
  # create Create ClusterRole and ClusterRoleBinding resources
  create: false

serviceAccount:
  create: false
  name: flyte-service-account



ingress:
  create: true
  ingressClassName: nginx
  commonAnnotations:
    <http://kubernetes.io/ingress.class|kubernetes.io/ingress.class>: nginx
  httpAnnotations:
    <http://nginx.ingress.kubernetes.io/app-root|nginx.ingress.kubernetes.io/app-root>: /console
  grpcAnnotations:
    <http://nginx.ingress.kubernetes.io/backend-protocol|nginx.ingress.kubernetes.io/backend-protocol>: GRPC
  host: flyte.<dns>.com

Any ideas? Attaching prometheus metrics for visualization Our workload has many many smaller tasks which is why we had to edit inline.propeller to get some decent throughput

I think it might be the metrics