gray-lifeguard-89900
09/15/2022, 10:06 AMglamorous-carpet-83516
09/15/2022, 10:43 AMgray-lifeguard-89900
09/15/2022, 10:46 AMgray-lifeguard-89900
09/15/2022, 10:47 AMconfigmap:
adminServer:
server:
httpPort: 8088
grpcPort: 8089
security:
secure: false
useAuth: false
allowCors: true
allowedOrigins:
# Accepting all domains for Sandbox installation
- "*"
allowedHeaders:
- "Content-Type"
flyteadmin:
roleNameKey: "<http://eks.amazonaws.com/role-arn|eks.amazonaws.com/role-arn>"
queues:
# A list of items, one per AWS Batch Job Queue.
executionQueues:
# The name of the job queue from AWS Batch
- dynamic: "fiolito-job-queue-1"
# A list of tags/attributes that can be used to match workflows to this queue.
attributes:
- default
# A list of configs to match project and/or domain and/or workflows to job queues using tags.
workflowConfigs:
# An empty rule to match any workflow to the queue tagged as "default"
- tags:
- default
task_resource_defaults:
task_resources:
defaults:
cpu: 1000m
memory: 1000Mi
storage: 1000Mi
limits:
cpu: 256
memory: 512Gi
storage: 512Gi
core:
propeller:
rawoutput-prefix: "s3://{{ .Values.userSettings.bucketName }}/"
workers: 40
gc-interval: 12h
max-workflow-retries: 50
kube-client-config:
qps: 100
burst: 25
timeout: 30s
queue:
sub-queue:
type: bucket
rate: 100
capacity: 1000
plugins:
aws:
batch:
# Must match that set in flyteAdmin's configMap flyteadmin.roleNameKey
roleAnnotationKey: <http://eks.amazonaws.com/role-arn|eks.amazonaws.com/role-arn>
# Must match the desired region to launch these tasks.
region: eu-west-1
# -- Resource manager configuration
resource_manager:
# -- resource manager configuration
propeller:
resourcemanager:
type: noop
# Note: By default resource manager is disable for propeller, Please use `type: redis` to enaable
# type: redis
# resourceMaxQuota: 10000
# redis:
# hostPath: "{{ .Values.userSettings.redisHostUrl }}"
# hostKey: "{{ .Values.userSettings.redisHostKey }}"
enabled_plugins:
plugins:
aws:
batch:
# Must match that set in flyteAdmin's configMap flyteadmin.roleNameKey
roleAnnotationKey: <http://eks.amazonaws.com/role-arn|eks.amazonaws.com/role-arn>
# Must match the desired region to launch these tasks.
region: eu-west-1
# -- Tasks specific configuration [structure](<https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig>)
tasks:
# -- Plugins configuration, [structure](<https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig>)
task-plugins:
# -- [Enabled Plugins](<https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/config#Config>). Enable sagemaker*, athena if you install the backend
# plugins
enabled-plugins:
- container
- sidecar
- k8s-array
- aws_array
# - sagemaker_hyperparameter_tuning
# - sagemaker_custom_training
# - sagemaker_training
default-for-task-types:
container: aws_array
sidecar: sidecar
container_array: k8s-array
aws-batch: aws_array
# sagemaker_custom_training_task: sagemaker_custom_training
# sagemaker_custom_training_job_task: sagemaker_custom_training
glamorous-carpet-83516
09/15/2022, 11:46 AMaws_array
handler for aws-batch task, could you share the propeller log?gray-lifeguard-89900
09/15/2022, 11:56 AM{"json":{"exec_id":"azfljccclzdnfslv7bnn","node":"n0/n0","ns":"fiolito-monto-erp-score-development","res_ver":"13342782","routine":"worker-4","tasktype":"python-task","wf":"fiolito-monto-erp-score:development:src.main.wrapper_pipeline"},"level":"warning","msg":"No plugin found for Handler-type [python-task], defaulting to [container]","ts":"2022-09-15T11:55:43Z"}
{"json":{"exec_id":"azfljccclzdnfslv7bnn","node":"n0/n0","ns":"fiolito-monto-erp-score-development","res_ver":"13342787","routine":"worker-4","tasktype":"python-task","wf":"fiolito-monto-erp-score:development:src.main.wrapper_pipeline"},"level":"warning","msg":"No plugin found for Handler-type [python-task], defaulting to [container]","ts":"2022-09-15T11:55:43Z"}
{"json":{"exec_id":"azfljccclzdnfslv7bnn","node":"n0/n0","ns":"fiolito-monto-erp-score-development","res_ver":"13342792","routine":"worker-4","tasktype":"python-task","wf":"fiolito-monto-erp-score:development:src.main.wrapper_pipeline"},"level":"warning","msg":"No plugin found for Handler-type [python-task], defaulting to [container]","ts":"2022-09-15T11:55:43Z"}
gray-lifeguard-89900
09/15/2022, 11:59 AMgray-lifeguard-89900
09/15/2022, 12:01 PMglamorous-carpet-83516
09/15/2022, 12:02 PM@task(task_config=AWSBatchConfig(...))
def t1(a: int) -> int:
return a + 2
gray-lifeguard-89900
09/15/2022, 12:03 PMgray-lifeguard-89900
09/15/2022, 12:03 PMgray-lifeguard-89900
09/15/2022, 12:04 PMglamorous-carpet-83516
09/15/2022, 12:04 PMglamorous-carpet-83516
09/15/2022, 12:04 PM