fast-doctor-21364
09/14/2022, 4:42 PMflytectl sandbox start --source . --imagePullPolicy IfNotPresent
flytectl sandbox exec -- docker build -t new_image .
When I run
flytectl sandbox exec -- docker images
I see my image inside of the flyte-sandbox container.
REPOSITORY TAG IMAGE ID CREATED SIZE
new_image latest 044c6f6eb860 44 minutes ago 1.49GB
Within my @task
, I reference new_image
. When I go to run the workflow remotely, I receive the following error after about 30 seconds:
[1/1] currentAttempt done. Last Error: USER::containers with unready status: [ffdb7fdd18e544b99a24-n0-0]|Back-off pulling image "new_image:latest"
Any thoughts?thankful-minister-83577
-o yaml
and grep for -i imagepullpolicy?fast-doctor-21364
09/14/2022, 9:06 PMapiVersion: v1
kind: Pod
metadata:
annotations:
<http://cluster-autoscaler.kubernetes.io/safe-to-evict|cluster-autoscaler.kubernetes.io/safe-to-evict>: "false"
creationTimestamp: "2022-09-14T16:13:26Z"
labels:
domain: development
execution-id: ffdb7fdd18e544b99a24
interruptible: "false"
node-id: n0
project: flytesnacks
shard-key: "8"
task-name: v1-in-flyte-auto-entity-encounters
workflow-name: v1-in-flyte-auto-drug-adherence
name: ffdb7fdd18e544b99a24-n0-0
namespace: flytesnacks-development
ownerReferences:
- apiVersion: <http://flyte.lyft.com/v1alpha1|flyte.lyft.com/v1alpha1>
blockOwnerDeletion: true
controller: true
kind: flyteworkflow
name: ffdb7fdd18e544b99a24
uid: 5f2d1869-19c3-4222-9eb3-ac5dab73ad95
resourceVersion: "32838"
uid: da4879de-7aa3-41a5-92a9-8771564eb583
spec:
containers:
- args:
- pyflyte-fast-execute
- --additional-distribution
- <s3://my-s3-bucket/uo/flytesnacks/development/GAG47EGB6FX7VTOKI3RTFWRBCQ======/scriptmode.tar.gz>
- --dest-dir
- /root
- --
- pyflyte-execute
- --inputs
- <s3://my-s3-bucket/metadata/propeller/flytesnacks-development-ffdb7fdd18e544b99a24/n0/data/inputs.pb>
- --output-prefix
- <s3://my-s3-bucket/metadata/propeller/flytesnacks-development-ffdb7fdd18e544b99a24/n0/data/0>
- --raw-output-data-prefix
- <s3://my-s3-bucket/va/ffdb7fdd18e544b99a24-n0-0>
- --checkpoint-path
- <s3://my-s3-bucket/va/ffdb7fdd18e544b99a24-n0-0/_flytecheckpoints>
- --prev-checkpoint
- '""'
- --resolver
- flytekit.core.python_auto_container.default_task_resolver
- --
- task-module
- v1_in_flyte_auto
- task-name
- entity_encounters
env:
- name: FLYTE_INTERNAL_EXECUTION_WORKFLOW
value: flytesnacks:development:v1_in_flyte_auto.drug_adherence
- name: FLYTE_INTERNAL_EXECUTION_ID
value: ffdb7fdd18e544b99a24
- name: FLYTE_INTERNAL_EXECUTION_PROJECT
value: flytesnacks
- name: FLYTE_INTERNAL_EXECUTION_DOMAIN
value: development
- name: FLYTE_ATTEMPT_NUMBER
value: "0"
- name: FLYTE_INTERNAL_TASK_PROJECT
value: flytesnacks
- name: FLYTE_INTERNAL_TASK_DOMAIN
value: development
- name: FLYTE_INTERNAL_TASK_NAME
value: v1_in_flyte_auto.entity_encounters
- name: FLYTE_INTERNAL_TASK_VERSION
value: hLXD0Y5ROcNIHsf2hPlPqA==
- name: FLYTE_INTERNAL_PROJECT
value: flytesnacks
- name: FLYTE_INTERNAL_DOMAIN
value: development
- name: FLYTE_INTERNAL_NAME
value: v1_in_flyte_auto.entity_encounters
- name: FLYTE_INTERNAL_VERSION
value: hLXD0Y5ROcNIHsf2hPlPqA==
- name: FLYTE_AWS_ACCESS_KEY_ID
value: minio
- name: FLYTE_AWS_SECRET_ACCESS_KEY
value: miniostorage
- name: FLYTE_AWS_ENDPOINT
value: <http://minio.flyte.svc.cluster.local:9000>
image: aapiv2:latest
imagePullPolicy: Always
name: ffdb7fdd18e544b99a24-n0-0
resources:
limits:
cpu: 100m
memory: 500Mi
requests:
cpu: 100m
memory: 500Mi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: kube-api-access-bfblg
readOnly: true
dnsPolicy: ClusterFirst
enableServiceLinks: true
nodeName: ce0d27279e26
preemptionPolicy: PreemptLowerPriority
priority: 0
restartPolicy: Never
schedulerName: default-scheduler
securityContext: {}
serviceAccount: default
serviceAccountName: default
terminationGracePeriodSeconds: 30
tolerations:
- effect: NoExecute
key: <http://node.kubernetes.io/not-ready|node.kubernetes.io/not-ready>
operator: Exists
tolerationSeconds: 300
- effect: NoExecute
key: <http://node.kubernetes.io/unreachable|node.kubernetes.io/unreachable>
operator: Exists
tolerationSeconds: 300
volumes:
- name: kube-api-access-bfblg
projected:
defaultMode: 420
sources:
- serviceAccountToken:
expirationSeconds: 3607
path: token
- configMap:
items:
- key: ca.crt
path: ca.crt
name: kube-root-ca.crt
- downwardAPI:
items:
- fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
path: namespace
status:
conditions:
- lastProbeTime: null
lastTransitionTime: "2022-09-14T16:13:26Z"
status: "True"
type: Initialized
- lastProbeTime: null
lastTransitionTime: "2022-09-14T16:13:26Z"
message: 'containers with unready status: [ffdb7fdd18e544b99a24-n0-0]'
reason: ContainersNotReady
status: "False"
type: Ready
- lastProbeTime: null
lastTransitionTime: "2022-09-14T16:13:26Z"
message: 'containers with unready status: [ffdb7fdd18e544b99a24-n0-0]'
reason: ContainersNotReady
status: "False"
type: ContainersReady
- lastProbeTime: null
lastTransitionTime: "2022-09-14T16:13:26Z"
status: "True"
type: PodScheduled
containerStatuses:
- image: new_image:latest
imageID: ""
lastState: {}
name: ffdb7fdd18e544b99a24-n0-0
ready: false
restartCount: 0
started: false
state:
waiting:
message: Back-off pulling image "new_image:latest"
reason: ImagePullBackOff
hostIP: 172.17.0.2
phase: Pending
podIP: 10.42.0.16
podIPs:
- ip: 10.42.0.16
qosClass: Guaranteed
startTime: "2022-09-14T16:13:26Z"