billions-midnight-10687
10/10/2023, 4:15 PMglamorous-carpet-83516
10/10/2023, 4:38 PMhigh-accountant-32689
10/10/2023, 5:30 PMbillions-midnight-10687
10/10/2023, 10:58 PMbillions-midnight-10687
10/10/2023, 10:58 PMbillions-midnight-10687
10/10/2023, 10:58 PMbillions-midnight-10687
10/11/2023, 12:24 AMbillions-midnight-10687
10/11/2023, 12:25 AMbillions-midnight-10687
10/11/2023, 12:25 AM{"json":{"routine":"databricks-worker-0","src":"cache.go:78"},"level":"debug","msg":"Sync loop - processing resource with cache key [fd06daeb1efbc438390b-n0-0]","ts":"2023-10-11T00:22:53Z"}
{"json":{"routine":"databricks-worker-0","src":"cache.go:101"},"level":"debug","msg":"Querying AsyncPlugin for fd06daeb1efbc438390b-n0-0","ts":"2023-10-11T00:22:53Z"}
{"json":{"routine":"databricks-worker-0","src":"cache.go:104"},"level":"info","msg":"Error retrieving resource [fd06daeb1efbc438390b-n0-0]. Error: [CorruptedPluginState] can't get the job
billions-midnight-10687
10/11/2023, 12:25 AMhigh-accountant-32689
10/11/2023, 12:26 AMglamorous-carpet-83516
10/11/2023, 12:30 AMbillions-midnight-10687
10/11/2023, 12:47 AMbillions-midnight-10687
10/11/2023, 12:47 AMbillions-midnight-10687
10/11/2023, 12:48 AMbillions-midnight-10687
10/11/2023, 12:48 AMbillions-midnight-10687
10/11/2023, 12:48 AMbillions-midnight-10687
10/11/2023, 12:48 AMbillions-midnight-10687
10/11/2023, 12:48 AM{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"ns": "flytesnacks-development",
"routine": "worker-0",
"src": "handler.go:181"
},
"level": "info",
"msg": "Processing Workflow.",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "executor.go:364",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "info",
"msg": "Handling Workflow [f5661a9a05a7b485b956], id: [project:\"flytesnacks\" domain:\"development\" name:\"f5661a9a05a7b485b956\" ], p [Running]",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"node": "start-node",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "executor.go:1024",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "debug",
"msg": "Node has [Succeeded], traversing downstream.",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "executor.go:826",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "debug",
"msg": "Handling downstream Nodes",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"node": "n0",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "executor.go:986",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "debug",
"msg": "Handling node Status [Running]",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "executor.go:959",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "debug",
"msg": "Parallelism criteria not met, Current [0], Max [25]",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"node": "n0",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "executor.go:740",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "debug",
"msg": "Handling Node [n0]",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"node": "n0",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "executor.go:586",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "debug",
"msg": "node executing, current phase [Running]",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"node": "n0",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "executor.go:457",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "debug",
"msg": "Executing node",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"node": "n0",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "handler.go:190",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "info",
"msg": "Dynamic handler.Handle's called with phase 0.",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"node": "n0",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "handler.go:348",
"tasktype": "spark",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "debug",
"msg": "Plugin [databricks] resolved for Handler type [spark]",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"node": "n0",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "pre_post_execution.go:117",
"tasktype": "spark",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "info",
"msg": "Catalog CacheSerializeDisabled: for Task [flytesnacks/development/dbx_simplified_example.print_spark_config/8TWBtAb_p0C_8jXnVr8IyA==]",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"node": "n0",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "secrets.go:41",
"tasktype": "spark",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "debug",
"msg": "reading secrets from filePath [/vault/secrets/databricks_api_token]",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"node": "n0",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "template.go:87",
"tasktype": "spark",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "debug",
"msg": "Using [f5661a9a05a7b485b956_n0_0] from [f5661a9a05a7b485b956-n0-0]",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"node": "n0",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "launcher.go:23",
"tasktype": "spark",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "info",
"msg": "Created Resource Name [f5661a9a05a7b485b956-n0-0] and Meta [&{<nil> xxx.cloud.databricks.com token_xxx}]",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"node": "n0",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "launcher.go:27",
"tasktype": "spark",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "error",
"msg": "Failed to check resource status. Error: [SystemError] unknown execution phase [403].",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"node": "n0",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "handler.go:400",
"tasktype": "spark",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "warning",
"msg": "Runtime error from plugin [databricks]. Error: [SystemError] unknown execution phase [403].",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"node": "n0",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "handler.go:233",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "error",
"msg": "handling parent node failed with error: failed at Node[n0]. RuntimeExecutionError: failed during plugin execution, caused by: failed to execute handle for plugin [databricks]: [SystemError] unknown execution phase [403].",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"node": "n0",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "executor.go:462",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "debug",
"msg": "Node execution round complete",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"node": "n0",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "executor.go:595",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "error",
"msg": "failed Execute for node. Error: failed at Node[n0]. RuntimeExecutionError: failed during plugin execution, caused by: failed to execute handle for plugin [databricks]: [SystemError] unknown execution phase [403].",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"node": "n0",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "executor.go:596",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "debug",
"msg": "node execution completed",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"node": "n0",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "executor.go:820",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "debug",
"msg": "Completed node [n0]",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "executor.go:395",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "warning",
"msg": "Error in handling running workflow [failed at Node[n0]. RuntimeExecutionError: failed during plugin execution, caused by: failed to execute handle for plugin [databricks]: [SystemError] unknown execution phase [403].]",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "executor.go:396",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "info",
"msg": "Handling Workflow [f5661a9a05a7b485b956] Done",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"ns": "flytesnacks-development",
"res_ver": "527200620",
"routine": "worker-0",
"src": "handler.go:145",
"wf": "flytesnacks:development:dbx_simplified_example.my_databricks_job"
},
"level": "error",
"msg": "Error when trying to reconcile workflow. Error [failed at Node[n0]. RuntimeExecutionError: failed during plugin execution, caused by: failed to execute handle for plugin [databricks]: [SystemError] unknown execution phase [403].]. Error Type[*errors.NodeErrorWithCause]",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"ns": "flytesnacks-development",
"routine": "worker-0",
"src": "passthrough.go:89"
},
"level": "debug",
"msg": "Observed FlyteWorkflow Update (maybe finalizer)",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"ns": "flytesnacks-development",
"routine": "worker-0",
"src": "passthrough.go:109"
},
"level": "debug",
"msg": "Updated workflow.",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"src": "controller.go:159"
},
"level": "info",
"msg": "==> Enqueueing workflow [flytesnacks-development/f5661a9a05a7b485b956]",
"ts": "2023-10-11T00:44:56Z"
}
{
"json": {
"exec_id": "f5661a9a05a7b485b956",
"ns": "flytesnacks-development",
"routine": "worker-0",
"src": "handler.go:367"
},
"level": "info",
"msg": "Completed processing workflow.",
"ts": "2023-10-11T00:44:56Z"
}
billions-midnight-10687
10/11/2023, 12:48 AMbillions-midnight-10687
10/11/2023, 12:56 AMbillions-midnight-10687
10/11/2023, 1:00 AMglamorous-carpet-83516
10/11/2023, 5:16 AMglamorous-carpet-83516
10/11/2023, 5:17 AMcurl --netrc --request GET --header "Authorization: Bearer $DATABRICKS_TOKEN" \
'https://<databricks_instance>.<http://cloud.databricks.com/api/2.0/jobs/runs/get?run_id=<job_id|cloud.databricks.com/api/2.0/jobs/runs/get?run_id=<job_id>>'
use the token you pass to flytebillions-midnight-10687
10/11/2023, 9:42 AMbillions-midnight-10687
10/11/2023, 9:42 AMbillions-midnight-10687
10/11/2023, 3:34 PMglamorous-carpet-83516
10/11/2023, 4:46 PMbillions-midnight-10687
10/11/2023, 9:32 PMglamorous-carpet-83516
10/12/2023, 6:40 AMglamorous-carpet-83516
10/12/2023, 6:40 AMglamorous-carpet-83516
10/12/2023, 6:43 AMbillions-midnight-10687
10/12/2023, 8:50 AMbillions-midnight-10687
10/12/2023, 8:50 AMglamorous-carpet-83516
10/12/2023, 8:59 AMbillions-midnight-10687
10/12/2023, 10:21 AMbillions-midnight-10687
10/12/2023, 10:21 AMbillions-midnight-10687
10/12/2023, 10:21 AMbillions-midnight-10687
10/12/2023, 10:21 AMbillions-midnight-10687
10/12/2023, 10:22 AMbillions-midnight-10687
10/12/2023, 10:22 AMbillions-midnight-10687
10/12/2023, 10:23 AM{
"json": {
"routine": "databricks-worker-0",
"src": "plugin.go:163"
},
"level": "info",
"msg": "Get databricks job requestreq&{GET <https://xxx.cloud.databricks.com/api/2.0/jobs/runs/get?run_id=2.57406661136039e+14> HTTP/1.1 1 1 map[Authorization:[Bearer xxx] Content-Type:[application/json]] <nil> <nil> 0 [] false <http://xxx.cloud.databricks.com|xxx.cloud.databricks.com> map[] map[] <nil> map[] <nil> <nil> <nil> 0xc000060020}",
"ts": "2023-10-12T10:11:26Z"
}
billions-midnight-10687
10/12/2023, 10:23 AMbillions-midnight-10687
10/12/2023, 10:24 AMbillions-midnight-10687
10/12/2023, 10:25 AMbillions-midnight-10687
10/12/2023, 10:25 AM{
"json": {
"routine": "databricks-worker-4",
"src": "plugin.go:165"
},
"level": "info",
"msg": "Get databricks job responseresp&{400 Bad Request 400 HTTP/2.0 2 0 map[Content-Type:[application/json] Date:[Thu, 12 Oct 2023 10:10:56 GMT] Server:[databricks] Strict-Transport-Security:[max-age=31536000; includeSubDomains; preload] Vary:[Accept-Encoding] X-Content-Type-Options:[nosniff] X-Databricks-Org-Id:[xxx]] 0xc002455290 -1 [] false true map[] 0xc00538a600 0xc00c5c5ad0}",
"ts": "2023-10-12T10:10:56Z"
}
billions-midnight-10687
10/12/2023, 10:26 AMbillions-midnight-10687
10/12/2023, 10:26 AM{
"json": {
"routine": "databricks-worker-4",
"src": "cache.go:114"
},
"level": "info",
"msg": "Error retrieving resource [f96d0eebbeef841c0ac3-n0-0]. Error: [CorruptedPluginState] can't get the job state",
"ts": "2023-10-12T10:10:56Z"
}
billions-midnight-10687
10/12/2023, 10:29 AM{
"json": {
"routine": "databricks-worker-0",
"src": "plugin.go:155"
},
"level": "info",
"msg": "Get databricks job statusrunID2.57406661136039e+14",
"ts": "2023-10-12T10:11:26Z"
}
billions-midnight-10687
10/12/2023, 10:29 AMglamorous-carpet-83516
10/12/2023, 10:30 AMglamorous-carpet-83516
10/12/2023, 10:30 AMbillions-midnight-10687
10/12/2023, 10:30 AMbillions-midnight-10687
10/12/2023, 10:30 AMbillions-midnight-10687
10/12/2023, 10:30 AMbillions-midnight-10687
10/12/2023, 10:30 AMglamorous-carpet-83516
10/12/2023, 10:31 AMbillions-midnight-10687
10/12/2023, 10:31 AMbillions-midnight-10687
10/12/2023, 10:32 AMbillions-midnight-10687
10/12/2023, 10:32 AMglamorous-carpet-83516
10/12/2023, 10:33 AMglamorous-carpet-83516
10/12/2023, 10:34 AMuser
10/12/2023, 10:34 AMbillions-midnight-10687
10/12/2023, 11:00 AMbillions-midnight-10687
10/13/2023, 9:02 AMglamorous-carpet-83516
10/13/2023, 5:58 PMglamorous-carpet-83516
10/13/2023, 6:03 PMbillions-midnight-10687
10/13/2023, 7:08 PMglamorous-carpet-83516
10/13/2023, 7:12 PMbillions-midnight-10687
10/13/2023, 10:47 PMglamorous-carpet-83516
10/13/2023, 10:47 PMbillions-midnight-10687
10/13/2023, 10:47 PMbillions-midnight-10687
10/13/2023, 10:47 PMglamorous-carpet-83516
10/13/2023, 10:48 PMbillions-midnight-10687
10/13/2023, 10:48 PMbillions-midnight-10687
10/13/2023, 10:48 PMbillions-midnight-10687
10/13/2023, 10:48 PMglamorous-carpet-83516
10/13/2023, 10:49 PMbillions-midnight-10687
10/13/2023, 10:49 PMbillions-midnight-10687
10/13/2023, 10:49 PMglamorous-carpet-83516
10/13/2023, 10:50 PMglamorous-carpet-83516
10/13/2023, 10:50 PMcurl --netrc --request GET --header "Authorization: Bearer $DATABRICKS_TOKEN" \
'https://<databricks_instance>.<http://cloud.databricks.com/api/2.0/jobs/runs/get?run_id=<job_id|cloud.databricks.com/api/2.0/jobs/runs/get?run_id=<job_id>>'
billions-midnight-10687
10/13/2023, 10:51 PMbillions-midnight-10687
10/13/2023, 10:51 PMbillions-midnight-10687
10/13/2023, 10:52 PMbillions-midnight-10687
10/13/2023, 10:52 PM"number_in_job": 1028557138752855,
"state": {
"life_cycle_state": "INTERNAL_ERROR",
"result_state": "FAILED",
billions-midnight-10687
10/13/2023, 10:54 PMbillions-midnight-10687
10/13/2023, 10:57 PMbillions-midnight-10687
10/13/2023, 10:58 PMbillions-midnight-10687
10/13/2023, 10:58 PMNo state change for Task, previously observed same transition. Short circuiting
billions-midnight-10687
10/13/2023, 11:01 PMbillions-midnight-10687
10/13/2023, 11:08 PMbillions-midnight-10687
10/13/2023, 11:08 PMbillions-midnight-10687
10/13/2023, 11:09 PMbillions-midnight-10687
10/13/2023, 11:11 PMbillions-midnight-10687
10/13/2023, 11:12 PMbillions-midnight-10687
10/13/2023, 11:22 PMhttp.StatusOK:
case should be something like this:billions-midnight-10687
10/13/2023, 11:23 PMbillions-midnight-10687
10/13/2023, 11:24 PM