salmon-refrigerator-32115
01/06/2023, 6:46 PM@task(
container_image="xyz.dkr.ecr.us-east-1.amazonaws.com/flyte-pyspark:latest",
task_config=Spark(
spark_conf={...
}
),
)
def read_spark_df() -> pandas.DataFrame:
sess = flytekit.current_context().spark_session
spark_df = sess.read.parquet("s3a://bucket/key.parquet").toPandas()
....