salmon-refrigerator-32115
01/06/2023, 6:46 PM@task(
container_image="<http://xyz.dkr.ecr.us-east-1.amazonaws.com/flyte-pyspark:latest|xyz.dkr.ecr.us-east-1.amazonaws.com/flyte-pyspark:latest>",
task_config=Spark(
spark_conf={...
}
),
)
def read_spark_df() -> pandas.DataFrame:
sess = flytekit.current_context().spark_session
spark_df = sess.read.parquet("<s3a://bucket/key.parquet>").toPandas()
....