acoustic-carpenter-78188
08/29/2023, 12:55 PMtry:
from typing import Annotated
except ImportError:
from typing_extensions import Annotated
import pandas as pd
from flytekit import StructuredDataset, kwtypes, task, workflow
from flytekitplugins.snowflake import SnowflakeConfig, SnowflakeTask
from flytekit.types.schema import FlyteSchema
# we need to type our account, and user, and prepare the private key to build a connection.
# Note that, the table field is for building the correct uri for future reference.
snowflake_task_templatized_query2 = SnowflakeTask(
name="sf",
# Define inputs as well as their types that can be used to customize the query.1
inputs=kwtypes(number=int),
output_schema_type=StructuredDataset ,
task_config=SnowflakeConfig( user='hhcs9527',
account='ed61997.europe-west4.gcp',
database='AGENT',
schema='PEOPLE',
table='INTRO',
warehouse='COMPUTE_WH'),
query_template="SELECT AGE, NAME from INTRO limit %(number)s",
)
@task
def convert_bq_table_to_pandas_dataframe(sd: DogeCoinDataset) -> pd.DataFrame:
return sd.open(pd.DataFrame).all()
@workflow
def sf_query_wf(number: int) -> pd.DataFrame:
sd = snowflake_task_templatized_query2(number=number)
return convert_bq_table_to_pandas_dataframe(sd=sd)
if __name__ == "__main__":
print(sf_query_wf(number=2)
Tracking Issue
Flyte Agent Ecosystem
Follow-up issue
Add encoder/decoder in structureDataset for snowflake.
flyteorg/flytekit
✅ All checks have passed
30/30 successful checksacoustic-carpenter-78188
08/29/2023, 12:55 PMacoustic-carpenter-78188
09/15/2023, 12:57 AM