<#73 Dask Plugin> Pull request opened by <pingsutw...
# flyte-github
c
#73 Dask Plugin Pull request opened by pingsutw import asyncio import typing from pathlib import Path from distributed import Client from flyteplugins.dask import Dask, Scheduler, WorkerGroup import flyte.remote import flyte.storage from flyte import Resources image = ( flyte.Image.from_debian_base(python_version=(3, 12)) .with_env_vars({"UV_COMPILE_BYTECODE": "0"}) .with_workdir("/root") .with_apt_packages("wget") .with_pip_packages("dask[distributed]") .with_source_folder(Path(file).parent.parent.parent / "plugins/dask", "./dask") .with_env_vars({"PYTHONPATH": "./dask/src:${PYTHONPATH}", "hello": "world2"}) ) dask_config = Dask( scheduler=Scheduler(resources=Resources(cpu="1", memory="1Gi")), workers=WorkerGroup(number_of_workers=4, resources=Resources(cpu="1", memory="1Gi")), ) task_env = flyte.TaskEnvironment( name="hello_dask", resources=Resources(cpu=(1, 2), memory=("400Mi", "1000Mi")), image=image ) dask_env = flyte.TaskEnvironment( name="dask_env", plugin_config=dask_config, image=image, resources=Resources(cpu="1", memory="1Gi"), ) @task_env.task() async def hello_dask(): await asyncio.sleep(5) print("Hello from the Dask task!") @dask_env.task async def hello_dask_nested(n: int = 3) -> typing.List[int]: print("running dask task") t = asyncio.create_task(hello_dask()) client = Client() futures = client.map(lambda x: x + 1, range(n)) res = client.gather(futures) await t return res if name == "__main__": flyte.init_from_config("../../config.yaml") run = flyte.run(hello_dask_nested, n=3) print("run name:", run.name) print("run url:", run.url) run.wait(run) action_details = flyte.remote.ActionDetails.get(run_name=run.name, name="a0") for log in action_details.pb2.attempts[-1].log_info: print(f"{log.name}: {log.uri}") flyteorg/flyte-sdk