Srishti Mehra
10/07/2022, 3:21 PMclass LGBMDatasetTransformer(extend.TypeTransformer[lgb.Dataset]):
_TYPE_INFO = flytekit.BlobType(
format="binary", dimensionality=flytekit.BlobType.BlobDimensionality.MULTIPART
)
def __init__(self) -> None:
super(LGBMDatasetTransformer, self).__init__(name="lgbdataset-transform", t=lgb.Dataset)
def get_literal_type(self, t: Type[lgb.Dataset]) -> flytekit.LiteralType:
return flytekit.LiteralType(blob=self._TYPE_INFO)
def to_literal(
self,
ctx: flytekit.FlyteContext,
python_val: lgb.Dataset,
python_type: Type[lgb.Dataset],
expected: flytekit.LiteralType,
) -> flytekit.Literal:
"""
Convert from the given python type object ``lgb.Dataset`` to the Literal representation.
"""
# Step 1: Upload all the data into a remote place recommended by Flyte
local_file = ctx.file_access.get_random_local_path() + ".data"
lgb.Dataset.save_binary(python_val, local_file)
remote_dir = ctx.file_access.get_random_remote_directory()
ctx.file_access.upload_directory(local_file, remote_dir)
# Step 2: Return a pointer to this remote_dir in the form of a Literal
return flytekit.Literal(
scalar=flytekit.Scalar(
blob=flytekit.Blob(
uri=remote_dir, metadata=flytekit.BlobMetadata(type=self._TYPE_INFO)
)
)
)
def to_python_value(
self,
ctx: flytekit.FlyteContext,
lv: flytekit.Literal,
expected_python_type: Type[lgb.Dataset],
) -> lgb.Dataset:
"""
In this method, we want to be able to re-hydrate the custom object from Flyte Literal value.
"""
# Step 1: Download remote data locally
local_dir = ctx.file_access.get_random_local_directory()
ctx.file_access.download_directory(lv.scalar.blob.uri, local_dir)
# Step 2: Create the ``lgb.Dataset`` object
return lgb.Dataset.construct(lgb.Dataset(local_dir))
extend.TypeEngine.register(LGBMDatasetTransformer())
Ketan (kumare3)
Rahul Mehta
10/07/2022, 3:58 PMKetan (kumare3)
Rahul Mehta
10/07/2022, 3:59 PM[3/3] currentAttempt done. Last Error: SYSTEM::Traceback (most recent call last):
File "/app/thm/flyte/workflows/model_training/lgbm_model_test.image.py.binary.runfiles/prod_flytekit/flytekit/exceptions/scopes.py", line 165, in system_entry_point
return wrapped(*args, **kwargs)
File "/app/thm/flyte/workflows/model_training/lgbm_model_test.image.py.binary.runfiles/prod_flytekit/flytekit/core/base_task.py", line 473, in dispatch_execute
native_inputs = TypeEngine.literal_map_to_kwargs(exec_ctx, input_literal_map, self.python_interface.inputs)
File "/app/thm/flyte/workflows/model_training/lgbm_model_test.image.py.binary.runfiles/prod_flytekit/flytekit/core/type_engine.py", line 798, in literal_map_to_kwargs
return {k: TypeEngine.to_python_value(ctx, lm.literals[k], python_types[k]) for k, v in lm.literals.items()}
File "/app/thm/flyte/workflows/model_training/lgbm_model_test.image.py.binary.runfiles/prod_flytekit/flytekit/core/type_engine.py", line 798, in <dictcomp>
return {k: TypeEngine.to_python_value(ctx, lm.literals[k], python_types[k]) for k, v in lm.literals.items()}
File "/app/thm/flyte/workflows/model_training/lgbm_model_test.image.py.binary.runfiles/prod_flytekit/flytekit/core/type_engine.py", line 762, in to_python_value
return transformer.to_python_value(ctx, lv, expected_python_type)
File "/app/thm/flyte/workflows/model_training/lgbm_model_test.image.py.binary.runfiles/prod_flytekit/flytekit/types/pickle/pickle.py", line 59, in to_python_value
with open(uri, "rb") as infile:
Message:
[Errno 2] No such file or directory: '/tmp/flyte-ln4wnwph/raw/7ca592ea206ad1fc4fc3c279ac764d95/815803b1b0c20031e2708f676b4bc107'
TypeEngine.register(LGBMDatasetTransformer())
)Kevin Su
10/07/2022, 4:35 PMimport flytekit
...
TypeEngine.register(LGBMDatasetTransformer())
@workflow
def wf():
...
Rahul Mehta
10/07/2022, 6:26 PMKevin Su
10/07/2022, 7:00 PMKetan (kumare3)
Rahul Mehta
10/07/2022, 7:20 PMKetan (kumare3)