Channels
#ask-the-community
Title
# ask-the-community
n
Nan Qin
04/20/2023, 8:29 PMAttributeErrornode_executionFlyteFileNone Copy code
from attr import s
import flytekit
import flytekit.remote
import flytekit.configuration as flyte_config
from typing import List, NamedTuple, Optional
import dataclasses, dataclasses_json
@dataclasses_json.dataclass_json
@dataclasses.dataclass
class T2IO:
a: str
b: str
z: Optional[flytekit.types.file.FlyteFile] = None
@flytekit.task
def task1(x: int) -> NamedTuple('op', t1=List[T2IO]):
return [T2IO(str(t), str(-t)) for t in range(x)]
@flytekit.task
def task2(x: T2IO) -> NamedTuple('op', t2=T2IO):
return T2IO(x.a+'0', x.b+'0')
@flytekit.task
def task3(x: List[T2IO]) -> NamedTuple('op', t3=str):
return '-'.join([t.a for t in x]+[t.b for t in x])
@flytekit.workflow
def baby_training_wf(x: int) -> str:
t1, = task1(x=x).with_overrides(node_name='t-1')
t2 = flytekit.map_task(task2)(x=t1).with_overrides(node_name='t-2')
t3, = task3(x=t2).with_overrides(node_name='t-3')
return t3
if __name__ == '__main__':
print(baby_training_wf(x=3))execution.node_executions['t-2'].inputs['x'] Copy code
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
Cell In[308], line 1
----> 1 execution.node_executions['t-2'].inputs['x']
File /opt/homebrew/Caskroom/miniconda/base/envs/baby/lib/python3.10/site-packages/flytekit/core/type_engine.py:1691, in LiteralsResolver.__getitem__(self, key)
1688 if key in self._native_values:
1689 return self._native_values[key]
-> 1691 return self.get(key)
File /opt/homebrew/Caskroom/miniconda/base/envs/baby/lib/python3.10/site-packages/flytekit/core/type_engine.py:1720, in LiteralsResolver.get(self, attr, as_type)
1718 else:
1719 ValueError("as_type argument not supplied and Variable map not specified in LiteralsResolver")
-> 1720 val = TypeEngine.to_python_value(
1721 self._ctx or FlyteContext.current_context(), self._literals[attr], cast(Type, as_type)
1722 )
1723 self._native_values[attr] = val
1724 return val
File /opt/homebrew/Caskroom/miniconda/base/envs/baby/lib/python3.10/site-packages/flytekit/core/type_engine.py:831, in TypeEngine.to_python_value(cls, ctx, lv, expected_python_type)
827 """
828 Converts a Literal value with an expected python type into a python value.
829 """
830 transformer = cls.get_transformer(expected_python_type)
--> 831 return transformer.to_python_value(ctx, lv, expected_python_type)
File /opt/homebrew/Caskroom/miniconda/base/envs/baby/lib/python3.10/site-packages/flytekit/core/type_engine.py:1023, in ListTransformer.to_python_value(self, ctx, lv, expected_python_type)
1021 else:
1022 st = self.get_sub_type(expected_python_type)
-> 1023 return [TypeEngine.to_python_value(ctx, x, st) for x in lits]
File /opt/homebrew/Caskroom/miniconda/base/envs/baby/lib/python3.10/site-packages/flytekit/core/type_engine.py:1023, in <listcomp>(.0)
1021 else:
1022 st = self.get_sub_type(expected_python_type)
-> 1023 return [TypeEngine.to_python_value(ctx, x, st) for x in lits]
File /opt/homebrew/Caskroom/miniconda/base/envs/baby/lib/python3.10/site-packages/flytekit/core/type_engine.py:831, in TypeEngine.to_python_value(cls, ctx, lv, expected_python_type)
827 """
828 Converts a Literal value with an expected python type into a python value.
829 """
830 transformer = cls.get_transformer(expected_python_type)
--> 831 return transformer.to_python_value(ctx, lv, expected_python_type)
File /opt/homebrew/Caskroom/miniconda/base/envs/baby/lib/python3.10/site-packages/flytekit/core/type_engine.py:604, in DataclassTransformer.to_python_value(self, ctx, lv, expected_python_type)
602 json_str = _json_format.MessageToJson(lv.scalar.generic)
603 dc = cast(DataClassJsonMixin, expected_python_type).from_json(json_str)
--> 604 dc = self._fix_structured_dataset_type(expected_python_type, dc)
605 return self._fix_dataclass_int(expected_python_type, self._deserialize_flyte_type(dc, expected_python_type))
File /opt/homebrew/Caskroom/miniconda/base/envs/baby/lib/python3.10/site-packages/flytekit/core/type_engine.py:392, in DataclassTransformer._fix_structured_dataset_type(self, python_type, python_val)
390 for field in dataclasses.fields(python_type):
391 val = python_val.__getattribute__(field.name)
--> 392 python_val.__setattr__(field.name, self._fix_structured_dataset_type(field.type, val))
393 return python_val
File /opt/homebrew/Caskroom/miniconda/base/envs/baby/lib/python3.10/site-packages/flytekit/core/type_engine.py:391, in DataclassTransformer._fix_structured_dataset_type(self, python_type, python_val)
389 elif dataclasses.is_dataclass(python_type):
390 for field in dataclasses.fields(python_type):
--> 391 val = python_val.__getattribute__(field.name)
392 python_val.__setattr__(field.name, self._fix_structured_dataset_type(field.type, val))
393 return python_val
AttributeError: 'NoneType' object has no attribute 'path'execution.node_executions['t-2'].inputs Copy code
Partially converted to native values, call get(key, <type_hint>) to convert rest...
{
x:
collection {
literals {
scalar {
generic {
fields {
key: "z"
value {
null_value: NULL_VALUE
}
}
fields {
key: "b"
value {
string_value: "0"
}
}
fields {
key: "a"
value {
string_value: "0"
}
}
}
}
}
literals {
scalar {
generic {
fields {
key: "z"
value {
null_value: NULL_VALUE
}
}
fields {
key: "b"
value {
string_value: "-1"
}
}
fields {
key: "a"
value {
string_value: "1"
}
}
}
}
}
literals {
scalar {
generic {
fields {
key: "z"
value {
null_value: NULL_VALUE
}
}
fields {
key: "b"
value {
string_value: "-2"
}
}
fields {
key: "a"
value {
string_value: "2"
}
}
}
}
}
}
}Is there any method to use to get the node_execution inputs as python list or dict?
y
Yee
04/20/2023, 11:25 PM
you can force it by providing hints.
will that help? since flyte has its own internal typing system there is necessarily some typing lost
n
Nan Qin
04/21/2023, 1:39 AMWhere to put the type hints?
the tasks are already annotated, is there a way to add type hints to
execution.node_executionsy
Yee
04/21/2023, 3:45 PM
i mean when you get it.. the object you get back is a LiteralsResolver, which takes a map of type hints.
execution.node_executions['t-2'].inputs.get("x", type_hints={"x": int})n
Nan Qin
04/21/2023, 4:00 PM Copy code
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
Cell In[35], line 1
----> 1 execution.node_executions['t-2'].inputs.get('x', type_hints={"x": int})
TypeError: LiteralsResolver.get() got an unexpected keyword argument 'type_hints's
Samhita Alla
04/24/2023, 4:30 AM
I think the problem here is that accessing the inputs of a task downloads the values, and since FlyteFile's a None in this case, it's throwing an error. We should handle the Optional case when retrieving inputs/outputs with remote and I think this is a bug. @Yee / @Kevin Su, can you confirm?
n
Nan Qin
04/29/2023, 1:01 AM@Samhita Alla should I create a github issue for this?
s
Samhita Alla
04/30/2023, 5:40 AM
Please do!
21 Views