nutritious-addition-55433
10/24/2023, 10:07 AMnutritious-addition-55433
10/27/2023, 6:56 AMflaky-beach-18445
12/07/2023, 9:28 PMcool-lifeguard-49380
12/08/2023, 8:46 AMflaky-beach-18445
12/09/2023, 6:04 AMflaky-beach-18445
12/09/2023, 6:04 AMfrom typing import Type, Optional
from flyteidl.core.types_pb2 import SimpleType
from flytekit import LiteralType, FlyteContext, Literal, Scalar
from flytekit.extend import TypeTransformer, T
from google.protobuf.json_format import MessageToDict
from omegaconf import DictConfig, OmegaConf
from google.protobuf.struct_pb2 import Struct
class DictConfigTransformer(TypeTransformer[DictConfig]):
_TYPE_INFO = LiteralType(simple=SimpleType.STRUCT)
def to_literal(self, ctx: FlyteContext, python_val: DictConfig, python_type: Type[DictConfig],
expected: LiteralType) -> Literal:
s = Struct()
conf = OmegaConf.to_container(python_val)
s.update(conf)
return Literal(scalar=Scalar(generic=s))
def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: Type[DictConfig]) -> Optional[T]:
message = lv.scalar.generic
return OmegaConf.create(MessageToDict(message))
def get_literal_type(self, t: Type[T]) -> LiteralType:
return Type[DictConfig]
flaky-beach-18445
12/09/2023, 6:05 AMflaky-beach-18445
12/09/2023, 6:06 AMtasks, workflow, launch_plan = flyte_core.extract_flyte_entities_from_module(self.config.module_name)
flaky-beach-18445
12/09/2023, 6:20 AMcool-lifeguard-49380
12/09/2023, 7:27 AMthankful-flag-82043
12/13/2023, 6:23 PMflaky-beach-18445
12/13/2023, 10:27 PMflaky-beach-18445
12/13/2023, 10:28 PMdelightful-greece-6207
12/14/2023, 11:25 AMflaky-beach-18445
12/19/2023, 8:11 PMDictConf
as proposed in the video)
• in the main parse DictConf into a dataclass object, which is the same config represented as a dataclass
• pass dataclass object as an argument to the workflow
That elimitanes the need to deal with DictConf
serialization.
@dataclass_json
@dataclass
class Configuration:
trainer: TrainerConfig
pruner: PrunerConfig
flyte_environment: FlyteEnvironment
@workflow
def finetune_workflow(config: Configuration) -> None:
train(config=config)
@hydra.main(version_base=None, config_path='conf', config_name="default")
def main(cfg) -> None:
config = instantiate(cfg, _convert_="none")
finetune_workflow(config=config)