Jose Navarro
09/27/2023, 4:31 PM# This is the code I want to move out
def do_something_with_data(data: pd.DataFrame):
# Some complex logic
return data
@task
def get_data() -> pd.DataFrame:
"""Get the wine dataset."""
return load_wine(as_frame=True).frame
@task
def process_data(data: pd.DataFrame) -> pd.DataFrame:
new_data = do_something_with_data(data)
return new_data
@task
def train_model(data: pd.DataFrame, hyperparameters: dict) -> LogisticRegression:
#model.fit()
return model
@workflow
def training_workflow(hyperparameters: dict) -> LogisticRegression:
data = get_data()
processed_data = process_data(data=data)
return train_model(
data=processed_data,
hyperparameters=hyperparameters,
)
but if I move the function out to a separate module, I can't get it to work:
try:
from module import do_something_with_data
except ImportError:
from .module import do_something_with_data
@task
def get_data() -> pd.DataFrame:
"""Get the wine dataset."""
return load_wine(as_frame=True).frame
@task
def process_data(data: pd.DataFrame) -> pd.DataFrame:
new_data = do_something_with_data(data)
return new_data
@task
def train_model(data: pd.DataFrame, hyperparameters: dict) -> LogisticRegression:
#model.fit()
return model
@workflow
def training_workflow(hyperparameters: dict) -> LogisticRegression:
data = get_data()
processed_data = process_data(data=data)
return train_model(
data=processed_data,
hyperparameters=hyperparameters,
)
Michael Tinsley
09/27/2023, 5:26 PMJose Navarro
09/27/2023, 7:08 PMKevin Su
09/27/2023, 10:47 PMpyflyte run --remote --copy-all …
, it will copy your other module to s3 as well.Jose Navarro
09/28/2023, 8:56 AMcopy-all