I have a repo with the following structure

.
├── local_scripts
│   ├── program1_input.csv
│   └── program1_quickrun.py
├── pyproject.toml
├── README.md
└── src
    └── internal_flytekit
        ├── __init__.py
        ├── tasks
        │   ├── __init__.py
        │   └── program1.py
        ├── utils.py
        └── workflows
            ├── __init__.py
            └── program1.py

I'm running short test scripts in

local_scripts

via

python local_scripts/program1_quickrun.py

The contents of

local_scripts/program1_quickrun.py

are very much like:

#!/usr/bin/env python

import os

FLYTE_TASK_VERSION = "17.4.0-alpha.2.program1"
os.environ["JNJ_FLYTE_IMAGE_VERSION"] = FLYTE_TASK_VERSION

from flytekit import Config, FlyteFile, FlyteRemote

from internal_flytekit.workflows.program1 import (
    run_program1_wf,
)


def main() -> None:
    remote = FlyteRemote(
        config=Config.for_endpoint(endpoint="<http://flyteserver.internal.com|flyteserver.internal.com>"),
        default_project="flyteproject",
        default_domain="predev",
        interactive_mode_enabled=True,
    )
    exe = remote.execute(
        run_program1_wf,
        inputs={
            "input_csv": FlyteFile("local_scripts/program1_input.csv"),
        },
    )
    print("job", exe.execution_url)

if __name__ == "__main__":
    main()

When i run this workflow directly via pyflyte run: it works fine because it's packaging the relevant parts of internal_flytekit in a tar.gz file and serving that to the worker. but when i just run

python local_scripts/program1_quickrun.py

it is only packaging a pkl file and the task fails with the error:

Exception when loading_task, reason USER:RuntimeError: error=No module named 'internal_flytekit', cause=No module named 'internal_flytekit'

Is there something that I'm doing wrong here? or some way to convince the python script to package the whole repo or the relevant parts of the repo with the task?

@average-secretary-61436 this is really simpler in v2. in v1 it is pickling, but pickling may not work with other modules. I think you might have to copy mode all. but not 100% sure

Came up with this solution:

#!/usr/bin/env python
""" """


import os
from typing import Any
from pathlib import Path

from flytekit.configuration import ImageConfig
from flytekit.constants import CopyFileDetection
from flytekit.remote.remote import FastPackageOptions

from flytekit import Config, FlyteFile, FlyteRemote, Workflow

os.environ["INTERNAL_FLYTE_IMAGE_VERSION"] = FLYTE_TASK_VERSION
FLYTE_TASK_VERSION = "17.4.0-alpha.2.program1"

from internal_flytekit.workflows.immunebuilder import (
    program1,
)


def run_workflow(
    remote: FlyteRemote,
    workflow: Workflow,
    project: str,
    domain: str,
    default_image: ImageConfig,
    inputs: dict[str, Any],
):
    fast_package_options=FastPackageOptions(ignores=[], keep_default_ignores=True, copy_style=CopyFileDetection.LOADED_MODULES, show_files=False)
    remote_entity = remote.register_script(
        workflow,
        project=project,
        domain=domain,
        image_config=default_image,
        destination_dir=".",
        source_path=str(Path("src").absolute()),
        module_name=workflow.__module__,
        fast_package_options=fast_package_options,
    )
    return remote.execute(
        remote_entity,
        inputs=inputs,
        project=project,
        domain=domain,
    )


def main() -> None:
    project = "flyteproject"
    domain = "predev"
    remote = FlyteRemote(
        config=Config.for_endpoint(endpoint="<http://flyteserver.internal.com|flyteserver.internal.com>"),
        default_domain=domain,
        default_project=project,
        # interactive_mode_enabled=True,
    )
    default_image = ImageConfig.validate_image(None, param="image", values=(f"<http://repo.internal.com/internal-flytekit:{FLYTE_TASK_VERSION}|repo.internal.com/internal-flytekit:{FLYTE_TASK_VERSION}>",))
    exe = run_workflow(
        remote=remote,
        workflow=run_program1_wf,
        project=project,
        domain=domain,
        default_image=default_image,
        inputs={
            "input_csv": FlyteFile("local_scripts/program1_input.csv"),
        },
    )
    print("exec", exe.execution_url)


if __name__ == "__main__":
    main()