#5962 [Housekeeping] flytekit cli unit tests are flaky Issue created by eapolinario Describe the issue As of the last 10 days we've seen sporadic failures of some of the

pyflyte

unit tests (e.g. https://github.com/flyteorg/flytekit/actions/runs/11688067523/job/32547641613#step:7:3443):

_____________ test_pyflyte_run_wf[WorkflowFileLocation.NORMAL--r] ______________
[gw0] linux -- Python 3.9.20 /opt/hostedtoolcache/Python/3.9.20/x64/bin/python

remote = <flytekit.remote.remote.FlyteRemote object at 0x7fbf003ca8b0>
remote_flag = '-r'
workflow_file = '/home/runner/work/flytekit/flytekit/tests/flytekit/unit/cli/pyflyte/workflow.py'

    @pytest.mark.parametrize(
        "remote_flag",
        [
            "-r",
            "--remote",
        ],
    )
    @pytest.mark.parametrize(
        "workflow_file",
        [
            WorkflowFileLocation.NORMAL,
            WorkflowFileLocation.TEMP_DIR,
        ],
        indirect=["workflow_file"],
    )
    def test_pyflyte_run_wf(remote, remote_flag, workflow_file):
        with mock.patch("flytekit.configuration.plugin.FlyteRemote"):
            runner = CliRunner()
>           result = runner.invoke(
                pyflyte.main,
                ["run", remote_flag, workflow_file, "my_wf", "--help"],
                catch_exceptions=False,
            )

tests/flytekit/unit/cli/pyflyte/test_run.py:91: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <click.testing.CliRunner object at 0x7fbf003d1220>
cli = <ErrorHandlingCommand pyflyte>
args = ['run', '-r', '/home/runner/work/flytekit/flytekit/tests/flytekit/unit/cli/pyflyte/workflow.py', 'my_wf', '--help']
input = None, env = None, catch_exceptions = False, color = False, extra = {}
exc_info = (<class 'SystemExit'>, SystemExit(0), <traceback object at 0x7fbf003cbb00>)
outstreams = (<_io.BytesIO object at 0x7fbf00b28b80>, None), return_value = None
exception = None, exit_code = 0, prog_name = 'pyflyte'

    def invoke(
        self,
        cli: "BaseCommand",
        args: t.Optional[t.Union[str, t.Sequence[str]]] = None,
        input: t.Optional[t.Union[str, bytes, <http://t.IO[t.Any]|t.IO[t.Any]>]] = None,
        env: t.Optional[t.Mapping[str, t.Optional[str]]] = None,
        catch_exceptions: bool = True,
        color: bool = False,
        **extra: t.Any,
    ) -> Result:
        """Invokes a command in an isolated environment.  The arguments are
        forwarded directly to the command line script, the `extra` keyword
        arguments are passed to the :meth:`~clickpkg.Command.main` function of
        the command.
    
        This returns a :class:`Result` object.
    
        :param cli: the command to invoke
        :param args: the arguments to invoke. It may be given as an iterable
                     or a string. When given as string it will be interpreted
                     as a Unix shell command. More details at
                     :func:`shlex.split`.
        :param input: the input data for `sys.stdin`.
        :param env: the environment overrides.
        :param catch_exceptions: Whether to catch any other exceptions than
                                 ``SystemExit``.
        :param extra: the keyword arguments to pass to :meth:`main`.
        :param color: whether the output should contain color codes. The
                      application can still override this explicitly.
    
        .. versionchanged:: 8.0
            The result object has the ``return_value`` attribute with
            the value returned from the invoked command.
    
        .. versionchanged:: 4.0
            Added the ``color`` parameter.
    
        .. versionchanged:: 3.0
            Added the ``catch_exceptions`` parameter.
    
        .. versionchanged:: 3.0
            The result object has the ``exc_info`` attribute with the
            traceback if available.
        """
        exc_info = None
        with self.isolation(input=input, env=env, color=color) as outstreams:
            return_value = None
            exception: t.Optional[BaseException] = None
            exit_code = 0
    
            if isinstance(args, str):
                args = shlex.split(args)
    
            try:
                prog_name = extra.pop("prog_name")
            except KeyError:
                prog_name = self.get_default_prog_name(cli)
    
            try:
                return_value = cli.main(args=args or (), prog_name=prog_name, **extra)
            except SystemExit as e:
                exc_info = sys.exc_info()
                e_code = t.cast(t.Optional[t.Union[int, t.Any]], e.code)
    
                if e_code is None:
                    e_code = 0
    
                if e_code != 0:
                    exception = e
    
                if not isinstance(e_code, int):
                    sys.stdout.write(str(e_code))
                    sys.stdout.write("\n")
                    e_code = 1
    
                exit_code = e_code
    
            except Exception as e:
                if not catch_exceptions:
                    raise
                exception = e
                exit_code = 1
                exc_info = sys.exc_info()
            finally:
                sys.stdout.flush()
>               stdout = outstreams[0].getvalue()
E               ValueError: I/O operation on closed file.

In pallets/click#824 (comment) someone mentioned that pytest and click's capturing of stdout+stderr do not mix well. What if we do not do this? Unit tests should run fast and not flake. Related component(s) flytekit Are you sure this issue hasn't been raised already? • Yes Have you read the Code of Conduct? • Yes flyteorg/flyte