Alex Lyashok
01/11/2024, 11:30 PM__init__(
File "/usr/local/lib/python3.10/dist-packages/flytekit/core/python_auto_container.py", line 90, in __init__
super().__init__(
File "/usr/local/lib/python3.10/dist-packages/flytekit/core/base_task.py", line 442, in __init__
interface=transform_interface_to_typed_interface(interface),
File "/usr/local/lib/python3.10/dist-packages/flytekit/core/interface.py", line 248, in transform_interface_to_typed_interface
outputs_map = transform_variable_map(interface.outputs, output_descriptions)
File "/usr/local/lib/python3.10/dist-packages/flytekit/core/interface.py", line 345, in transform_variable_map
res[k] = transform_type(v, descriptions.get(k, k))
File "/usr/local/lib/python3.10/dist-packages/flytekit/core/interface.py", line 350, in transform_type
return _interface_models.Variable(type=TypeEngine.to_literal_type(x), description=description)
File "/usr/local/lib/python3.10/dist-packages/flytekit/core/type_engine.py", line 1016, in to_literal_type
transformer = cls.get_transformer(python_type)
File "/usr/local/lib/python3.10/dist-packages/flytekit/core/type_engine.py", line 912, in get_transformer
cls.lazy_import_transformers()
File "/usr/local/lib/python3.10/dist-packages/flytekit/core/type_engine.py", line 998, in lazy_import_transformers
from flytekit.types import schema # noqa: F401
File "/usr/local/lib/python3.10/dist-packages/flytekit/types/schema/__init__.py", line 1, in <module>
from .types import (
File "/usr/local/lib/python3.10/dist-packages/flytekit/types/schema/types.py", line 184, in <module>
class FlyteSchema(DataClassJSONMixin):
File "/usr/local/lib/python3.10/dist-packages/flytekit/types/schema/types.py", line 273, in FlyteSchema
self, dataframe_fmt: type = pandas.DataFrame, override_mode: typing.Optional[SchemaOpenMode] = None
AttributeError: module 'pandas' has no attribute 'DataFrame'
Yee
Alex Lyashok
01/12/2024, 12:50 AML godlike
01/12/2024, 12:51 AMpip show flytekit
Alex Lyashok
01/12/2024, 12:51 AML godlike
01/12/2024, 12:52 AML godlike
01/12/2024, 12:52 AMAlex Lyashok
01/12/2024, 12:52 AMYee
L godlike
01/12/2024, 12:52 AMAlex Lyashok
01/12/2024, 12:52 AML godlike
01/12/2024, 12:52 AMAlex Lyashok
01/12/2024, 12:53 AMAlex Lyashok
01/12/2024, 12:53 AMAlex Lyashok
01/12/2024, 12:53 AML godlike
01/12/2024, 12:53 AML godlike
01/12/2024, 12:53 AML godlike
01/12/2024, 12:53 AMAlex Lyashok
01/12/2024, 12:53 AML godlike
01/12/2024, 12:54 AMAlex Lyashok
01/12/2024, 12:54 AML godlike
01/12/2024, 12:54 AMAlex Lyashok
01/12/2024, 12:54 AMFROM <http://nvcr.io/nvidia/pytorch:23.10-py3|nvcr.io/nvidia/pytorch:23.10-py3>
# 530 driver 12.2
USER root
WORKDIR /root
ENV VENV /opt/venv
ENV LANG C.UTF-8
ENV LC_ALL C.UTF-8
ENV PYTHONPATH /root
RUN apt-get update && apt-get install -y sudo
RUN apt-get update && apt-get install -y sudo build-essential
RUN apt-get install -y git wget curl mc gcc g++ vim ffmpeg libsm6 libxext6 make tesseract-ocr sqlite3 iputils-ping netcat apt-transport-https ca-certificates
RUN python -m pip install --upgrade pip
RUN pip install --extra-index-url=<https://pypi.nvidia.com> cudf-cu12==23.12.*
COPY requirements.txt /root
RUN pip install -r /root/requirements.txt
COPY . /root
ARG tag
ENV FLYTE_INTERNAL_IMAGE $tag
L godlike
01/12/2024, 12:54 AMAlex Lyashok
01/12/2024, 12:54 AMflytekit>=1.5.0
flytekitplugins.flyin
flytekitplugins-deck-standard
langchain
openai
unstructured
pypdf
pdf2image
pymupdf
pdfplumber
pypdfium2
llama-index==0.9.22
llama-hub
opencv-python==4.8.0.74
opencv-contrib-python==4.8.0.74
pytesseract
google-cloud-vision
googlemaps
earthengine-api
geopy
geopandas
shapely
mahotas
tritonclient[all]
qdrant-client
bertopic
bertopic[spacy]
spacy
tiktoken
tabulate
keybert
rake_nltk
yake
git+<https://github.com/boudinfl/pke.git>
azure-identity
azure-storage-blob
azure-ai-ml
azure-identity
azure-ai-formrecognizer
aim
Alex Lyashok
01/12/2024, 12:55 AML godlike
01/12/2024, 12:55 AMflytekit --pre
L godlike
01/12/2024, 12:55 AMAlex Lyashok
01/12/2024, 12:55 AML godlike
01/12/2024, 12:55 AMAlex Lyashok
01/12/2024, 12:56 AML godlike
01/12/2024, 12:56 AMflytekit>=1.5.0
to
flytekit --pre
Alex Lyashok
01/12/2024, 12:56 AML godlike
01/12/2024, 12:56 AMAlex Lyashok
01/12/2024, 12:56 AML godlike
01/12/2024, 12:57 AMAlex Lyashok
01/12/2024, 1:14 AMAlex Lyashok
01/12/2024, 1:15 AM__init__(
File "/usr/local/lib/python3.10/dist-packages/flytekit/core/python_auto_container.py", line 90, in __init__
super().__init__(
File "/usr/local/lib/python3.10/dist-packages/flytekit/core/base_task.py", line 442, in __init__
interface=transform_interface_to_typed_interface(interface),
File "/usr/local/lib/python3.10/dist-packages/flytekit/core/interface.py", line 248, in transform_interface_to_typed_interface
outputs_map = transform_variable_map(interface.outputs, output_descriptions)
File "/usr/local/lib/python3.10/dist-packages/flytekit/core/interface.py", line 345, in transform_variable_map
res[k] = transform_type(v, descriptions.get(k, k))
File "/usr/local/lib/python3.10/dist-packages/flytekit/core/interface.py", line 350, in transform_type
return _interface_models.Variable(type=TypeEngine.to_literal_type(x), description=description)
File "/usr/local/lib/python3.10/dist-packages/flytekit/core/type_engine.py", line 1016, in to_literal_type
transformer = cls.get_transformer(python_type)
File "/usr/local/lib/python3.10/dist-packages/flytekit/core/type_engine.py", line 912, in get_transformer
cls.lazy_import_transformers()
File "/usr/local/lib/python3.10/dist-packages/flytekit/core/type_engine.py", line 998, in lazy_import_transformers
from flytekit.types import schema # noqa: F401
File "/usr/local/lib/python3.10/dist-packages/flytekit/types/schema/__init__.py", line 1, in <module>
from .types import (
File "/usr/local/lib/python3.10/dist-packages/flytekit/types/schema/types.py", line 184, in <module>
class FlyteSchema(DataClassJSONMixin):
File "/usr/local/lib/python3.10/dist-packages/flytekit/types/schema/types.py", line 273, in FlyteSchema
self, dataframe_fmt: type = pandas.DataFrame, override_mode: typing.Optional[SchemaOpenMode] = None
AttributeError: module 'pandas' has no attribute 'DataFrame'
Getting <s3://archipelago-flyte/flytesnacks/development/4OZY3LKCZ6YFFL4K6T6E5B3HNM======/fastf6b856a5c518c1d4cf5a7ea2bf6bd1aa.tar.gz> to /root/
L godlike
01/12/2024, 1:18 AML godlike
01/12/2024, 1:18 AMAlex Lyashok
01/12/2024, 1:18 AMroot@feb27927592b:~# pip show flytekit
Name: flytekit
Version: 1.10.2
Summary: Flyte SDK for Python
Home-page: <https://github.com/flyteorg/flytekit>
Author:
Author-email:
License: apache2
Location: /usr/local/lib/python3.10/dist-packages
Requires: adlfs, click, cloudpickle, cookiecutter, croniter, dataclasses-json, diskcache, docker, docstring-parser, flyteidl, fsspec, gcsfs, googleapis-common-protos, grpcio, grpcio-status, importlib-metadata, joblib, jsonpickle, keyring, kubernetes, marshmallow-enum, marshmallow-jsonschema, mashumaro, numpy, pandas, protobuf, pyarrow, python-json-logger, pytimeparse, pyyaml, requests, rich, rich-click, s3fs, statsd, typing-extensions, urllib3
Required-by: flytekitplugins-deck-standard, flytekitplugins-flyin
Alex Lyashok
01/12/2024, 1:18 AMAlex Lyashok
01/12/2024, 1:19 AMroot@feb27927592b:~# pip show pandas
Name: pandas
Version: 1.5.3
Summary: Powerful data structures for data analysis, time series, and statistics
Home-page: <https://pandas.pydata.org>
Author: The Pandas Development Team
Author-email: <mailto:pandas-dev@python.org|pandas-dev@python.org>
License: BSD-3-Clause
Location: /usr/local/lib/python3.10/dist-packages
Requires: numpy, python-dateutil, pytz
Required-by: bertopic, cudf, cudf-cu12, dask-cuda, dask-cudf, flytekit, geopandas, llama-index, phik, seaborn, statsmodels, visions, ydata-profiling
L godlike
01/12/2024, 1:20 AMAlex Lyashok
01/12/2024, 1:40 AMimport flytekit
from flytekit.testing import SecretsManager
from flytekit import Resources, Secret, task, workflow, dynamic
import logging
import os
logger = logging.getLogger(__name__)
SECRET_OPENAI_API_KEY_ARN = "arn:aws:secretsmanager:us-east-2:099604434934:secret:"
SECRET_OPENAI_API_KEY_KEY = "OPENAI_API_KEY-61ZT7z"
SECRET_COPILOT_SECRET = "COPILOT_SECRET"
SECRET_K8_GROUP = "copilot-test-secret"
SECRET_K8_KEY = "copilot-test-secret-data"
#### testing secrets
@task(
secret_requests=[Secret(group=SECRET_OPENAI_API_KEY_ARN, key=SECRET_OPENAI_API_KEY_KEY, mount_requirement=Secret.MountType.FILE)],
)
def secret_taskAws() -> str:
secret_val = flytekit.current_context().secrets.get(SECRET_OPENAI_API_KEY_ARN, SECRET_OPENAI_API_KEY_KEY)
<http://logger.info|logger.info>(f"Got secret value from AWS for {SECRET_OPENAI_API_KEY_KEY}: {secret_val}")
return secret_val
@task()
def secret_taskEnv() -> str:
secret_val = "not found"
if SECRET_COPILOT_SECRET in os.environ:
<http://logger.info|logger.info>(f"Got secret value for {SECRET_COPILOT_SECRET}: {os.environ[SECRET_COPILOT_SECRET]}")
secret_val = os.environ[SECRET_COPILOT_SECRET]
return secret_val
@task(secret_requests=[Secret(group=SECRET_K8_GROUP, key=SECRET_K8_KEY)])
def secret_taskK8() -> str:
context = flytekit.current_context()
secret_val = context.secrets.get(SECRET_K8_GROUP, SECRET_K8_KEY)
<http://logger.info|logger.info>(f"Got secret value for {SECRET_K8_GROUP}: {secret_val}")
return secret_val
####
Alex Lyashok
01/12/2024, 1:41 AMAlex Lyashok
01/12/2024, 1:42 AML godlike
01/12/2024, 1:42 AML godlike
01/12/2024, 1:43 AML godlike
01/12/2024, 1:43 AMtree .
L godlike
01/12/2024, 1:43 AML godlike
01/12/2024, 1:43 AMAlex Lyashok
01/12/2024, 1:44 AML godlike
01/12/2024, 1:44 AML godlike
01/12/2024, 1:44 AMAlex Lyashok
01/12/2024, 1:51 AML godlike
01/12/2024, 1:54 AML godlike
01/12/2024, 1:54 AML godlike
01/12/2024, 1:54 AM