create_splits

from fsspec.config import conf
conf["gather_batch_size"] = 50

def create_splits(file_paths: list[Path], split_ratios: list[float]) -> dict[str, list[str]]:
    """Create splits from filepaths and renamed split data queried using UUID(filepath names)."""
    # Get the event ids from logs in order to fetch meta data.
    event_id_to_filepath: dict[str, Path] = {}
    event_ids = []
    for file_path in file_paths:
        filename = file_path.stem
        event_id = filename
        event_ids.append(event_id)
        event_id_to_filepath[event_id] = file_path

    # Get the log meta data for each event.
    logs_metadata = _get_event_metadata(
        renamed_client,
        event_uuids=event_ids,
    )

    # Group the logs by renamed split. If a logs meta data was not returned we don't include it in any split.
    renamed_split_logs: dict[str, list[Path]] = {"train": [], "validate": [], "test": []}
    for event_uuid, metadata in logs_metadata.items():
        split = metadata[0]
        renamed_split_logs[split].append(event_id_to_filepath[event_uuid])

    random.seed(0)
    random.shuffle(renamed_split_logs["train"])
    renamed_train_size = len(renamed_split_logs["train"])
    ml_training_size = round(renamed_train_size * split_ratios[0] / sum(split_ratios))

    # Create the splits
    output_splits: dict[str, list[str]] = {}
    output_splits["training"] = [str(file_path) for file_path in renamed_split_logs["train"][:ml_training_size]]
    output_splits["validation"] = [str(file_path) for file_path in renamed_split_logs["train"][ml_training_size:]]
    output_splits["testing"] = [str(file_path) for file_path in renamed_split_logs["validate"]]
    return output_splits

Container jparraga-7775b4ddc4f040f4961-n2-0 failed with exit code 137 because OOMKilled: 

[INFO][credentials.py:567] Found credentials in environment variables.
[INFO][configprovider.py:974] Found endpoint for s3 via: environment_global.
[INFO][tasks.py:315] enter data splitter
[INFO][split_dataset.py:20] enter create splits
[INFO][split_dataset.py:30] getting event metadata
[INFO][split_dataset.py:38] grouping
[INFO][split_dataset.py:56] creating the splits
[INFO][split_dataset.py:62] exit create splits
[INFO][tasks.py:340] Creating splits with size: ML training: 11708, ML validation:  1301, ML testing: 4153
[INFO][tasks.py:347] exit data splitter