import pandas as pd

df = pd.read_parquet("hf://buckets/username/my-bucket/data.parquet")
df.to_parquet("hf://buckets/username/my-bucket/output.parquet")

import dask.dataframe as dd

df = dd.read_parquet("hf://buckets/username/my-bucket/data/")
df.to_parquet("hf://buckets/username/my-bucket/output/")

import daft
from daft.io import IOConfig, HuggingFaceConfig
from huggingface_hub import get_token

io_config = IOConfig(hf=HuggingFaceConfig(token=get_token()))
df = daft.read_parquet("hf://buckets/username/my-bucket/data.parquet", io_config=io_config)

import pyarrow.parquet as pq

table = pq.read_table("hf://buckets/username/my-bucket/data.parquet")

df = (
    spark.read.format("huggingface")
    .option("data_files", '["data.parquet"]')
    .load("buckets/username/my-bucket")
)

from datasets import load_dataset

ds = load_dataset("buckets/username/my-bucket", data_files=["data.parquet"])

hf auth login
hf buckets create username/my-bucket --private

export INSPECT_LOG_DIR=hf://buckets/username/my-bucket/eval-logs
inspect eval popularity.py --model openai/gpt-4
inspect view

# qwen-sft.yaml — launch anywhere: sky launch qwen-sft.yaml --infra aws|gcp|...
resources:
  accelerators: H100:1

file_mounts:
  /base-model:
    source: hf://Qwen/Qwen2.5-3B           # model repo, read-only
    store: hf
    mode: MOUNT
  /data:
    source: hf://datasets/username/my-data@v1.0   # dataset repo, pinned to a tag, read-only
    store: hf
    mode: MOUNT
  /checkpoints:
    source: hf://buckets/username/qwen-sft   # bucket, read-write — checkpoints sync back
    store: hf
    mode: MOUNT

run: |
  python train.py --model /base-model --output_dir /checkpoints

pip install "skypilot[huggingface]"
hf auth login                              # or: export HF_TOKEN=<your-token>
sky launch qwen-sft.yaml

from huggingface_hub import hffs

with hffs.open("buckets/username/my-bucket/hello.txt", "w") as f:
    f.write("Hello world!")

hffs.cp("buckets/username/my-bucket/hello.txt", "buckets/username/my-bucket/hello2.txt")
hffs.rm("buckets/username/my-bucket/hello2.txt")
files = hffs.ls("buckets/username/my-bucket")
text_files = hffs.glob("buckets/username/my-bucket/*.txt")

SELECT * FROM 'hf://buckets/username/my-bucket/data/**/*.parquet' LIMIT 10;
COPY tbl TO 'hf://buckets/username/my-bucket/out.parquet';