smallFIlesSplit

import pandas as pd
from pathlib import Path

source_dir = Path("/ibex/user/harbihh/r6.2")
test_dir = Path("/ibex/user/harbihh/split/r6.2")
test_dir.mkdir(parents=True, exist_ok=True)

files = ["device.csv", "psychometric.csv"]

for name in files:
    src = source_dir / name
    dst = test_dir / name

    print(f"Processing {name}...")
    df = pd.read_csv(src)
    test = df.sample(frac=0.2, random_state=42)
    train = df.drop(test.index)

    test.to_csv(dst, index=False)
    train.to_csv(src, index=False)

print("Small file split complete.")