"""Pinned raw Grabette inventory. No LeRobot or generic video SLAM inputs.""" import json import math import re import tempfile from pathlib import Path from huggingface_hub import HfApi, hf_hub_download from huggingface_hub.utils import validate_repo_id REQUIRED = { "raw_video.mp4", "frame_timestamps.json", "rpi_camera_intrinsics.json", "metadata.json", "dcam_imu.json", "frames.json", "dcam_calib_offline.json", } RECORDING = re.compile(r"[0-9]{8}_[0-9]{6}") def inventory(dataset, token=False, revision=None): """Pin a repository revision and require complete raw Grabette RGB and IMU inputs.""" dataset = dataset.strip() validate_repo_id(dataset) if dataset.count("/") != 1: raise ValueError("Use a raw Grabette repository ID: owner/dataset") api = HfApi(token=token or False) info = api.dataset_info(dataset, revision=revision) files = set(api.list_repo_files(dataset, repo_type="dataset", revision=info.sha)) streams = [] for name in sorted(files): parts = name.split("/") if len(parts) != 3 or parts[2] != "raw_video.mp4": continue recording, hand, _ = parts if not RECORDING.fullmatch(recording) or hand not in {"left", "right"}: continue prefix = f"{recording}/{hand}" missing = sorted(f for f in REQUIRED if f"{prefix}/{f}" not in files) if missing: raise ValueError(f"{prefix}: raw Grabette + IMU inputs missing: {', '.join(missing)}") streams.append({"recording": recording, "hand": hand, "prefix": prefix}) if not streams: raise ValueError( "Grabette SLAM requires raw recordings at YYYYMMDD_HHMMSS/right/ or left/, including raw_video.mp4, per-recording intrinsics and OAK IMU calibration. Converted LeRobot datasets are not supported here." ) return info, streams def read_raw_cameras(dataset, token=False): """Discover available Grabette hands without downloading video or IMU streams.""" info, streams = inventory(dataset, token) return { "dataset": info.id, "revision": info.sha, "cameras": sorted({s["hand"] for s in streams}), "source": "raw-grabette", "recordings": len({s["recording"] for s in streams}), } def read_raw_selection(dataset, start, count, camera=None, token=False, revision=None): """Read pinned per-recording metadata for one hand and a reproducible range.""" if type(start) is not int or type(count) is not int or min(start, count) < 0: raise ValueError("Starting recording and count must be nonnegative integers") info, streams = inventory(dataset, token, revision) cameras = sorted({s["hand"] for s in streams}) camera = camera or cameras[0] if camera not in cameras: raise ValueError("Choose a Grabette hand available in this repository") streams = [s for s in streams if s["hand"] == camera] total = len(streams) chosen = streams[start : start + count] if count else streams[start:] if not chosen: raise ValueError("No raw Grabette recordings in this range") rows, seconds = [], 0.0 with tempfile.TemporaryDirectory(prefix="grabette-metadata-") as cache: for index, stream in enumerate(chosen, start=start): path = hf_hub_download( info.id, f"{stream['prefix']}/metadata.json", repo_type="dataset", revision=info.sha, token=token or False, cache_dir=cache, ) meta = json.loads(Path(path).read_text()) duration, frames = float(meta["duration_seconds"]), int(meta["frame_count"]) if ( not math.isfinite(duration) or duration <= 0 or frames < 2 or meta.get("backend") != "rpi" or meta.get("hand") != camera ): raise ValueError(f"{stream['prefix']}: unsupported raw Grabette metadata") rows.append(dict(stream, episode_index=index, length=frames, duration_seconds=duration)) seconds += duration return { "dataset": info.id, "revision": info.sha, "source": "raw-grabette", "info": {"fps": 50, "features": {}}, "episodes": rows, "cameras": cameras, "camera": camera, "seconds": seconds, "total_episodes": total, "start": start, "count": count, "private": bool(info.private), }