| """Pinned raw Grabette inventory. No LeRobot or generic video SLAM inputs.""" |
|
|
| import json |
| import math |
| import re |
| import tempfile |
| from pathlib import Path |
|
|
| from huggingface_hub import HfApi, hf_hub_download |
| from huggingface_hub.utils import validate_repo_id |
|
|
| REQUIRED = { |
| "raw_video.mp4", |
| "frame_timestamps.json", |
| "rpi_camera_intrinsics.json", |
| "metadata.json", |
| "dcam_imu.json", |
| "frames.json", |
| "dcam_calib_offline.json", |
| } |
| RECORDING = re.compile(r"[0-9]{8}_[0-9]{6}") |
|
|
|
|
| def inventory(dataset, token=False, revision=None): |
| """Pin a repository revision and require complete raw Grabette RGB and IMU inputs.""" |
| dataset = dataset.strip() |
| validate_repo_id(dataset) |
| if dataset.count("/") != 1: |
| raise ValueError("Use a raw Grabette repository ID: owner/dataset") |
| api = HfApi(token=token or False) |
| info = api.dataset_info(dataset, revision=revision) |
| files = set(api.list_repo_files(dataset, repo_type="dataset", revision=info.sha)) |
| streams = [] |
| for name in sorted(files): |
| parts = name.split("/") |
| if len(parts) != 3 or parts[2] != "raw_video.mp4": |
| continue |
| recording, hand, _ = parts |
| if not RECORDING.fullmatch(recording) or hand not in {"left", "right"}: |
| continue |
| prefix = f"{recording}/{hand}" |
| missing = sorted(f for f in REQUIRED if f"{prefix}/{f}" not in files) |
| if missing: |
| raise ValueError(f"{prefix}: raw Grabette + IMU inputs missing: {', '.join(missing)}") |
| streams.append({"recording": recording, "hand": hand, "prefix": prefix}) |
| if not streams: |
| raise ValueError( |
| "Grabette SLAM requires raw recordings at YYYYMMDD_HHMMSS/right/ or left/, including raw_video.mp4, per-recording intrinsics and OAK IMU calibration. Converted LeRobot datasets are not supported here." |
| ) |
| return info, streams |
|
|
|
|
| def read_raw_cameras(dataset, token=False): |
| """Discover available Grabette hands without downloading video or IMU streams.""" |
| info, streams = inventory(dataset, token) |
| return { |
| "dataset": info.id, |
| "revision": info.sha, |
| "cameras": sorted({s["hand"] for s in streams}), |
| "source": "raw-grabette", |
| "recordings": len({s["recording"] for s in streams}), |
| } |
|
|
|
|
| def read_raw_selection(dataset, start, count, camera=None, token=False, revision=None): |
| """Read pinned per-recording metadata for one hand and a reproducible range.""" |
| if type(start) is not int or type(count) is not int or min(start, count) < 0: |
| raise ValueError("Starting recording and count must be nonnegative integers") |
| info, streams = inventory(dataset, token, revision) |
| cameras = sorted({s["hand"] for s in streams}) |
| camera = camera or cameras[0] |
| if camera not in cameras: |
| raise ValueError("Choose a Grabette hand available in this repository") |
| streams = [s for s in streams if s["hand"] == camera] |
| total = len(streams) |
| chosen = streams[start : start + count] if count else streams[start:] |
| if not chosen: |
| raise ValueError("No raw Grabette recordings in this range") |
| rows, seconds = [], 0.0 |
| with tempfile.TemporaryDirectory(prefix="grabette-metadata-") as cache: |
| for index, stream in enumerate(chosen, start=start): |
| path = hf_hub_download( |
| info.id, |
| f"{stream['prefix']}/metadata.json", |
| repo_type="dataset", |
| revision=info.sha, |
| token=token or False, |
| cache_dir=cache, |
| ) |
| meta = json.loads(Path(path).read_text()) |
| duration, frames = float(meta["duration_seconds"]), int(meta["frame_count"]) |
| if ( |
| not math.isfinite(duration) |
| or duration <= 0 |
| or frames < 2 |
| or meta.get("backend") != "rpi" |
| or meta.get("hand") != camera |
| ): |
| raise ValueError(f"{stream['prefix']}: unsupported raw Grabette metadata") |
| rows.append(dict(stream, episode_index=index, length=frames, duration_seconds=duration)) |
| seconds += duration |
| return { |
| "dataset": info.id, |
| "revision": info.sha, |
| "source": "raw-grabette", |
| "info": {"fps": 50, "features": {}}, |
| "episodes": rows, |
| "cameras": cameras, |
| "camera": camera, |
| "seconds": seconds, |
| "total_episodes": total, |
| "start": start, |
| "count": count, |
| "private": bool(info.private), |
| } |
|
|
