prompt-samples (#6)

- video edit prompt samples (cc9cf93e1d03e5ece0cded717c4f2572536dc614)

app.py

@@ -1783,6 +1783,9 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
     max-width: 100% !important;
 }
 .compare-cell { min-width: 0; }
+.compare-cell.compare-source .compare-model-label {
+    color: var(--pruna-text-muted);
+}
 .compare-model-label {
     margin-bottom: 6px;
     color: var(--pruna-lavender);
@@ -1790,15 +1793,23 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
     font-weight: 700;
     word-break: break-word;
 }
-.compare-cell img {
+.compare-cell img,
+.compare-cell video {
     display: block;
     width: 100%;
-    aspect-ratio: 1 / 1;
-    object-fit: cover;
     border-radius: 12px;
     border: 1px solid var(--pruna-border);
     background: var(--pruna-bg-elevated);
 }
+.compare-cell img {
+    aspect-ratio: 1 / 1;
+    object-fit: cover;
+}
+.compare-cell video {
+    aspect-ratio: 16 / 9;
+    max-height: 360px;
+    object-fit: contain;
+}
 .compare-empty,
 .pareto-note-copy {
     margin: 0;
@@ -2157,14 +2168,20 @@ def load_sample_comparison_data(folder):
         return None
 
     prompts = {}
+    source_videos = {}
     with prompts_path.open() as handle:
         for line in handle:
             if not line.strip():
                 continue
             row = json.loads(line)
-            prompts[row["prompt_id"]] = row.get("text", "")
+            prompt_id = row["prompt_id"]
+            prompts[prompt_id] = row.get("text", "")
+            source_video = row.get("source_video")
+            if source_video:
+                source_videos[prompt_id] = source_video
 
     images = defaultdict(dict)
+    kinds = set()
     with generations_path.open() as handle:
         for line in handle:
             if not line.strip():
@@ -2172,20 +2189,33 @@ def load_sample_comparison_data(folder):
             row = json.loads(line)
             model_id = row["model_id"]
             prompt_id = row["prompt_id"]
-            image_url = row.get("image")
-            if model_id and prompt_id and image_url:
-                images[model_id][prompt_id] = image_url
+            media_url = row.get("image") or row.get("video")
+            if row.get("video"):
+                kinds.add("video")
+            elif row.get("image"):
+                kinds.add("image")
+            if model_id and prompt_id and media_url:
+                images[model_id][prompt_id] = media_url
+            if prompt_id and prompt_id not in source_videos:
+                params = row.get("params") or {}
+                input_video = params.get("input_video") or row.get("input_video")
+                if input_video:
+                    source_videos[prompt_id] = input_video
 
     models = sorted(images)
     if not models or not prompts:
         return None
 
-    return {
+    loaded = {
         "prompts": prompts,
         "images": {model: dict(prompt_map) for model, prompt_map in images.items()},
         "models": models,
         "prompt_ids": sorted(prompts),
+        "kind": "video" if "video" in kinds else "image",
     }
+    if source_videos:
+        loaded["source_videos"] = source_videos
+    return loaded
 
 
 def _as_numeric(df, columns):
@@ -2416,6 +2446,10 @@ qwen_combined_dir = _resolve_data_path(
     data_dir / "qwen_image_bench_combined",
     space_root.parent / "qwen_image_bench_combined",
 )
+video_combined_dir = _resolve_data_path(
+    data_dir / "video_editing_combined",
+    space_root.parent / "video_editing_combined",
+)
 qwen_path = _resolve_data_path(
     data_dir / "qwen_image_bench_model_price_and_median_generation_time_10_august.csv",
     space_root.parent / "qwen_image_bench_model_price_and_median_generation_time_10_august.csv",
@@ -2483,6 +2517,7 @@ video_display_columns = [
 
 oneig_samples = load_sample_comparison_data(oneig_combined_dir)
 qwen_samples = load_sample_comparison_data(qwen_combined_dir)
+video_samples = load_sample_comparison_data(video_combined_dir)
 
 metrics = [
     {"id": "datapoint_elo", "column": "Datapoint Elo"},
@@ -2556,7 +2591,7 @@ datasets = [
             "per second of output video. Generation time per second of video "
             "is end-to-end wall time to produce one second of output."
         ),
-        "samples": None,
+        "samples": video_samples,
     },
     {
         "id": "qwen",

data/video_editing_combined/generations.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ec4ea0a431484e65f30990c8cc4292b00c665be96db587775426da1c84a4ab8
+size 762227

data/video_editing_combined/prompts.jsonl

@@ -0,0 +1,78 @@
+{"prompt_id": "video_edit_internal__advertising__p0000", "text": "Replace the white bottle with an orange sunscreen bottle.", "dataset": "video_edit_internal", "category": "advertising", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/advertising/4620329ad65820ce.mp4"}
+{"prompt_id": "video_edit_internal__advertising__p0001", "text": "Replace the woman with an asian woman riding on a donkey through a chinese small town.", "dataset": "video_edit_internal", "category": "advertising", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/advertising/44db5689846be901.mp4"}
+{"prompt_id": "video_edit_internal__advertising__p0002", "text": "Place the couple on a snowy mountain next to a mountain hut.", "dataset": "video_edit_internal", "category": "advertising", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/advertising/481db4a047688bbe.mp4"}
+{"prompt_id": "video_edit_internal__advertising__p0003", "text": "Turn this into a scene in the summer with birds flying in the sky and butterflies in the foreground.", "dataset": "video_edit_internal", "category": "advertising", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/advertising/32b9f56763c56482.mp4"}
+{"prompt_id": "video_edit_internal__advertising__p0004", "text": "Replace the robot arm with a dancing hamster on a small table.", "dataset": "video_edit_internal", "category": "advertising", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/advertising/22d02ae2df7f9045.mp4"}
+{"prompt_id": "video_edit_internal__anonymization__p0000", "text": "Anonymize the womans face", "dataset": "video_edit_internal", "category": "anonymization", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/anonymization/7c410085f2fd939d.mp4"}
+{"prompt_id": "video_edit_internal__anonymization__p0001", "text": "Anonymize the mans face", "dataset": "video_edit_internal", "category": "anonymization", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/anonymization/7b495805c0c270ea.mp4"}
+{"prompt_id": "video_edit_internal__anonymization__p0002", "text": "Anonymize the mans face", "dataset": "video_edit_internal", "category": "anonymization", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/anonymization/736b6fb504bd32be.mp4"}
+{"prompt_id": "video_edit_internal__anonymization__p0003", "text": "Anonymize the girls face", "dataset": "video_edit_internal", "category": "anonymization", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/anonymization/2c82413d39579cd5.mp4"}
+{"prompt_id": "video_edit_internal__artificial_analysis__p0000", "text": "Change the rainforest setting to a neon-lit urban alley at night with steam from vents and wet reflective asphalt, and move into an ariel shot of the character after the initial camera movement to focus on the character", "dataset": "video_edit_internal", "category": "artificial_analysis", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/artificial_analysis/e2f6121269a206d3.mp4"}
+{"prompt_id": "video_edit_internal__artificial_analysis__p0001", "text": "Replace the magenta hover-car with a chrome-blue car, keeping the drift and neon light trails.", "dataset": "video_edit_internal", "category": "artificial_analysis", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/artificial_analysis/55d1baa7f3f71b26.mp4"}
+{"prompt_id": "video_edit_internal__artificial_analysis__p0002", "text": "Make the footage look like it was shot on a 1970s film camera, with grainy film texture, faded warm colors, slight softness, and slightly darker corners around the edges.", "dataset": "video_edit_internal", "category": "artificial_analysis", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/artificial_analysis/fd1ab436de020152.mp4"}
+{"prompt_id": "video_edit_internal__artificial_analysis__p0003", "text": "A person appears at the top of the waterfall, leaps into the lake below, and disappears into the misty water beneath the falls.", "dataset": "video_edit_internal", "category": "artificial_analysis", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/artificial_analysis/7ee1623303a8da68.mp4"}
+{"prompt_id": "video_edit_internal__artificial_analysis__p0004", "text": "A fluffy orange cat leaps gracefully from the floor onto the couch, paws sinking into the soft cushions. It circles once in place, tail swaying gently, before curling up tightly on one of the cushions and settling in comfortably.", "dataset": "video_edit_internal", "category": "artificial_analysis", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/artificial_analysis/9a73c608b343bbbe.mp4"}
+{"prompt_id": "video_edit_internal__camera_editing__p0000", "text": "Zoom in on the man's face to show his focused expression", "dataset": "video_edit_internal", "category": "camera_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/camera_editing/0336e64b0594bd7a.mp4"}
+{"prompt_id": "video_edit_internal__camera_editing__p0001", "text": "Perform an arc shot around the tram as it arrives at the station", "dataset": "video_edit_internal", "category": "camera_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/camera_editing/204aa93703da117b.mp4"}
+{"prompt_id": "video_edit_internal__camera_editing__p0002", "text": "Change the view to a high angle.", "dataset": "video_edit_internal", "category": "camera_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/camera_editing/c2ccb5351be5c718.mp4"}
+{"prompt_id": "video_edit_internal__camera_editing__p0003", "text": "Change the view to a high angle.", "dataset": "video_edit_internal", "category": "camera_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/camera_editing/b725aaded02d65e9.mp4"}
+{"prompt_id": "video_edit_internal__camera_editing__p0004", "text": "Gradually move the camera away from the doctor", "dataset": "video_edit_internal", "category": "camera_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/camera_editing/33f24f4d083df743.mp4"}
+{"prompt_id": "video_edit_internal__design_arena__p0000", "text": "Create a transition of the video of the product whihc is the jeans on the lady model with an atitiude that goes into a zoom out camera that she is in a party .Focus on the vibes", "dataset": "video_edit_internal", "category": "design_arena", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/design_arena/b0b4642272a481b1.mp4"}
+{"prompt_id": "video_edit_internal__design_arena__p0001", "text": "add ethnic urban women dancing at the bottem and at the bar wearing DMI Tshirts", "dataset": "video_edit_internal", "category": "design_arena", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/design_arena/0f730f98f3c51356.mp4"}
+{"prompt_id": "video_edit_internal__design_arena__p0002", "text": "Generate a commercial of this dog drinking beer at an electronic music party in a world where dogs and humans are mixed together.", "dataset": "video_edit_internal", "category": "design_arena", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/design_arena/575fc4b9b6d7b3ab.mp4"}
+{"prompt_id": "video_edit_internal__design_arena__p0003", "text": "everything is the same excpet the winter season, snowing", "dataset": "video_edit_internal", "category": "design_arena", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/design_arena/8674fd022867a736.mp4"}
+{"prompt_id": "video_edit_internal__design_arena__p0004", "text": "A cinematic character introduction of Jason, framed in a medium close-up with subtle camera movement, confident body language, and expressive facial detail. Moody, high-contrast lighting with a cool-toned color palette, shallow depth of field, and a slow dramatic reveal that builds intrigue over a few seconds. I want a video in a 9:16 aspect ratio for tiktok. remove all text and have him in a mid day campus setting", "dataset": "video_edit_internal", "category": "design_arena", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/design_arena/91873bbafe4173da.mp4"}
+{"prompt_id": "video_edit_internal__e_commerce__p0000", "text": "Remove the black blazer.", "dataset": "video_edit_internal", "category": "e_commerce", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/e_commerce/b89faaa5080c31db.mp4"}
+{"prompt_id": "video_edit_internal__e_commerce__p0001", "text": "Replace her skirt with a jeans skirt.", "dataset": "video_edit_internal", "category": "e_commerce", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/e_commerce/c34a33d07e843804.mp4"}
+{"prompt_id": "video_edit_internal__e_commerce__p0002", "text": "Replace the black leggings and the black shirt with a white dress.", "dataset": "video_edit_internal", "category": "e_commerce", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/e_commerce/489d2439cbaeed79.mp4"}
+{"prompt_id": "video_edit_internal__e_commerce__p0003", "text": "Replace the white woman with a lebanese-looking woman.", "dataset": "video_edit_internal", "category": "e_commerce", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/e_commerce/0ad8fb773f303a11.mp4"}
+{"prompt_id": "video_edit_internal__e_commerce__p0004", "text": "Remove all items on the table and place an eyeshadow pallete on the table next to the woman.", "dataset": "video_edit_internal", "category": "e_commerce", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/e_commerce/f28f3aa12b1ec220.mp4"}
+{"prompt_id": "video_edit_internal__lighting__p0000", "text": "After the sun sets behind the mountains, the scene transitions into nighttime.", "dataset": "video_edit_internal", "category": "lighting", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/lighting/229c165e0ac97daf.mp4"}
+{"prompt_id": "video_edit_internal__lighting__p0001", "text": "Change the weather to a dazzling starry night.", "dataset": "video_edit_internal", "category": "lighting", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/lighting/f42c41ab2f3adc26.mp4"}
+{"prompt_id": "video_edit_internal__lighting__p0002", "text": "Change the weather to a thunderstorm with heavy rain.", "dataset": "video_edit_internal", "category": "lighting", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/lighting/daa36567634ca6be.mp4"}
+{"prompt_id": "video_edit_internal__lighting__p0003", "text": "Change the weather to a torrential downpour.", "dataset": "video_edit_internal", "category": "lighting", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/lighting/6817a4142b69d602.mp4"}
+{"prompt_id": "video_edit_internal__lighting__p0004", "text": "Change the weather to a torrential downpour.", "dataset": "video_edit_internal", "category": "lighting", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/lighting/ac01488a685a1096.mp4"}
+{"prompt_id": "video_edit_internal__long__p0000", "text": "Change the weather to a dazzling starry night.", "dataset": "video_edit_internal", "category": "long", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/long/e9af2a7039e43f16.mp4"}
+{"prompt_id": "video_edit_internal__long__p0001", "text": "Adjust the color of book to blue", "dataset": "video_edit_internal", "category": "long", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/long/eab696d1555be316.mp4"}
+{"prompt_id": "video_edit_internal__long__p0002", "text": "Make the young woman turn into sand and blow away.", "dataset": "video_edit_internal", "category": "long", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/long/d18378f1d7e4ca0e.mp4"}
+{"prompt_id": "video_edit_internal__long__p0003", "text": "Make the bird flap its wings", "dataset": "video_edit_internal", "category": "long", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/long/3835b37b43f5e5bf.mp4"}
+{"prompt_id": "video_edit_internal__long__p0004", "text": "Transform the video into a ukiyo-e style", "dataset": "video_edit_internal", "category": "long", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/long/09fe5bb4b5304b66.mp4"}
+{"prompt_id": "video_edit_internal__movie_concept_art__p0000", "text": "Add more blood and wounds to the mans face.", "dataset": "video_edit_internal", "category": "movie_concept_art", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/movie_concept_art/1d1a99167475720b.mp4"}
+{"prompt_id": "video_edit_internal__movie_concept_art__p0001", "text": "Make the scene less dark.", "dataset": "video_edit_internal", "category": "movie_concept_art", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/movie_concept_art/10cb44d8c0bf4116.mp4"}
+{"prompt_id": "video_edit_internal__movie_concept_art__p0002", "text": "Replace the female warrior with a male warrior with ginger hair.", "dataset": "video_edit_internal", "category": "movie_concept_art", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/movie_concept_art/9e45454390c20b8a.mp4"}
+{"prompt_id": "video_edit_internal__movie_concept_art__p0003", "text": "Turn the man's hand into a robotic hand.", "dataset": "video_edit_internal", "category": "movie_concept_art", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/movie_concept_art/99977e7a91a950b4.mp4"}
+{"prompt_id": "video_edit_internal__real_estate__p0000", "text": "Replace the interior with a gold-black interior heavy luxury style.", "dataset": "video_edit_internal", "category": "real_estate", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/real_estate/869064ab06bb5daa.mp4"}
+{"prompt_id": "video_edit_internal__real_estate__p0001", "text": "Replace the grey wallpaper with a beige painted wall and turn the grey curtains a dark brown.", "dataset": "video_edit_internal", "category": "real_estate", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/real_estate/1c2f358bfe9b8299.mp4"}
+{"prompt_id": "video_edit_internal__real_estate__p0002", "text": "Remove all decoration from the walls and keep only the furniture.", "dataset": "video_edit_internal", "category": "real_estate", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/real_estate/2058bcf38389f8fe.mp4"}
+{"prompt_id": "video_edit_internal__real_estate__p0003", "text": "Exchange the wooden floor for a marble floor.", "dataset": "video_edit_internal", "category": "real_estate", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/real_estate/6a4c09a23dda57d9.mp4"}
+{"prompt_id": "video_edit_internal__real_estate__p0004", "text": "Replace the bedframe with a modern wooden bed.", "dataset": "video_edit_internal", "category": "real_estate", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/real_estate/3e0f6dc5df891001.mp4"}
+{"prompt_id": "video_edit_internal__style_transfer__p0000", "text": "Transform the video into a cyberpunk style", "dataset": "video_edit_internal", "category": "style_transfer", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/style_transfer/2253f6ed3674f006.mp4"}
+{"prompt_id": "video_edit_internal__style_transfer__p0001", "text": "Convert to different shades of orange", "dataset": "video_edit_internal", "category": "style_transfer", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/style_transfer/edb6a105be03bae0.mp4"}
+{"prompt_id": "video_edit_internal__style_transfer__p0002", "text": "Convert to black and white", "dataset": "video_edit_internal", "category": "style_transfer", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/style_transfer/c3dc058b3be26cbd.mp4"}
+{"prompt_id": "video_edit_internal__style_transfer__p0003", "text": "Apply Ghibli-style editing to the video", "dataset": "video_edit_internal", "category": "style_transfer", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/style_transfer/83a638223a20911c.mp4"}
+{"prompt_id": "video_edit_internal__style_transfer__p0004", "text": "Relight the scene as if it were shot during golden hour, with warm low-angle sunlight, soft shadows, and natural highlights on faces and surfaces.", "dataset": "video_edit_internal", "category": "style_transfer", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/style_transfer/daa36567634ca6be.mp4"}
+{"prompt_id": "video_edit_internal__subject_editing__p0000", "text": "Replace the rainbow colors of the logo with different shades of purple", "dataset": "video_edit_internal", "category": "subject_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/subject_editing/6358f26b46baadf5.mp4"}
+{"prompt_id": "video_edit_internal__subject_editing__p0001", "text": "Add a small dog running beside the scooter", "dataset": "video_edit_internal", "category": "subject_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/subject_editing/052c0bfd7b68fea7.mp4"}
+{"prompt_id": "video_edit_internal__subject_editing__p0002", "text": "Replace the splashing waves with a calm water surface", "dataset": "video_edit_internal", "category": "subject_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/subject_editing/c3f79578a39f415d.mp4"}
+{"prompt_id": "video_edit_internal__subject_editing__p0003", "text": "Add a violinist in the background", "dataset": "video_edit_internal", "category": "subject_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/subject_editing/a0eebf32a076e2fc.mp4"}
+{"prompt_id": "video_edit_internal__subject_editing__p0004", "text": "Add a group of people walking on the pathway", "dataset": "video_edit_internal", "category": "subject_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/subject_editing/3c2b7e3a5b284e1d.mp4"}
+{"prompt_id": "video_edit_internal__subject_motion_editing__p0000", "text": "Make the bird hop around instead of walking and foraging", "dataset": "video_edit_internal", "category": "subject_motion_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/subject_motion_editing/ac38b2a97d1a5c3a.mp4"}
+{"prompt_id": "video_edit_internal__subject_motion_editing__p0001", "text": "The male colleague is walking around to observe.", "dataset": "video_edit_internal", "category": "subject_motion_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/subject_motion_editing/8aaab7d304c5ad99.mp4"}
+{"prompt_id": "video_edit_internal__subject_motion_editing__p0002", "text": "Make the static spider-man in the mural dynamic and make him swing faster", "dataset": "video_edit_internal", "category": "subject_motion_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/subject_motion_editing/3412b39686fd5879.mp4"}
+{"prompt_id": "video_edit_internal__subject_motion_editing__p0003", "text": "Change the woman's jogging to taking off.", "dataset": "video_edit_internal", "category": "subject_motion_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/subject_motion_editing/17c4dc0514772321.mp4"}
+{"prompt_id": "video_edit_internal__subject_motion_editing__p0004", "text": "Make the knight lunging forward and the creature swiping at the knight", "dataset": "video_edit_internal", "category": "subject_motion_editing", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/subject_motion_editing/980354550011911b.mp4"}
+{"prompt_id": "video_edit_internal__synthetic_data__p0000", "text": "Turn the scene into nighttime.", "dataset": "video_edit_internal", "category": "synthetic_data", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/synthetic_data/e3b1e603bfaa1aec.mp4"}
+{"prompt_id": "video_edit_internal__synthetic_data__p0001", "text": "Remove the crosswalk.", "dataset": "video_edit_internal", "category": "synthetic_data", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/synthetic_data/30df5fd5fe28af65.mp4"}
+{"prompt_id": "video_edit_internal__synthetic_data__p0002", "text": "Add a bicycle riding in front of the car.", "dataset": "video_edit_internal", "category": "synthetic_data", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/synthetic_data/d093491e25682b3a.mp4"}
+{"prompt_id": "video_edit_internal__synthetic_data__p0003", "text": "Turn the scene into a snowstorm scene.", "dataset": "video_edit_internal", "category": "synthetic_data", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/synthetic_data/1da5913f13362f8d.mp4"}
+{"prompt_id": "video_edit_internal__synthetic_data__p0004", "text": "Make it rain in the scene.", "dataset": "video_edit_internal", "category": "synthetic_data", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/synthetic_data/936bdb89558f5a7f.mp4"}
+{"prompt_id": "video_edit_internal__text__p0000", "text": "Add the text \"True Love\" in the foreground in pink romantic font.", "dataset": "video_edit_internal", "category": "text", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/text/c763407793ca9b0e.mp4"}
+{"prompt_id": "video_edit_internal__text__p0001", "text": "Replace any mention of \"Fanta\" with the branding \"Cola\".", "dataset": "video_edit_internal", "category": "text", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/text/7b878ea6d99834b2.mp4"}
+{"prompt_id": "video_edit_internal__text__p0002", "text": "Remove all text from the video.", "dataset": "video_edit_internal", "category": "text", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/text/76763c31b2609109.mp4"}
+{"prompt_id": "video_edit_internal__text__p0003", "text": "Replace the branding \"Royalty\" with the phrasing \"Princess\".", "dataset": "video_edit_internal", "category": "text", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/text/d221d299c9e49064.mp4"}
+{"prompt_id": "video_edit_internal__text__p0004", "text": "Remove all text from the video.", "dataset": "video_edit_internal", "category": "text", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/text/5c169901af8e4ed6.mp4"}
+{"prompt_id": "video_edit_internal__transitions__p0000", "text": "After a black screen transition, the road transforms into lush grass.", "dataset": "video_edit_internal", "category": "transitions", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/transitions/964c35b19ee1fc1f.mp4"}
+{"prompt_id": "video_edit_internal__transitions__p0001", "text": "After a wave-foam transition, the small fishing boat is eaten by a giant whale.", "dataset": "video_edit_internal", "category": "transitions", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/transitions/b8b88f43316363f3.mp4"}
+{"prompt_id": "video_edit_internal__transitions__p0002", "text": "Add a cut transition, then show a bowl of ramen topped with parsley.", "dataset": "video_edit_internal", "category": "transitions", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/transitions/c823d6c77bdc87f4.mp4"}
+{"prompt_id": "video_edit_internal__transitions__p0003", "text": "Add a smoke transition, then show the circuit board burning.", "dataset": "video_edit_internal", "category": "transitions", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/transitions/edb9927ff06c1175.mp4"}
+{"prompt_id": "video_edit_internal__transitions__p0004", "text": "After a smoke transition, the Basilica of the Sacred Heart of Paris catches fire.", "dataset": "video_edit_internal", "category": "transitions", "source_video": "https://d2j1a65dna040x.cloudfront.net/benchmark_inputs/video_edit_internal/transitions/77bb900383d4f370.mp4"}

model_display.py

@@ -95,6 +95,15 @@ MODEL_DISPLAY_NAMES = {
     "P-Video Edit Final (draft)": "P-Video-Edit Draft",
     "p_video_edit_preview__replicate_final": "P-Video-Edit",
     "p_video_edit_preview__replicate_final__draft": "P-Video-Edit Draft",
+    # Video-to-video leaderboard
+    "gemini_omni_flash_edit__fal": "Gemini Omni Flash Edit",
+    "grok_imagine_video__replicate": "Grok Imagine Video",
+    "happyhorse_1_0__wavespeed": "HappyHorse 1.0",
+    "ltx_2_3_quality_reference_video_to_video__fal": "LTX 2.3 Video Edit",
+    "lucy_edit_pro__fal": "Lucy Edit Pro",
+    "minimax_h3_reference_to_video__fal": "MiniMax H3 Reference-to-Video",
+    "seedance_2_5_video_edit_turbo__wavespeed": "Seedance 2.5 Video Edit Turbo",
+    "wan_2_7_video_edit__wavespeed": "Wan 2.7 Video Edit",
     # Others overlapping P-Bench
     "z_image": "Z-Image",
     "glm_image": "GLM-Image",

ui.py

@@ -86,7 +86,9 @@ There is no single score across P-Bench.
    and lower price (or time). Only datasets with price or generation time
    can open this tab (not Arena AI).
 4. **Samples**: the same prompts, side by side. Only for datasets we
-   generated (Qwen Image Dataset and OneIG Alignment Dataset).
+   generated (Qwen Image Dataset, OneIG Alignment Dataset, and the
+   Pruna Internal Video-Edit Benchmark). Video samples show the source
+   clip first, then each model's edit.
 
 ## How a score is made
 
@@ -113,8 +115,8 @@ e-commerce, real estate, concept art, and similar work. The suite also
 covers camera-angle and movement changes, lighting, and text in video
 (altering, adding, or removing it). Quality is Datapoint Elo from
 pairwise preference. Price is USD per second of output video;
-generation time is wall time per second of output video. Samples are
-not shown yet.
+generation time is wall time per second of output video. Samples show
+the source clip beside each model's edit.
 
 ### Qwen Image Dataset
 100 prompts from the 1,000-prompt Qwen Image Bench set, sampled for coverage
@@ -179,14 +181,18 @@ models *within* a Dataset | Metric view.
 - **Prompt counts:** OneIG Alignment uses 100 anime, 100 human, and 99 object
   prompts (299 total). Qwen Image Dataset uses 100 prompts sampled from the
   1,000-prompt pool for roughly even coverage of its fine-grained (L3)
-  categories. Artificial Analysis and Arena AI use their own private prompt
-  sets.
+  categories. The Pruna Internal Video-Edit Benchmark uses 78 prompts
+  across advertising, e-commerce, real estate, camera, lighting, text, and
+  related categories. Artificial Analysis and Arena AI use their own
+  private prompt sets.
 - **Generation (Qwen and OneIG):** one image per prompt per endpoint when
   the run exists. Default resolution is 1024×1024. Exceptions: FLUX 1.1 Pro
   Ultra at 2K, FLUX 2 Flex at 1008×1008, and any endpoint labeled 2K. The
   seed is derived from the prompt, so every model gets the same seed for the
   same prompt. Steps, CFG, prompt rewrite, and safety filters follow each
   endpoint's default. This does not describe Artificial Analysis or Arena AI.
+- **Generation (Video-Edit):** one edited clip per prompt per endpoint when
+  the run exists. Every model sees the same source video for a prompt.
 - **Datapoint (Qwen and OneIG):** every model pair is compared on every
   prompt, with 10 votes per battle.
 - **Rapidata (Qwen and OneIG):** prompts longer than 400 characters are
@@ -324,7 +330,37 @@ def _sample_model_ids(datasets, dataset_id):
     samples = dataset.get("samples") if dataset else None
     if not samples:
         return set()
-    return set(samples.get("models") or [])
+    models = set(samples.get("models") or [])
+    return models | {display_model_name(model) for model in models}
+
+
+def _sample_media_map(samples):
+    return (samples or {}).get("images") or {}
+
+
+def _resolve_sample_model(samples, model):
+    media = _sample_media_map(samples)
+    if model in media:
+        return model
+    wanted = {str(model or "").strip(), display_model_name(model)}
+    wanted.discard("")
+    for key in media:
+        if key in wanted or display_model_name(key) in wanted:
+            return key
+    return None
+
+
+def _default_sample_models(samples):
+    models = list((samples or {}).get("models") or [])
+    preferred = [model for model in models if _is_pruna_model(model)]
+    preferred.sort(
+        key=lambda model: (
+            "draft" in str(model).casefold()
+            or "draft" in display_model_name(model).casefold(),
+            display_model_name(model).casefold(),
+        )
+    )
+    return (preferred or models)[:2]
 
 
 def _pareto_price_column(data):
@@ -1205,15 +1241,48 @@ def _samples_html(samples, selected_models, num_prompts, seed=0):
         return _pareto_unavailable_html(
             "Samples aren't available for this dataset."
         )
-    images = samples.get("images", {})
-    models = [model for model in (selected_models or []) if model in images]
+    models = [
+        resolved
+        for model in (selected_models or [])
+        if (resolved := _resolve_sample_model(samples, model))
+    ]
     if not models:
-        models = (samples.get("models") or [])[:2]
+        models = _default_sample_models(samples)
     return _build_compare_samples_html(samples, models, num_prompts, seed)
 
 
+def _compare_media_html(url, label, *, kind):
+    safe_url = escape(url, quote=True)
+    safe_label = escape(label)
+    if kind == "video":
+        return (
+            f'<video src="{safe_url}" controls preload="metadata" '
+            f'playsinline></video>'
+        )
+    return (
+        f'<a href="{safe_url}" target="_blank" rel="noopener noreferrer">'
+        f'<img src="{safe_url}" alt="{safe_label} sample" loading="lazy" />'
+        f"</a>"
+    )
+
+
+def _compare_cell_html(label, url, *, kind, extra_class=""):
+    classes = "compare-cell"
+    if extra_class:
+        classes = f"{classes} {extra_class}"
+    return f"""
+                <div class="{classes}">
+                    <div class="compare-model-label">{escape(label)}</div>
+                    {_compare_media_html(url, label, kind=kind)}
+                </div>
+                """
+
+
 def _build_compare_samples_html(samples, selected_models, num_prompts, seed=0):
     selected_models = list(selected_models or [])[:MAX_COMPARE_MODELS]
+    media = _sample_media_map(samples)
+    kind = (samples or {}).get("kind") or "image"
+    source_videos = (samples or {}).get("source_videos") or {}
 
     if not selected_models:
         return (
@@ -1224,7 +1293,7 @@ def _build_compare_samples_html(samples, selected_models, num_prompts, seed=0):
 
     shared_prompt_ids = None
     for model in selected_models:
-        model_prompt_ids = set(samples["images"][model])
+        model_prompt_ids = set(media.get(model) or [])
         shared_prompt_ids = (
             model_prompt_ids
             if shared_prompt_ids is None
@@ -1244,29 +1313,31 @@ def _build_compare_samples_html(samples, selected_models, num_prompts, seed=0):
     rng.shuffle(prompt_pool)
     chosen = prompt_pool[: max(1, min(int(num_prompts), len(prompt_pool)))]
 
-    columns = len(selected_models)
     blocks = []
     for index, prompt_id in enumerate(chosen, start=1):
         prompt_text = escape(samples["prompts"].get(prompt_id, ""))
         cells = []
+        source_url = source_videos.get(prompt_id)
+        if source_url:
+            cells.append(
+                _compare_cell_html(
+                    "Source", source_url, kind="video", extra_class="compare-source"
+                )
+            )
         for model in selected_models:
-            image_url = escape(samples["images"][model][prompt_id], quote=True)
             cells.append(
-                f"""
-                <div class="compare-cell">
-                    <div class="compare-model-label">{escape(display_model_name(model))}</div>
-                    <a href="{image_url}" target="_blank" rel="noopener noreferrer">
-                        <img src="{image_url}" alt="{escape(display_model_name(model))} sample" loading="lazy" />
-                    </a>
-                </div>
-                """
+                _compare_cell_html(
+                    display_model_name(model),
+                    media[model][prompt_id],
+                    kind=kind,
+                )
             )
+        columns = len(cells)
         blocks.append(
             f"""
             <div class="compare-prompt-block">
                 <div class="compare-prompt-meta">
                     <span>Prompt {index}</span>
-                    <span>{escape(prompt_id)}</span>
                 </div>
                 <p class="compare-prompt-text">{prompt_text}</p>
                 <div class="compare-row" style="grid-template-columns: repeat({columns}, minmax(0, 1fr));">
@@ -1524,7 +1595,8 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
                 with gr.Column(visible=bool(initial_samples)) as samples_panel:
                     gr.Markdown(
                         f"<p class='view-help'>"
-                        f"The same prompts, side by side. Select up to "
+                        f"The same prompts, side by side. Video edits show the "
+                        f"source clip first. Select up to "
                         f"<strong>{MAX_COMPARE_MODELS}</strong> models above, or leave "
                         f"Models empty for two defaults."
                         f"</p>",