{
  "TabletopGen": {
    "cohort": {
      "attempted": 80,
      "completed": [
        "abc_1",
        "abc_2",
        "abc_3",
        "abc_5",
        "airoa_moma_000_row0",
        "airoa_moma_005_row1379",
        "airoa_moma_005_row1493",
        "bridge_1",
        "bridge_2",
        "bridge_3",
        "bridge_4",
        "bridge_6",
        "bridge_7",
        "misc_IMG_8226",
        "misc_IMG_8334",
        "misc_IMG_9096",
        "misc_genai2",
        "misc_online2",
        "misc_online3",
        "misc_online5",
        "misc_still_life",
        "robodojo_arrange_largest_number_random",
        "robodojo_arrange_largest_number_standard",
        "robodojo_fold_clothes_random",
        "robodojo_fold_clothes_standard",
        "robodojo_hang_mugs_standard",
        "robodojo_make_toast_random",
        "robodojo_make_toast_standard",
        "robodojo_pack_objects_into_box_random",
        "robodojo_pack_objects_into_box_standard",
        "robodojo_pour_liquid_into_cup_random",
        "robodojo_pour_liquid_into_cup_standard",
        "robodojo_push_T_random",
        "robodojo_push_T_standard",
        "robodojo_sort_nesting_dolls_by_size_random",
        "robodojo_stack_blocks_random",
        "robodojo_stack_blocks_standard",
        "robodojo_stack_bowls_standard",
        "robodojo_store_laptop_and_headphones_random",
        "robodojo_store_laptop_and_headphones_standard",
        "robodojo_sweep_blocks_random",
        "robodojo_sweep_blocks_standard",
        "robolab_bagel_plate_banana_bowl",
        "robolab_bin_condiments",
        "robolab_bin_mug_mustard_marker_bowl",
        "robolab_breakfast_table",
        "robolab_cartons_on_box",
        "robolab_clutter_fruit_bottle_bluebin",
        "robolab_clutter_shelf",
        "robolab_cutlery_shelf",
        "robolab_food_packing",
        "robolab_food_packing_dense",
        "robolab_foodpacking_1bin_3box_3can",
        "robolab_mugs4_measuringcup_drill_bowl",
        "robolab_mugs4_measuringcup_drill_bowl_v2",
        "robolab_mugs_on_shelf",
        "robolab_tools_container",
        "robolab_workdesk_bin",
        "robolab_workdesk_snacks"
      ],
      "synthetic": [
        "robodojo_arrange_largest_number_random",
        "robodojo_arrange_largest_number_standard",
        "robodojo_fold_clothes_random",
        "robodojo_fold_clothes_standard",
        "robodojo_hang_mugs_standard",
        "robodojo_make_toast_random",
        "robodojo_make_toast_standard",
        "robodojo_pack_objects_into_box_random",
        "robodojo_pack_objects_into_box_standard",
        "robodojo_pour_liquid_into_cup_random",
        "robodojo_pour_liquid_into_cup_standard",
        "robodojo_push_T_random",
        "robodojo_push_T_standard",
        "robodojo_sort_nesting_dolls_by_size_random",
        "robodojo_stack_blocks_random",
        "robodojo_stack_blocks_standard",
        "robodojo_stack_bowls_standard",
        "robodojo_store_laptop_and_headphones_random",
        "robodojo_store_laptop_and_headphones_standard",
        "robodojo_sweep_blocks_random",
        "robodojo_sweep_blocks_standard",
        "robolab_bagel_plate_banana_bowl",
        "robolab_bin_condiments",
        "robolab_bin_mug_mustard_marker_bowl",
        "robolab_breakfast_table",
        "robolab_cartons_on_box",
        "robolab_clutter_fruit_bottle_bluebin",
        "robolab_clutter_shelf",
        "robolab_cutlery_shelf",
        "robolab_food_packing",
        "robolab_food_packing_dense",
        "robolab_foodpacking_1bin_3box_3can",
        "robolab_mugs4_measuringcup_drill_bowl",
        "robolab_mugs4_measuringcup_drill_bowl_v2",
        "robolab_mugs_on_shelf",
        "robolab_tools_container",
        "robolab_workdesk_bin",
        "robolab_workdesk_snacks"
      ],
      "novel_views_per_scene": 35
    },
    "metrics": {
      "TabletopGen": {
        "source": {
          "psnr": 7.9271312843694295,
          "ssim": 0.5160368071312313,
          "lpips": 0.7523205517712286,
          "negative_clip": 0.22770805884215792,
          "psnr_objects": 9.49907537937816,
          "ssim_objects": 0.3577681605366798,
          "lpips_obj_white": 0.30091078861342657,
          "nclip_obj_white": 0.13214540235479558
        },
        "source_transferred": {
          "psnr": 12.578208320300453,
          "ssim": 0.6036176336612279,
          "lpips": 0.6857961416244507,
          "negative_clip": 0.17958936095237732
        },
        "novel_all": {
          "psnr": 12.34144652320162,
          "ssim": 0.6273099909988525,
          "lpips": 0.6970112220237129,
          "negative_clip": 0.18893440719833948
        },
        "instances": {
          "ari": 0.506056035756546,
          "fg_iou": 0.4183540955439134,
          "count_mae": 2.8684210526315788
        },
        "layout": {
          "ospa5": 0.0444,
          "ospa10": 0.07283684210526316,
          "f1_10": 0.5674473684210526
        }
      },
      "SceneRig": {
        "source": {
          "psnr": 13.387694745762712,
          "ssim": 0.5363015254237288,
          "lpips": 0.5172927118644067,
          "negative_clip": 0.12724830508474577,
          "psnr_objects": 13.68075947368421,
          "ssim_objects": 0.4407042105263158,
          "lpips_obj_white": 0.16577184210526316,
          "nclip_obj_white": 0.061464736842105266
        },
        "source_transferred": {
          "psnr": 13.05418552631579,
          "ssim": 0.519285,
          "lpips": 0.5907118421052632,
          "negative_clip": 0.12109421052631579
        },
        "novel_all": {
          "psnr": 12.60044460150376,
          "ssim": 0.5379923684210526,
          "lpips": 0.6106322030075187,
          "negative_clip": 0.12582933834586466
        },
        "instances": {
          "ari": 0.8773000960547885,
          "fg_iou": 0.8175626576141246,
          "count_mae": 0.23684210526315788
        },
        "layout": {
          "ospa5": 0.02459736842105263,
          "ospa10": 0.03188421052631579,
          "f1_10": 0.92595
        }
      }
    },
    "vlm": {
      "methods": [
        "scenerig",
        "tabletopgen"
      ],
      "judges": [
        "gpt-5.6-terra",
        "qwen3.8-max-fireworks"
      ],
      "n_rows": 118,
      "dimensions": {
        "completeness": {
          "flip_rate": 0.07627118644067798,
          "win_matrix": {
            "scenerig": {
              "scenerig": 0.0,
              "tabletopgen": 101.5
            },
            "tabletopgen": {
              "scenerig": 7.5,
              "tabletopgen": 0.0
            }
          },
          "bt_strength": {
            "scenerig": 1.2727656358022175,
            "tabletopgen": -1.2727656358022177
          },
          "bt_ci95": {
            "scenerig": [
              1.0166988015892147,
              1.6480336411909757
            ],
            "tabletopgen": [
              -1.6480336411909762,
              -1.0166988015892147
            ]
          },
          "win_rate": {
            "scenerig": 0.9311926605504587,
            "tabletopgen": 0.06880733944954129
          }
        },
        "layout": {
          "flip_rate": 0.008474576271186418,
          "win_matrix": {
            "scenerig": {
              "scenerig": 0.0,
              "tabletopgen": 114.0
            },
            "tabletopgen": {
              "scenerig": 3.0,
              "tabletopgen": 0.0
            }
          },
          "bt_strength": {
            "scenerig": 1.743905927249463,
            "tabletopgen": -1.743905927249463
          },
          "bt_ci95": {
            "scenerig": [
              1.2733420047421218,
              2.7340300705675658
            ],
            "tabletopgen": [
              -2.7340300705675658,
              -1.2733420047421216
            ]
          },
          "win_rate": {
            "scenerig": 0.9743589743589743,
            "tabletopgen": 0.02564102564102564
          }
        },
        "appearance": {
          "flip_rate": 0.016949152542372836,
          "win_matrix": {
            "scenerig": {
              "scenerig": 0.0,
              "tabletopgen": 115.0
            },
            "tabletopgen": {
              "scenerig": 1.0,
              "tabletopgen": 0.0
            }
          },
          "bt_strength": {
            "scenerig": 2.1719027109268416,
            "tabletopgen": -2.171902710926842
          },
          "bt_ci95": {
            "scenerig": [
              1.7305181978281232,
              2.7340300705675658
            ],
            "tabletopgen": [
              -2.7340300705675658,
              -1.7305181978281232
            ]
          },
          "win_rate": {
            "scenerig": 0.9913793103448276,
            "tabletopgen": 0.008620689655172414
          }
        }
      },
      "cross_judge_kappa": 0.8158765159867691
    }
  },
  "PAT3D": {
    "cohort": {
      "attempted": 80,
      "completed": [
        "abc_2",
        "abc_4",
        "abc_5",
        "airoa_moma_000_row0",
        "airoa_moma_005_row1379",
        "airoa_moma_016",
        "bridge_1",
        "bridge_2",
        "bridge_3",
        "bridge_4",
        "bridge_5",
        "bridge_6",
        "bridge_7",
        "bridge_8",
        "misc_IMG_8210",
        "misc_IMG_8219",
        "misc_IMG_8226",
        "misc_IMG_8334",
        "misc_IMG_9096",
        "misc_genai1",
        "misc_genai2",
        "misc_genai3",
        "misc_genai4",
        "misc_online1",
        "misc_online2",
        "misc_online3",
        "misc_online4",
        "misc_online5",
        "misc_online6",
        "misc_online7",
        "misc_still_life",
        "misc_wendy1",
        "robodojo_arrange_largest_number_random",
        "robodojo_arrange_largest_number_standard",
        "robodojo_fold_clothes_random",
        "robodojo_hang_mugs_standard",
        "robodojo_make_toast_random",
        "robodojo_make_toast_standard",
        "robodojo_pack_objects_into_box_standard",
        "robodojo_pour_liquid_into_cup_random",
        "robodojo_pour_liquid_into_cup_standard",
        "robodojo_push_T_random",
        "robodojo_sort_nesting_dolls_by_size_random",
        "robodojo_sort_nesting_dolls_by_size_standard",
        "robodojo_stack_blocks_random",
        "robodojo_stack_blocks_standard",
        "robodojo_stack_bowls_standard",
        "robodojo_store_laptop_and_headphones_random",
        "robodojo_store_laptop_and_headphones_standard",
        "robodojo_sweep_blocks_random",
        "robodojo_sweep_blocks_standard",
        "robolab_bagel_plate_banana_bowl",
        "robolab_bin_condiments",
        "robolab_bin_mug_mustard_marker_bowl",
        "robolab_bin_mug_mustard_marker_bowl_spoon_mayo_soup",
        "robolab_breakfast_table",
        "robolab_cartons_on_box",
        "robolab_clutter_fruit_bottle_bluebin",
        "robolab_clutter_shelf",
        "robolab_cutlery_shelf",
        "robolab_foodpacking_1bin_3box_3can",
        "robolab_mugs4_measuringcup_drill_bowl",
        "robolab_mugs4_measuringcup_drill_bowl_v2",
        "robolab_mugs_on_shelf",
        "robolab_tools_container",
        "robolab_workdesk_bin",
        "robolab_workdesk_snacks"
      ],
      "synthetic": [
        "robodojo_arrange_largest_number_random",
        "robodojo_arrange_largest_number_standard",
        "robodojo_fold_clothes_random",
        "robodojo_hang_mugs_standard",
        "robodojo_make_toast_random",
        "robodojo_make_toast_standard",
        "robodojo_pack_objects_into_box_standard",
        "robodojo_pour_liquid_into_cup_random",
        "robodojo_pour_liquid_into_cup_standard",
        "robodojo_push_T_random",
        "robodojo_sort_nesting_dolls_by_size_random",
        "robodojo_sort_nesting_dolls_by_size_standard",
        "robodojo_stack_blocks_random",
        "robodojo_stack_blocks_standard",
        "robodojo_stack_bowls_standard",
        "robodojo_store_laptop_and_headphones_random",
        "robodojo_store_laptop_and_headphones_standard",
        "robodojo_sweep_blocks_random",
        "robodojo_sweep_blocks_standard",
        "robolab_bagel_plate_banana_bowl",
        "robolab_bin_condiments",
        "robolab_bin_mug_mustard_marker_bowl",
        "robolab_bin_mug_mustard_marker_bowl_spoon_mayo_soup",
        "robolab_breakfast_table",
        "robolab_cartons_on_box",
        "robolab_clutter_fruit_bottle_bluebin",
        "robolab_clutter_shelf",
        "robolab_cutlery_shelf",
        "robolab_foodpacking_1bin_3box_3can",
        "robolab_mugs4_measuringcup_drill_bowl",
        "robolab_mugs4_measuringcup_drill_bowl_v2",
        "robolab_mugs_on_shelf",
        "robolab_tools_container",
        "robolab_workdesk_bin",
        "robolab_workdesk_snacks"
      ],
      "frozen_at": "2026-11-10T06:57:20.589027+00:00"
    },
    "metrics": {
      "PAT3D": {
        "source": {
          "psnr": 9.794940040167406,
          "ssim": 0.5380128268393704,
          "lpips": 0.743166495614977,
          "negative_clip": 0.22161932934576006
        },
        "objects": {
          "psnr_objects": 9.50676012405164,
          "ssim_objects": 0.3784187951996015,
          "lpips_obj_white": 0.3150040223128454,
          "nclip_obj_white": 0.15836215527752764
        },
        "instances": {
          "ari": 0.12392252530485807,
          "fg_iou": 0.12474602921197898,
          "count_mae": 2.5714285714285716
        },
        "layout": {
          "ospa5": 0.04906,
          "ospa10": 0.09265714285714285,
          "f1_10": 0.21207142857142858
        }
      },
      "SceneRig": {
        "source": {
          "psnr": 13.271874328358209,
          "ssim": 0.5339207462686567,
          "lpips": 0.5169935820895523,
          "negative_clip": 0.12097746268656716
        },
        "objects": {
          "psnr_objects": 13.548342285714286,
          "ssim_objects": 0.4281891428571429,
          "lpips_obj_white": 0.17035485714285714,
          "nclip_obj_white": 0.06266171428571429
        },
        "instances": {
          "ari": 0.8752175588149583,
          "fg_iou": 0.814676565593948,
          "count_mae": 0.2571428571428571
        },
        "layout": {
          "ospa5": 0.024062857142857143,
          "ospa10": 0.030065714285714286,
          "f1_10": 0.9481742857142857
        }
      }
    },
    "vlm": {
      "methods": [
        "pat3d",
        "scenerig"
      ],
      "judges": [
        "gpt-5.6-terra",
        "qwen3.8-max-fireworks"
      ],
      "n_rows": 134,
      "dimensions": {
        "completeness": {
          "flip_rate": 0.10447761194029848,
          "win_matrix": {
            "pat3d": {
              "pat3d": 0.0,
              "scenerig": 4.5
            },
            "scenerig": {
              "pat3d": 115.5,
              "scenerig": 0.0
            }
          },
          "bt_strength": {
            "pat3d": -1.5720761393361322,
            "scenerig": 1.5720761393361322
          },
          "bt_ci95": {
            "pat3d": [
              -2.1762130824487955,
              -1.2445868075114022
            ],
            "scenerig": [
              1.2445868075114022,
              2.1762130824487955
            ]
          },
          "win_rate": {
            "pat3d": 0.0375,
            "scenerig": 0.9625
          }
        },
        "layout": {
          "flip_rate": 0.007462686567164201,
          "win_matrix": {
            "pat3d": {
              "pat3d": 0.0,
              "scenerig": 0.0
            },
            "scenerig": {
              "pat3d": 133.0,
              "scenerig": 0.0
            }
          },
          "bt_strength": {
            "pat3d": -2.793624329200125,
            "scenerig": 2.7936243292001244
          },
          "bt_ci95": {
            "pat3d": [
              -2.797355689800919,
              -2.7860770160888824
            ],
            "scenerig": [
              2.786077016088882,
              2.7973556898009195
            ]
          },
          "win_rate": {
            "pat3d": 0.0,
            "scenerig": 1.0
          }
        },
        "appearance": {
          "flip_rate": 0.007462686567164201,
          "win_matrix": {
            "pat3d": {
              "pat3d": 0.0,
              "scenerig": 0.0
            },
            "scenerig": {
              "pat3d": 133.0,
              "scenerig": 0.0
            }
          },
          "bt_strength": {
            "pat3d": -2.793624329200125,
            "scenerig": 2.7936243292001244
          },
          "bt_ci95": {
            "pat3d": [
              -2.797355689800919,
              -2.7860770160888824
            ],
            "scenerig": [
              2.786077016088882,
              2.7973556898009195
            ]
          },
          "win_rate": {
            "pat3d": 0.0,
            "scenerig": 1.0
          }
        }
      },
      "cross_judge_kappa": 0.8861838101034688
    }
  }
}
