{
  "date": "2026-09-22",
  "model": "gpt-6-astra",
  "reasoningEffort": "high",
  "imageDetail": "high",
  "imageEncoding": "Native-size JPEG, quality 85",
  "note": "Selected literature reports, not a controlled head-to-head. VSTI literature rows use historical labels; Astra uses corrected labels. MRA and accuracy share a 0–100 scale but are different metrics. Missing task values are null.",
  "generation": {
    "vsi": "8,192-token first pass; seven truncated responses completed with the client token cap omitted. The other 5,123 predictions are unchanged.",
    "vsti": "8,192 max output tokens; no truncations or API errors."
  },
  "vstiAggregation": {
    "macroFivePooled": 70.33431488946037,
    "macroNine": 78.83883208946025,
    "macroFiveSubtypes": 70.00681301877627,
    "microAll": 75.33298465829847
  },
  "benchmarks": [
    {
      "id": "vsi",
      "name": "VSI-Bench",
      "questions": 5130,
      "datasetRevision": "bdcadb3fea447621a828a24911801faba3587c12",
      "aggregation": "Eight-task macro; direction difficulties averaged first",
      "sampling": "1 FPS, full video, no frame cap",
      "tasks": [
        {
          "key": "object_counting",
          "label": "Object counting",
          "zh": "物体数量",
          "metric": "MRA"
        },
        {
          "key": "object_abs_distance",
          "label": "Absolute distance",
          "zh": "绝对距离",
          "metric": "MRA"
        },
        {
          "key": "object_size_estimation",
          "label": "Object size",
          "zh": "物体大小",
          "metric": "MRA"
        },
        {
          "key": "room_size_estimation",
          "label": "Room size",
          "zh": "房间大小",
          "metric": "MRA"
        },
        {
          "key": "object_rel_distance",
          "label": "Relative distance",
          "zh": "相对距离",
          "metric": "Accuracy"
        },
        {
          "key": "object_rel_direction",
          "label": "Relative direction",
          "zh": "相对方向",
          "metric": "Accuracy"
        },
        {
          "key": "route_planning",
          "label": "Route planning",
          "zh": "路线规划",
          "metric": "Accuracy"
        },
        {
          "key": "obj_appearance_order",
          "label": "Appearance order",
          "zh": "出现顺序",
          "metric": "Accuracy"
        }
      ],
      "rows": [
        {
          "model": "GPT-6-Astra",
          "group": "general",
          "local": true,
          "score": 73.0476972219933,
          "precision": 2,
          "tasks": [
            61.823008849557525,
            61.534772182254194,
            78.42602308499477,
            56.49305555555555,
            76.05633802816901,
            82.48423901512777,
            87.62886597938144,
            79.93527508090615
          ],
          "sources": []
        },
        {
          "model": "GPT-5.4",
          "group": "general",
          "local": false,
          "score": 49.2,
          "precision": 1,
          "tasks": [
            null,
            null,
            null,
            null,
            null,
            null,
            null,
            null
          ],
          "sources": [
            {
              "title": "RynnBrain 1.1",
              "url": "https://arxiv.org/html/2607.17977v2#S5.T5",
              "version": "2607.17977v2 · 2026-07-31"
            }
          ]
        },
        {
          "model": "Gemini-3-Pro",
          "group": "general",
          "local": false,
          "score": 56,
          "precision": 1,
          "tasks": [
            49,
            42.8,
            71.5,
            41.8,
            56.6,
            57.5,
            61.9,
            60
          ],
          "sources": [
            {
              "title": "SpaceMind · CVPR 2026 终稿",
              "url": "https://openaccess.thecvf.com/content/CVPR2026/papers/Zhao_SpaceMind_Camera-Guided_Modality_Fusion_for_Spatial_Reasoning_in_Vision-Language_Models_CVPR_2026_paper.pdf#page=6",
              "version": "CVPR 2026 正式论文集 · pp. 16811–16822"
            }
          ]
        },
        {
          "model": "Claude Sonnet 4.6",
          "group": "general",
          "local": false,
          "score": 44.4,
          "precision": 1,
          "tasks": [
            null,
            null,
            null,
            null,
            null,
            null,
            null,
            null
          ],
          "sources": [
            {
              "title": "RynnBrain 1.1",
              "url": "https://arxiv.org/html/2607.17977v2#S5.T5",
              "version": "2607.17977v2 · 2026-07-31"
            }
          ]
        },
        {
          "model": "Qwen3.5-122B-A10B",
          "group": "general",
          "local": false,
          "score": 66.6,
          "precision": 1,
          "tasks": [
            null,
            null,
            null,
            null,
            null,
            null,
            null,
            null
          ],
          "sources": [
            {
              "title": "RynnBrain 1.1",
              "url": "https://arxiv.org/html/2607.17977v2#S5.T5",
              "version": "2607.17977v2 · 2026-07-31"
            }
          ]
        },
        {
          "model": "Qwen3.5-9B",
          "group": "general",
          "local": false,
          "score": 57.9,
          "precision": 1,
          "tasks": [
            62,
            44,
            73.6,
            61.9,
            69.7,
            79.9,
            45.4,
            26.7
          ],
          "sources": [
            {
              "title": "Annotations as Rollouts",
              "url": "https://arxiv.org/html/2608.20492v1#S4.T6",
              "version": "2608.20492v1 · 2026-08-20"
            }
          ]
        },
        {
          "model": "InternVL3.5-8B",
          "group": "general",
          "local": false,
          "score": 56.1,
          "precision": 1,
          "tasks": [
            70.7,
            43.3,
            68.1,
            63.6,
            55.6,
            49.1,
            36.6,
            61.5
          ],
          "sources": [
            {
              "title": "SpatioLM",
              "url": "https://arxiv.org/html/2608.01899v1#S3.T2",
              "version": "2608.01899v1 · 2026-08-03"
            }
          ]
        },
        {
          "model": "ConsiSpace UC-SSRL",
          "group": "enhanced",
          "local": false,
          "score": 76.6,
          "precision": 1,
          "tasks": [
            77.9,
            78.9,
            82.2,
            76.3,
            72.1,
            90.1,
            54.7,
            79.9
          ],
          "sources": [
            {
              "title": "ConsiSpace",
              "url": "https://arxiv.org/html/2607.17599v1#S3.T1",
              "version": "2607.17599v1 · 2026-07-20"
            }
          ]
        },
        {
          "model": "RynnBrain 1.1-122B-A10B",
          "group": "enhanced",
          "local": false,
          "score": 75,
          "precision": 1,
          "tasks": [
            null,
            null,
            null,
            null,
            null,
            null,
            null,
            null
          ],
          "sources": [
            {
              "title": "RynnBrain 1.1",
              "url": "https://arxiv.org/html/2607.17977v2#S5.T5",
              "version": "2607.17977v2 · 2026-07-31"
            }
          ]
        },
        {
          "model": "Molmo2-ER-5B",
          "group": "enhanced",
          "local": false,
          "score": 74.5,
          "precision": 1,
          "tasks": [
            null,
            null,
            null,
            null,
            null,
            null,
            null,
            null
          ],
          "sources": [
            {
              "title": "RynnBrain 1.1",
              "url": "https://arxiv.org/html/2607.17977v2#S5.T4",
              "version": "2607.17977v2 · 2026-07-31"
            }
          ]
        },
        {
          "model": "Cambrian-P",
          "group": "enhanced",
          "local": false,
          "score": 73.7,
          "precision": 1,
          "tasks": [
            74.9,
            60.1,
            76,
            76.9,
            74.8,
            89.5,
            52.6,
            85
          ],
          "sources": [
            {
              "title": "Cambrian-P",
              "url": "https://arxiv.org/html/2605.22819v2#S4.T1",
              "version": "2605.22819v2 · 2026-07-23"
            }
          ]
        },
        {
          "model": "Cambrian-S-7B",
          "group": "enhanced",
          "local": false,
          "score": 67.5,
          "precision": 1,
          "tasks": [
            73.2,
            50.5,
            74.9,
            72.2,
            71.1,
            76.2,
            41.8,
            80.1
          ],
          "sources": [
            {
              "title": "Cambrian-S",
              "url": "https://arxiv.org/html/2511.04670v1#S3.T6",
              "version": "2511.04670v1 · 2025-11-06"
            }
          ]
        },
        {
          "model": "VLM-3R-7B",
          "group": "enhanced",
          "local": false,
          "score": 60.9,
          "precision": 1,
          "tasks": [
            70.2,
            49.4,
            69.2,
            67.1,
            65.4,
            80.5,
            45.4,
            40.1
          ],
          "sources": [
            {
              "title": "VLM-3R",
              "url": "https://arxiv.org/html/2505.20279v5#S5.T1",
              "version": "2505.20279v5 · 2026-04-21"
            }
          ]
        },
        {
          "model": "SpaceMind",
          "group": "enhanced",
          "local": false,
          "score": 70.2,
          "precision": 1,
          "tasks": [
            73.9,
            61.5,
            77.6,
            74.8,
            67.7,
            88.6,
            46.9,
            70.7
          ],
          "sources": [
            {
              "title": "SpaceMind · CVPR 2026 终稿",
              "url": "https://openaccess.thecvf.com/content/CVPR2026/papers/Zhao_SpaceMind_Camera-Guided_Modality_Fusion_for_Spatial_Reasoning_in_Vision-Language_Models_CVPR_2026_paper.pdf#page=6",
              "version": "CVPR 2026 正式论文集 · pp. 16811–16822"
            }
          ]
        },
        {
          "model": "Spatial-MLLM-4B",
          "group": "enhanced",
          "local": false,
          "score": 47,
          "precision": 1,
          "tasks": [
            65.3,
            34.8,
            63.1,
            45.1,
            41.3,
            46.9,
            33.5,
            46.3
          ],
          "sources": [
            {
              "title": "Spatial-MLLM",
              "url": "https://arxiv.org/html/2505.23747v2#A3.T5",
              "version": "2505.23747v2 · 2026-05-19"
            }
          ]
        },
        {
          "model": "SpatioLM (SenseNova-SI-8B)",
          "group": "enhanced",
          "local": false,
          "score": 71.6,
          "precision": 1,
          "tasks": [
            81.9,
            55.7,
            75.2,
            71.5,
            78.2,
            74.4,
            47.9,
            87.8
          ],
          "sources": [
            {
              "title": "SpatioLM",
              "url": "https://arxiv.org/html/2608.01899v1#S3.T2",
              "version": "2608.01899v1 · 2026-08-03"
            }
          ]
        },
        {
          "model": "LLaVA-OneVision-2-8B",
          "group": "enhanced",
          "local": false,
          "score": 70.9,
          "precision": 1,
          "tasks": [
            null,
            null,
            null,
            null,
            null,
            null,
            null,
            null
          ],
          "sources": [
            {
              "title": "Annotations as Rollouts",
              "url": "https://arxiv.org/html/2608.20492v1#S4.T6",
              "version": "2608.20492v1 · 2026-08-20"
            }
          ]
        }
      ],
      "defaultComparison": [
        "GPT-6-Astra",
        "Cambrian-S-7B",
        "Gemini-3-Pro",
        "ConsiSpace UC-SSRL",
        "Cambrian-P",
        "SpatioLM (SenseNova-SI-8B)",
        "Qwen3.5-9B"
      ]
    },
    {
      "id": "vsti",
      "name": "VSTI-Bench",
      "questions": 5736,
      "datasetRevision": "0903eb0b0c9d03da730d43503c48302a1f3b412d",
      "aggregation": "Five-family macro; questions pooled within each family",
      "sampling": "32 uniformly sampled frames",
      "tasks": [
        {
          "key": "camera_obj_abs_dist",
          "label": "Camera–object distance",
          "zh": "相机–物体距离",
          "metric": "MRA"
        },
        {
          "key": "camera_displacement",
          "label": "Camera displacement",
          "zh": "相机位移",
          "metric": "MRA"
        },
        {
          "key": "camera_movement_direction",
          "label": "Camera movement direction",
          "zh": "相机运动方向",
          "metric": "Accuracy"
        },
        {
          "key": "obj_obj_relative_pos",
          "label": "Object relative position",
          "zh": "物体相对位置",
          "metric": "Accuracy"
        },
        {
          "key": "camera_obj_rel_dist",
          "label": "Camera–object relative distance",
          "zh": "相机–物体相对距离",
          "metric": "Accuracy"
        }
      ],
      "rows": [
        {
          "model": "GPT-6-Astra",
          "group": "general",
          "local": true,
          "score": 70.33431488946037,
          "precision": 2,
          "tasks": [
            68.82872928176796,
            42.52100840336134,
            58.92661555312158,
            94.58966565349544,
            86.80555555555556
          ],
          "sources": []
        },
        {
          "model": "GPT-5",
          "group": "general",
          "local": false,
          "score": 44.1,
          "precision": 1,
          "tasks": [
            24.3,
            11.1,
            41.9,
            85.8,
            57.3
          ],
          "sources": [
            {
              "title": "VLM² · Table 2",
              "url": "https://arxiv.org/html/2511.20644v2#S4.T2",
              "version": "v2 · 2026-07-09"
            }
          ]
        },
        {
          "model": "Gemini-3-Pro",
          "group": "general",
          "local": false,
          "score": 43.3,
          "precision": 1,
          "tasks": [
            30.7,
            9.7,
            35.6,
            82.5,
            58.1
          ],
          "sources": [
            {
              "title": "VLM² · Table 2",
              "url": "https://arxiv.org/html/2511.20644v2#S4.T2",
              "version": "v2 · 2026-07-09"
            }
          ]
        },
        {
          "model": "Qwen3-VL-8B",
          "group": "general",
          "local": false,
          "score": 47,
          "precision": 1,
          "tasks": [
            26.1,
            31.8,
            37.1,
            79.7,
            60.1
          ],
          "sources": [
            {
              "title": "VLM² · Table 2",
              "url": "https://arxiv.org/html/2511.20644v2#S4.T2",
              "version": "v2 · 2026-07-09"
            }
          ]
        },
        {
          "model": "Cambrian-P",
          "group": "enhanced",
          "local": false,
          "score": 68.9,
          "precision": 1,
          "tasks": [
            42.5,
            46.6,
            87.7,
            94.3,
            73.2
          ],
          "sources": [
            {
              "title": "Cambrian-P",
              "url": "https://arxiv.org/html/2605.22819v1#S4.T2",
              "version": "v1 · 2026-05-21（v2 Table 2 数值相同）"
            }
          ]
        },
        {
          "model": "Q-GeoMem-7B",
          "group": "enhanced",
          "local": false,
          "score": 67.8,
          "precision": 1,
          "tasks": [
            44.3,
            44.6,
            84,
            91,
            74.9
          ],
          "sources": [
            {
              "title": "Q-GeoMem · Table 2",
              "url": "https://arxiv.org/html/2605.27318v2#S4.T2",
              "version": "v2 · 2026-07-06"
            }
          ]
        },
        {
          "model": "VLM²-7B",
          "group": "enhanced",
          "local": false,
          "score": 65.3,
          "precision": 1,
          "tasks": [
            43.1,
            44.1,
            76.8,
            87.7,
            74.9
          ],
          "sources": [
            {
              "title": "VLM² · Table 2",
              "url": "https://arxiv.org/html/2511.20644v2#S4.T2",
              "version": "v2 · 2026-07-09"
            }
          ]
        },
        {
          "model": "VLM-3R-7B",
          "group": "enhanced",
          "local": false,
          "score": 58.8,
          "precision": 1,
          "tasks": [
            39.4,
            39.6,
            60.6,
            86.5,
            68.6
          ],
          "sources": [
            {
              "title": "VLM-3R · CVPR 2026",
              "url": "https://arxiv.org/html/2505.20279",
              "version": "arXiv:2505.20279"
            }
          ]
        },
        {
          "model": "Cog3DMap-8B",
          "group": "enhanced",
          "local": false,
          "score": 67.5,
          "precision": 1,
          "tasks": [
            40.9,
            47.1,
            88.1,
            90.9,
            70.6
          ],
          "sources": [
            {
              "title": "Cog3DMap",
              "url": "https://arxiv.org/html/2603.23023v1#S4.T1",
              "version": "v1 · 2026-03-24"
            }
          ]
        },
        {
          "model": "GeoThinker",
          "group": "enhanced",
          "local": false,
          "score": 67.4,
          "precision": 1,
          "tasks": [
            38.4,
            45.8,
            84.2,
            93.6,
            75.2
          ],
          "sources": [
            {
              "title": "GeoThinker",
              "url": "https://arxiv.org/html/2602.06037v1#A6.T11",
              "version": "v1 · 2026-02-05"
            }
          ]
        },
        {
          "model": "4D-RGPT-8B",
          "group": "enhanced",
          "local": false,
          "score": 59.1,
          "precision": 1,
          "tasks": [
            null,
            null,
            null,
            null,
            null
          ],
          "sources": [
            {
              "title": "4D-RGPT · Table 2 & Table A1",
              "url": "https://arxiv.org/html/2512.17012v4",
              "version": "v4 · 2026-06-02"
            }
          ]
        }
      ],
      "defaultComparison": [
        "GPT-6-Astra",
        "Gemini-3-Pro",
        "Qwen3-VL-8B",
        "Cambrian-P",
        "Q-GeoMem-7B",
        "VLM²-7B",
        "VLM-3R-7B"
      ]
    }
  ]
}