{ "schemaVersion": "b18.sft-context-resolver.r1", "generatedAt": "2026-08-21T03:30:00.000Z", "implementation": { "name": "Glyphd SFT Context Resolver", "version": "0.2.0", "benchmarkVersion": "B18-R1", "benchmarkClass": "deterministic operation-directed context comparison", "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "latencySamplesPerCaseStrategy": 25 }, "cases": [ { "id": "architecture-fit", "scenarioId": "architecture", "query": "Can the lobby sofa fit through the front door?", "expectedSolverStatus": "PASS", "requiredEntityIds": [ "sofa", "front-door" ], "requiredRelationIds": [], "requiredMetricFactIds": [ "entity:sofa:bounds", "entity:front-door:bounds" ], "typedCommandId": "sha256:28f67bf8ae2199a8d9fc904e5592e8aab948cbf9f2c62f039fe0e2c971c412f6", "sceneRevisionId": "sha256:71880593b846615ca053fbdbfa272d37768a7f1ba74e9187f4b4dabc8b9a8248", "sceneContentId": "sha256:58a75ea1d736e5c3922fd123a48b5559106b3c98b950e3550c9862273fb8f6c2", "fullSceneStatus": "PASS", "sftOperationalStatus": "PASS", "sftIncludesIndependentOracle": true, "sftDetectsMissingEvidence": true, "rows": [ { "strategy": "FULL_SCENE", "contextEntityCount": 13, "contextRelationCount": 17, "serializedBytes": 7870, "serializedTokens": 2157, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 1, "requiredRelationRecall": 1, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 10, "irrelevantRelationCount": 17, "irrelevantEntityRate": 0.8333333333333334, "irrelevantRelationRate": 1, "solverStatus": "PASS", "solverStatusMatchesFullScene": true, "solverResultMatchesFullScene": true, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.466375, "p50": 0.560417, "p95": 0.765583, "max": 0.955, "mean": 0.579163 } }, { "strategy": "LEXICAL_TOP_K", "contextEntityCount": 8, "contextRelationCount": 8, "serializedBytes": 4280, "serializedTokens": 1192, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 1, "requiredRelationRecall": 1, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 5, "irrelevantRelationCount": 8, "irrelevantEntityRate": 0.7142857142857143, "irrelevantRelationRate": 1, "solverStatus": "PASS", "solverStatusMatchesFullScene": true, "solverResultMatchesFullScene": true, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.484208, "p50": 0.515834, "p95": 0.726959, "max": 0.753708, "mean": 0.55817 } }, { "strategy": "GRAPH_NEIGHBORHOOD", "contextEntityCount": 8, "contextRelationCount": 10, "serializedBytes": 4762, "serializedTokens": 1319, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 1, "requiredRelationRecall": 1, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 5, "irrelevantRelationCount": 10, "irrelevantEntityRate": 0.7142857142857143, "irrelevantRelationRate": 1, "solverStatus": "PASS", "solverStatusMatchesFullScene": true, "solverResultMatchesFullScene": true, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.489541, "p50": 0.524166, "p95": 0.701666, "max": 0.754666, "mean": 0.544972 } }, { "strategy": "SFT_OPERATIONAL", "contextEntityCount": 6, "contextRelationCount": 5, "serializedBytes": 3047, "serializedTokens": 853, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 1, "requiredRelationRecall": 1, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 3, "irrelevantRelationCount": 5, "irrelevantEntityRate": 0.6, "irrelevantRelationRate": 1, "solverStatus": "PASS", "solverStatusMatchesFullScene": true, "solverResultMatchesFullScene": true, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.374666, "p50": 0.408167, "p95": 0.562959, "max": 0.597458, "mean": 0.423927 } } ] }, { "id": "architecture-service-access", "scenarioId": "architecture", "query": "Audit service access for air handler", "expectedSolverStatus": "FAIL", "requiredEntityIds": [ "air-handler", "air-handler-access", "stored-crate" ], "requiredRelationIds": [ "services:air-handler:zone" ], "requiredMetricFactIds": [ "entity:air-handler-access:bounds", "entity:stored-crate:bounds" ], "typedCommandId": "sha256:5f53ec0738df83644545d96337668845b1efa04a5f0bee9ac8f7290e4589e875", "sceneRevisionId": "sha256:71880593b846615ca053fbdbfa272d37768a7f1ba74e9187f4b4dabc8b9a8248", "sceneContentId": "sha256:58a75ea1d736e5c3922fd123a48b5559106b3c98b950e3550c9862273fb8f6c2", "fullSceneStatus": "FAIL", "sftOperationalStatus": "FAIL", "sftIncludesIndependentOracle": true, "sftDetectsMissingEvidence": true, "rows": [ { "strategy": "FULL_SCENE", "contextEntityCount": 13, "contextRelationCount": 17, "serializedBytes": 7870, "serializedTokens": 2157, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 1, "requiredRelationRecall": 1, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 9, "irrelevantRelationCount": 16, "irrelevantEntityRate": 0.75, "irrelevantRelationRate": 0.9411764705882353, "solverStatus": "FAIL", "solverStatusMatchesFullScene": true, "solverResultMatchesFullScene": true, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.455, "p50": 0.508834, "p95": 0.76925, "max": 0.850292, "mean": 0.544944 } }, { "strategy": "LEXICAL_TOP_K", "contextEntityCount": 6, "contextRelationCount": 7, "serializedBytes": 3627, "serializedTokens": 981, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 0.6666666666666666, "requiredRelationRecall": 1, "requiredMetricFactRecall": 0.5, "irrelevantEntityCount": 3, "irrelevantRelationCount": 6, "irrelevantEntityRate": 0.6, "irrelevantRelationRate": 0.8571428571428571, "solverStatus": "UNKNOWN", "solverStatusMatchesFullScene": false, "solverResultMatchesFullScene": false, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.422, "p50": 0.445583, "p95": 0.545083, "max": 0.698708, "mean": 0.460248 } }, { "strategy": "GRAPH_NEIGHBORHOOD", "contextEntityCount": 11, "contextRelationCount": 14, "serializedBytes": 6608, "serializedTokens": 1809, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 1, "requiredRelationRecall": 1, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 7, "irrelevantRelationCount": 13, "irrelevantEntityRate": 0.7, "irrelevantRelationRate": 0.9285714285714286, "solverStatus": "FAIL", "solverStatusMatchesFullScene": true, "solverResultMatchesFullScene": true, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.612209, "p50": 0.662041, "p95": 0.85375, "max": 0.882209, "mean": 0.681545 } }, { "strategy": "SFT_OPERATIONAL", "contextEntityCount": 7, "contextRelationCount": 7, "serializedBytes": 3948, "serializedTokens": 1076, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 1, "requiredRelationRecall": 1, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 3, "irrelevantRelationCount": 6, "irrelevantEntityRate": 0.5, "irrelevantRelationRate": 0.8571428571428571, "solverStatus": "FAIL", "solverStatusMatchesFullScene": true, "solverResultMatchesFullScene": true, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.441875, "p50": 0.495125, "p95": 0.871125, "max": 0.979458, "mean": 0.568067 } } ] }, { "id": "architecture-clearance-policy", "scenarioId": "architecture", "query": "Check architectural clearance in ground floor", "expectedSolverStatus": "FAIL", "requiredEntityIds": [ "ground-floor", "front-door", "mechanical-door", "lobby", "corridor", "mechanical-room" ], "requiredRelationIds": [ "lobby-front-door-corridor", "corridor-mechanical-door-room" ], "requiredMetricFactIds": [ "entity:front-door:bounds", "entity:mechanical-door:bounds", "entity:lobby:attributes.routeWidthMm", "entity:corridor:attributes.routeWidthMm", "entity:mechanical-room:attributes.routeWidthMm", "policy:fixture-project-policy:minimumOpeningWidth", "policy:fixture-project-policy:minimumHeadroom", "policy:fixture-project-policy:minimumRouteWidth" ], "typedCommandId": "sha256:f1629ece7acfc0232865cf2931a0cbc1ab918047aa8957adcc6d26050cc56f27", "sceneRevisionId": "sha256:71880593b846615ca053fbdbfa272d37768a7f1ba74e9187f4b4dabc8b9a8248", "sceneContentId": "sha256:58a75ea1d736e5c3922fd123a48b5559106b3c98b950e3550c9862273fb8f6c2", "fullSceneStatus": "FAIL", "sftOperationalStatus": "FAIL", "sftIncludesIndependentOracle": true, "sftDetectsMissingEvidence": true, "rows": [ { "strategy": "FULL_SCENE", "contextEntityCount": 13, "contextRelationCount": 17, "serializedBytes": 7870, "serializedTokens": 2157, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 1, "requiredRelationRecall": 1, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 6, "irrelevantRelationCount": 15, "irrelevantEntityRate": 0.5, "irrelevantRelationRate": 0.8823529411764706, "solverStatus": "FAIL", "solverStatusMatchesFullScene": true, "solverResultMatchesFullScene": true, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.521167, "p50": 0.561708, "p95": 0.837083, "max": 0.844375, "mean": 0.60607 } }, { "strategy": "LEXICAL_TOP_K", "contextEntityCount": 6, "contextRelationCount": 6, "serializedBytes": 3444, "serializedTokens": 937, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 0.5, "requiredRelationRecall": 0.5, "requiredMetricFactRecall": 0.625, "irrelevantEntityCount": 2, "irrelevantRelationCount": 5, "irrelevantEntityRate": 0.4, "irrelevantRelationRate": 0.8333333333333334, "solverStatus": "UNKNOWN", "solverStatusMatchesFullScene": false, "solverResultMatchesFullScene": false, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.487084, "p50": 0.50475, "p95": 0.547958, "max": 0.8035, "mean": 0.519442 } }, { "strategy": "GRAPH_NEIGHBORHOOD", "contextEntityCount": 13, "contextRelationCount": 17, "serializedBytes": 7834, "serializedTokens": 2151, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 1, "requiredRelationRecall": 1, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 6, "irrelevantRelationCount": 15, "irrelevantEntityRate": 0.5, "irrelevantRelationRate": 0.8823529411764706, "solverStatus": "FAIL", "solverStatusMatchesFullScene": true, "solverResultMatchesFullScene": true, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.732, "p50": 0.791083, "p95": 0.961458, "max": 1.03275, "mean": 0.809053 } }, { "strategy": "SFT_OPERATIONAL", "contextEntityCount": 13, "contextRelationCount": 14, "serializedBytes": 7230, "serializedTokens": 1997, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 1, "requiredRelationRecall": 1, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 6, "irrelevantRelationCount": 12, "irrelevantEntityRate": 0.5, "irrelevantRelationRate": 0.8571428571428571, "solverStatus": "FAIL", "solverStatusMatchesFullScene": true, "solverResultMatchesFullScene": true, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.694542, "p50": 0.728959, "p95": 0.923208, "max": 0.937959, "mean": 0.755148 } } ] }, { "id": "assembly-order", "scenarioId": "assembly", "query": "Audit assembly order for cabinet assembly", "expectedSolverStatus": "PASS", "requiredEntityIds": [ "cabinet", "cabinet-base", "left-side", "right-side", "fixed-shelf", "cabinet-top", "cabinet-door" ], "requiredRelationIds": [ "order:base:left", "order:base:right", "order:left:shelf", "order:right:shelf", "order:shelf:top", "order:top:door" ], "requiredMetricFactIds": [], "typedCommandId": "sha256:fb375e31bac6a9afd0769700e6495c09f402b01b963255108f04080b0398f2ed", "sceneRevisionId": "sha256:e6fee0ee7a7ad9484b7113da27d2f68a16f4aa79258217ff85e8643992099c2a", "sceneContentId": "sha256:e7dfa157b713fcffee9c9f039fc46d2ab565fd1d03f214d63c56d808a353cce6", "fullSceneStatus": "PASS", "sftOperationalStatus": "PASS", "sftIncludesIndependentOracle": true, "sftDetectsMissingEvidence": true, "rows": [ { "strategy": "FULL_SCENE", "contextEntityCount": 8, "contextRelationCount": 14, "serializedBytes": 5125, "serializedTokens": 1417, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 1, "requiredRelationRecall": 1, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 0, "irrelevantRelationCount": 8, "irrelevantEntityRate": 0, "irrelevantRelationRate": 0.5714285714285714, "solverStatus": "PASS", "solverStatusMatchesFullScene": true, "solverResultMatchesFullScene": true, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.316209, "p50": 0.343625, "p95": 0.52225, "max": 0.602125, "mean": 0.370413 } }, { "strategy": "LEXICAL_TOP_K", "contextEntityCount": 5, "contextRelationCount": 5, "serializedBytes": 2575, "serializedTokens": 722, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 0.5714285714285714, "requiredRelationRecall": 0.16666666666666666, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 0, "irrelevantRelationCount": 4, "irrelevantEntityRate": 0, "irrelevantRelationRate": 0.8, "solverStatus": "UNKNOWN", "solverStatusMatchesFullScene": false, "solverResultMatchesFullScene": false, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.302833, "p50": 0.320625, "p95": 0.403791, "max": 0.57075, "mean": 0.34113 } }, { "strategy": "GRAPH_NEIGHBORHOOD", "contextEntityCount": 8, "contextRelationCount": 14, "serializedBytes": 5121, "serializedTokens": 1419, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 1, "requiredRelationRecall": 1, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 0, "irrelevantRelationCount": 8, "irrelevantEntityRate": 0, "irrelevantRelationRate": 0.5714285714285714, "solverStatus": "PASS", "solverStatusMatchesFullScene": true, "solverResultMatchesFullScene": true, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.46025, "p50": 0.48425, "p95": 0.714292, "max": 0.762667, "mean": 0.506807 } }, { "strategy": "SFT_OPERATIONAL", "contextEntityCount": 8, "contextRelationCount": 14, "serializedBytes": 5108, "serializedTokens": 1417, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 1, "requiredRelationRecall": 1, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 0, "irrelevantRelationCount": 8, "irrelevantEntityRate": 0, "irrelevantRelationRate": 0.5714285714285714, "solverStatus": "PASS", "solverStatusMatchesFullScene": true, "solverResultMatchesFullScene": true, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.455125, "p50": 0.48275, "p95": 1.214084, "max": 5.152083, "mean": 0.746848 } } ] }, { "id": "robot-alternate-route", "scenarioId": "robot", "query": "Find a path from mobile robot to charging dock", "expectedSolverStatus": "PASS", "requiredEntityIds": [ "mobile-robot", "robot-start", "junction", "wide-aisle", "charging-dock" ], "requiredRelationIds": [ "robot-at-start", "start-junction", "junction-wide", "wide-dock" ], "requiredMetricFactIds": [], "typedCommandId": "sha256:9acc2bd0387e70385dc72161e842ec2bcd57adfe18646902810f4971dda74c4b", "sceneRevisionId": "sha256:38b507c89cd9c02af209c426fa54660816bbb7ffabcbbd254e491e3f62f688ee", "sceneContentId": "sha256:3f8a676e211c1237db593d8d3f72963ad59dc2e365b472c0547f5ae8378c2aaa", "fullSceneStatus": "PASS", "sftOperationalStatus": "PASS", "sftIncludesIndependentOracle": true, "sftDetectsMissingEvidence": true, "rows": [ { "strategy": "FULL_SCENE", "contextEntityCount": 8, "contextRelationCount": 13, "serializedBytes": 5420, "serializedTokens": 1506, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 1, "requiredRelationRecall": 1, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 2, "irrelevantRelationCount": 9, "irrelevantEntityRate": 0.2857142857142857, "irrelevantRelationRate": 0.6923076923076923, "solverStatus": "PASS", "solverStatusMatchesFullScene": true, "solverResultMatchesFullScene": true, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.320084, "p50": 0.343083, "p95": 0.416958, "max": 0.574458, "mean": 0.356648 } }, { "strategy": "LEXICAL_TOP_K", "contextEntityCount": 6, "contextRelationCount": 9, "serializedBytes": 3983, "serializedTokens": 1103, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 0.8, "requiredRelationRecall": 0.5, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 1, "irrelevantRelationCount": 7, "irrelevantEntityRate": 0.2, "irrelevantRelationRate": 0.7777777777777778, "solverStatus": "UNKNOWN", "solverStatusMatchesFullScene": false, "solverResultMatchesFullScene": false, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.396666, "p50": 0.427958, "p95": 0.517042, "max": 0.581208, "mean": 0.437808 } }, { "strategy": "GRAPH_NEIGHBORHOOD", "contextEntityCount": 8, "contextRelationCount": 13, "serializedBytes": 5398, "serializedTokens": 1501, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 1, "requiredRelationRecall": 1, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 2, "irrelevantRelationCount": 9, "irrelevantEntityRate": 0.2857142857142857, "irrelevantRelationRate": 0.6923076923076923, "solverStatus": "PASS", "solverStatusMatchesFullScene": true, "solverResultMatchesFullScene": true, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.477958, "p50": 0.515667, "p95": 0.750542, "max": 0.789208, "mean": 0.542288 } }, { "strategy": "SFT_OPERATIONAL", "contextEntityCount": 7, "contextRelationCount": 12, "serializedBytes": 4894, "serializedTokens": 1360, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 1, "requiredRelationRecall": 1, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 1, "irrelevantRelationCount": 8, "irrelevantEntityRate": 0.16666666666666666, "irrelevantRelationRate": 0.6666666666666666, "solverStatus": "PASS", "solverStatusMatchesFullScene": true, "solverResultMatchesFullScene": true, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.442875, "p50": 0.461167, "p95": 0.572958, "max": 0.719792, "mean": 0.476233 } } ] }, { "id": "game-locked-exit", "scenarioId": "game", "query": "Find a path from player to exit", "expectedSolverStatus": "FAIL", "requiredEntityIds": [ "player", "start-room", "hall", "exit" ], "requiredRelationIds": [ "player-start", "start-hall", "hall-exit-door" ], "requiredMetricFactIds": [], "typedCommandId": "sha256:0ee34fa75c70bf8e8ea2ac56f685593785a8672e50de3fb14156d8e52b7009e6", "sceneRevisionId": "sha256:a6c6bc1bc03c60a131762be0445272382b681acea77c40f0624c8331d7378a36", "sceneContentId": "sha256:e41f727a1e80eede766d997f11c580181e4d42f170ae40efcb3f22bd71f1d7c6", "fullSceneStatus": "FAIL", "sftOperationalStatus": "FAIL", "sftIncludesIndependentOracle": true, "sftDetectsMissingEvidence": true, "rows": [ { "strategy": "FULL_SCENE", "contextEntityCount": 7, "contextRelationCount": 8, "serializedBytes": 3161, "serializedTokens": 848, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 1, "requiredRelationRecall": 1, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 2, "irrelevantRelationCount": 5, "irrelevantEntityRate": 0.3333333333333333, "irrelevantRelationRate": 0.625, "solverStatus": "FAIL", "solverStatusMatchesFullScene": true, "solverResultMatchesFullScene": true, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.202916, "p50": 0.222666, "p95": 0.269, "max": 0.420708, "mean": 0.230583 } }, { "strategy": "LEXICAL_TOP_K", "contextEntityCount": 4, "contextRelationCount": 4, "serializedBytes": 1791, "serializedTokens": 479, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 0.75, "requiredRelationRecall": 0.3333333333333333, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 0, "irrelevantRelationCount": 3, "irrelevantEntityRate": 0, "irrelevantRelationRate": 0.75, "solverStatus": "UNKNOWN", "solverStatusMatchesFullScene": false, "solverResultMatchesFullScene": false, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.216125, "p50": 0.221875, "p95": 0.263459, "max": 0.318125, "mean": 0.229202 } }, { "strategy": "GRAPH_NEIGHBORHOOD", "contextEntityCount": 5, "contextRelationCount": 7, "serializedBytes": 2497, "serializedTokens": 665, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 1, "requiredRelationRecall": 1, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 0, "irrelevantRelationCount": 4, "irrelevantEntityRate": 0, "irrelevantRelationRate": 0.5714285714285714, "solverStatus": "FAIL", "solverStatusMatchesFullScene": true, "solverResultMatchesFullScene": true, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.254583, "p50": 0.265792, "p95": 0.303917, "max": 0.369667, "mean": 0.274192 } }, { "strategy": "SFT_OPERATIONAL", "contextEntityCount": 5, "contextRelationCount": 7, "serializedBytes": 2484, "serializedTokens": 663, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 1, "requiredRelationRecall": 1, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 0, "irrelevantRelationCount": 4, "irrelevantEntityRate": 0, "irrelevantRelationRate": 0.5714285714285714, "solverStatus": "FAIL", "solverStatusMatchesFullScene": true, "solverResultMatchesFullScene": true, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.248667, "p50": 0.255958, "p95": 0.3795, "max": 0.445583, "mean": 0.27294 } } ] }, { "id": "game-clearance-missing-evidence-control", "scenarioId": "game", "query": "[typed control] Find a 900 mm-clear path from player to exit", "command": { "kind": "QUERY_REACHABILITY", "fromId": "player", "toId": "exit", "minimumClearance": { "min": 900, "max": 900, "unit": "mm" } }, "expectedSolverStatus": "UNKNOWN", "expectsMissingEvidence": true, "requiredEntityIds": [ "player", "start-room", "hall", "exit" ], "requiredRelationIds": [ "player-start", "start-hall", "hall-exit-door" ], "requiredMetricFactIds": [], "missingMetricFactIds": [ "relation:player-start:attributes.minimumClearanceMm", "relation:start-hall:attributes.minimumClearanceMm", "relation:hall-exit-door:attributes.minimumClearanceMm" ], "typedCommandId": "sha256:694d0bacff5d4e420db17368795dc0a8e42117ab21c3aeb91fe5725735d62b45", "sceneRevisionId": "sha256:a6c6bc1bc03c60a131762be0445272382b681acea77c40f0624c8331d7378a36", "sceneContentId": "sha256:e41f727a1e80eede766d997f11c580181e4d42f170ae40efcb3f22bd71f1d7c6", "fullSceneStatus": "UNKNOWN", "sftOperationalStatus": "UNKNOWN", "sftIncludesIndependentOracle": true, "sftDetectsMissingEvidence": true, "rows": [ { "strategy": "FULL_SCENE", "contextEntityCount": 7, "contextRelationCount": 8, "serializedBytes": 3161, "serializedTokens": 848, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 1, "requiredRelationRecall": 1, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 2, "irrelevantRelationCount": 5, "irrelevantEntityRate": 0.3333333333333333, "irrelevantRelationRate": 0.625, "solverStatus": "UNKNOWN", "solverStatusMatchesFullScene": true, "solverResultMatchesFullScene": true, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.224167, "p50": 0.235792, "p95": 0.377416, "max": 0.48775, "mean": 0.256413 } }, { "strategy": "LEXICAL_TOP_K", "contextEntityCount": 4, "contextRelationCount": 4, "serializedBytes": 1791, "serializedTokens": 479, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 0.75, "requiredRelationRecall": 0.3333333333333333, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 0, "irrelevantRelationCount": 3, "irrelevantEntityRate": 0, "irrelevantRelationRate": 0.75, "solverStatus": "UNKNOWN", "solverStatusMatchesFullScene": true, "solverResultMatchesFullScene": false, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.240791, "p50": 0.250709, "p95": 0.312042, "max": 0.396625, "mean": 0.262695 } }, { "strategy": "GRAPH_NEIGHBORHOOD", "contextEntityCount": 5, "contextRelationCount": 7, "serializedBytes": 2497, "serializedTokens": 665, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 1, "requiredRelationRecall": 1, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 0, "irrelevantRelationCount": 4, "irrelevantEntityRate": 0, "irrelevantRelationRate": 0.5714285714285714, "solverStatus": "UNKNOWN", "solverStatusMatchesFullScene": true, "solverResultMatchesFullScene": true, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.275167, "p50": 0.292667, "p95": 0.431541, "max": 0.459708, "mean": 0.304557 } }, { "strategy": "SFT_OPERATIONAL", "contextEntityCount": 5, "contextRelationCount": 7, "serializedBytes": 2484, "serializedTokens": 663, "tokenizer": "o200k_base@gpt-tokenizer-4.0.0", "requiredEntityRecall": 1, "requiredRelationRecall": 1, "requiredMetricFactRecall": 1, "irrelevantEntityCount": 0, "irrelevantRelationCount": 4, "irrelevantEntityRate": 0, "irrelevantRelationRate": 0.5714285714285714, "solverStatus": "UNKNOWN", "solverStatusMatchesFullScene": true, "solverResultMatchesFullScene": true, "falseAuthoritativeExecution": false, "sourceRevisionComplete": true, "constructionLatencyMs": { "samples": 25, "min": 0.274375, "p50": 0.297792, "p95": 0.378292, "max": 0.475292, "mean": 0.30539 } } ] } ], "aggregates": [ { "strategy": "FULL_SCENE", "caseCount": 7, "meanRequiredEntityRecall": 1, "meanRequiredRelationRecall": 1, "meanRequiredMetricFactRecall": 1, "meanIrrelevantEntityCount": 4.428571, "meanIrrelevantRelationCount": 10.714286, "meanSerializedBytes": 5782.428571, "meanSerializedTokens": 1584.285714, "medianSerializedBytesRatioToFull": 1, "meanConstructionLatencyMs": 0.420605, "solverStatusMatchRate": 1, "solverResultParityRate": 1, "falseAuthoritativeExecutionRate": 0, "falseUnknownOrUnsupportedRate": 0, "sourceRevisionCompletenessRate": 1 }, { "strategy": "LEXICAL_TOP_K", "caseCount": 7, "meanRequiredEntityRecall": 0.719728, "meanRequiredRelationRecall": 0.547619, "meanRequiredMetricFactRecall": 0.875, "meanIrrelevantEntityCount": 1.571429, "meanIrrelevantRelationCount": 5.142857, "meanSerializedBytes": 3070.142857, "meanSerializedTokens": 841.857143, "medianSerializedBytesRatioToFull": 0.543837, "meanConstructionLatencyMs": 0.401242, "solverStatusMatchRate": 0.285714, "solverResultParityRate": 0.142857, "falseAuthoritativeExecutionRate": 0, "falseUnknownOrUnsupportedRate": 0.714286, "sourceRevisionCompletenessRate": 1 }, { "strategy": "GRAPH_NEIGHBORHOOD", "caseCount": 7, "meanRequiredEntityRecall": 1, "meanRequiredRelationRecall": 1, "meanRequiredMetricFactRecall": 1, "meanIrrelevantEntityCount": 2.857143, "meanIrrelevantRelationCount": 9, "meanSerializedBytes": 4959.571429, "meanSerializedTokens": 1361.285714, "medianSerializedBytesRatioToFull": 0.839644, "meanConstructionLatencyMs": 0.523345, "solverStatusMatchRate": 1, "solverResultParityRate": 1, "falseAuthoritativeExecutionRate": 0, "falseUnknownOrUnsupportedRate": 0, "sourceRevisionCompletenessRate": 1 }, { "strategy": "SFT_OPERATIONAL", "caseCount": 7, "meanRequiredEntityRecall": 1, "meanRequiredRelationRecall": 1, "meanRequiredMetricFactRecall": 1, "meanIrrelevantEntityCount": 1.857143, "meanIrrelevantRelationCount": 6.714286, "meanSerializedBytes": 4170.714286, "meanSerializedTokens": 1147, "medianSerializedBytesRatioToFull": 0.785827, "meanConstructionLatencyMs": 0.506936, "solverStatusMatchRate": 1, "solverResultParityRate": 1, "falseAuthoritativeExecutionRate": 0, "falseUnknownOrUnsupportedRate": 0, "sourceRevisionCompletenessRate": 1 } ], "gates": { "hardSafety": { "sftRequiredEntityRecallIsComplete": true, "sftRequiredRelationRecallIsComplete": true, "sftRequiredMetricFactRecallIsComplete": true, "noFalseAuthoritativeExecution": true, "missingEvidenceControlsRefused": true, "sftSolverParityIsComplete": true, "sourceRevisionTraceabilityIsComplete": true }, "efficiency": { "targetMedianBytesRatioToFull": 0.6, "observedMedianBytesRatioToFull": 0.785827, "passed": false }, "modelComparisonEligible": false }, "resultState": "VERIFIED_MECHANISM", "conclusion": "FAILED", "modelExperimentDisposition": "NOT_RUN_EFFICIENCY_GATE_FAILED", "measuredMetrics": true, "performanceResult": false, "limitations": [ "The benchmark compares deterministic context construction on seven authored fixture cases, including one missing-evidence control; it is not a frontier-model answer-quality benchmark.", "Recall and irrelevant-context metrics use an independently authored entity, relation, and metric-fact oracle per case.", "LEXICAL_TOP_K is a transparent lexical baseline, not an embedding database or production Vector RAG implementation.", "GRAPH_NEIGHBORHOOD is a one-hop structural baseline, not Microsoft GraphRAG or another complete graph-retrieval system.", "Latency rows measure this local Node.js process and are descriptive, not a production performance claim.", "The predeclared median serialized-byte target was not met, so the frozen pinned-model holdout was not executed." ], "deterministicMembershipDigest": "sha256:35b64669d8f733ff700aa844a8701332de1492a28b46bc8727561417edb5713b", "id": "sha256:f81e74a1f17de90665b17015f457a215d2e20cbc62f80e3e696df2bf3ab43373" }