{
  "cases": 64,
  "tp": 0,
  "fp": 0,
  "fn": 36,
  "alert_precision": null,
  "conflict_recall": 0.0,
  "abstention": 0.875,
  "review_load": 0.875,
  "selective_accuracy": 1,
  "bootstrap_95": {
    "alert_precision": null,
    "conflict_recall": [
      0.0,
      0.0
    ],
    "selective_accuracy": [
      1,
      1
    ],
    "unit": "whole heldout topic families, including language/adversarial variants"
  },
  "languages": {
    "en": {
      "cases": 32,
      "tp": 0,
      "fp": 0,
      "fn": 18,
      "alert_precision": null,
      "conflict_recall": 0.0,
      "abstention": 0.875,
      "review_load": 0.875,
      "selective_accuracy": 1,
      "bootstrap_95": {
        "alert_precision": null,
        "conflict_recall": [
          0.0,
          0.0
        ],
        "selective_accuracy": [
          1,
          1
        ],
        "unit": "whole heldout topic families, including language/adversarial variants"
      }
    },
    "pt": {
      "cases": 32,
      "tp": 0,
      "fp": 0,
      "fn": 18,
      "alert_precision": null,
      "conflict_recall": 0.0,
      "abstention": 0.875,
      "review_load": 0.875,
      "selective_accuracy": 1,
      "bootstrap_95": {
        "alert_precision": null,
        "conflict_recall": [
          0.0,
          0.0
        ],
        "selective_accuracy": [
          1,
          1
        ],
        "unit": "whole heldout topic families, including language/adversarial variants"
      }
    }
  },
  "candidate_conflict_recall": 1.0,
  "threshold_selected_on_train": 0.5,
  "p95_cpu_ms": 0.004667090252041817,
  "rss_max_mib": 25.59375,
  "gate_passed": false,
  "model_load_cpu_ms": 0.0007499475032091141,
  "model_only_p95_cpu_ms": null,
  "raw_score_reliability": [],
  "expected_calibration_error": null,
  "calibration_claim": "unknown; empirical synthetic reliability only, no fitted confidence exported",
  "schema": "synapse.decision-benchmark/1",
  "provider": "rules",
  "revision": "exact-text-baseline/1",
  "corpus_sha256": "91731e40234a85b2df545eb69916d01cfdfe2c084f643dde7535861587b5dcb5",
  "confidence_status": "uncalibrated_softmax_or_heuristic; not probability of source truth",
  "model_artifacts": null,
  "general_semantic_proof": false,
  "default_enabled": false,
  "raw": [
    {
      "id": "tokens/en/plain",
      "split": "train",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.004667090252041817
    },
    {
      "id": "tokens/en/adversarial",
      "split": "train",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0014170072972774506
    },
    {
      "id": "tokens/pt/plain",
      "split": "train",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0019168946892023087
    },
    {
      "id": "tokens/pt/adversarial",
      "split": "train",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0022498425096273422
    },
    {
      "id": "tls/en/plain",
      "split": "train",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.000541098415851593
    },
    {
      "id": "tls/en/adversarial",
      "split": "train",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0004998873919248581
    },
    {
      "id": "tls/pt/plain",
      "split": "train",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.000833999365568161
    },
    {
      "id": "tls/pt/adversarial",
      "split": "train",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0010409858077764511
    },
    {
      "id": "database/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0004169996827840805
    },
    {
      "id": "database/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0005420297384262085
    },
    {
      "id": "database/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.000666128471493721
    },
    {
      "id": "database/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00100000761449337
    },
    {
      "id": "backend-access/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.000416068360209465
    },
    {
      "id": "backend-access/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00045797787606716156
    },
    {
      "id": "backend-access/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0007092021405696869
    },
    {
      "id": "backend-access/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.001708976924419403
    },
    {
      "id": "writes/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00037509016692638397
    },
    {
      "id": "writes/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0004998873919248581
    },
    {
      "id": "writes/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0007080379873514175
    },
    {
      "id": "writes/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0009580980986356735
    },
    {
      "id": "approval/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00041583552956581116
    },
    {
      "id": "approval/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00045821070671081543
    },
    {
      "id": "approval/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.000666128471493721
    },
    {
      "id": "approval/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0010409858077764511
    },
    {
      "id": "environment/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0003748573362827301
    },
    {
      "id": "environment/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00045797787606716156
    },
    {
      "id": "environment/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0008752103894948959
    },
    {
      "id": "environment/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0011248048394918442
    },
    {
      "id": "network/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0003748573362827301
    },
    {
      "id": "network/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0004169996827840805
    },
    {
      "id": "network/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0006658956408500671
    },
    {
      "id": "network/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.000916188582777977
    },
    {
      "id": "retention/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0004169996827840805
    },
    {
      "id": "retention/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00041583552956581116
    },
    {
      "id": "retention/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0007499475032091141
    },
    {
      "id": "retention/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0010419171303510666
    },
    {
      "id": "account/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00033387914299964905
    },
    {
      "id": "account/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00037509016692638397
    },
    {
      "id": "account/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00045797787606716156
    },
    {
      "id": "account/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0010828953236341476
    },
    {
      "id": "encryption-exception/en/plain",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00037509016692638397
    },
    {
      "id": "encryption-exception/en/adversarial",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00045797787606716156
    },
    {
      "id": "encryption-exception/pt/plain",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0009171199053525925
    },
    {
      "id": "encryption-exception/pt/adversarial",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00100000761449337
    },
    {
      "id": "approval-exception/en/plain",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00037509016692638397
    },
    {
      "id": "approval-exception/en/adversarial",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0006670597940683365
    },
    {
      "id": "approval-exception/pt/plain",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0007078051567077637
    },
    {
      "id": "approval-exception/pt/adversarial",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0010828953236341476
    },
    {
      "id": "identity/en/plain",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.000416068360209465
    },
    {
      "id": "identity/en/adversarial",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00045797787606716156
    },
    {
      "id": "identity/pt/plain",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0007080379873514175
    },
    {
      "id": "identity/pt/adversarial",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0010409858077764511
    },
    {
      "id": "key-storage/en/plain",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0004169996827840805
    },
    {
      "id": "key-storage/en/adversarial",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00045797787606716156
    },
    {
      "id": "key-storage/pt/plain",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0007080379873514175
    },
    {
      "id": "key-storage/pt/adversarial",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00100000761449337
    },
    {
      "id": "scope-topic/en/plain",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 0.00020791776478290558
    },
    {
      "id": "scope-topic/en/adversarial",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 0.000125030055642128
    },
    {
      "id": "scope-topic/pt/plain",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 8.288770914077759e-05
    },
    {
      "id": "scope-topic/pt/adversarial",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 8.288770914077759e-05
    },
    {
      "id": "exception-optional/en/plain",
      "split": "heldout",
      "label": "independent",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00045821070671081543
    },
    {
      "id": "exception-optional/en/adversarial",
      "split": "heldout",
      "label": "independent",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00062515027821064
    },
    {
      "id": "exception-optional/pt/plain",
      "split": "heldout",
      "label": "independent",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0007080379873514175
    },
    {
      "id": "exception-optional/pt/adversarial",
      "split": "heldout",
      "label": "independent",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0010409858077764511
    },
    {
      "id": "option-renamed/en/plain",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.000416068360209465
    },
    {
      "id": "option-renamed/en/adversarial",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0004169996827840805
    },
    {
      "id": "option-renamed/pt/plain",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00045797787606716156
    },
    {
      "id": "option-renamed/pt/adversarial",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0009578652679920197
    },
    {
      "id": "unknown-pronoun/en/plain",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "entity_unknown",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.000125030055642128
    },
    {
      "id": "unknown-pronoun/en/adversarial",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "entity_unknown",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 8.312053978443146e-05
    },
    {
      "id": "unknown-pronoun/pt/plain",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "entity_unknown",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 8.288770914077759e-05
    },
    {
      "id": "unknown-pronoun/pt/adversarial",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "entity_unknown",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 8.312053978443146e-05
    },
    {
      "id": "different-environment/en/plain",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 4.21423465013504e-05
    },
    {
      "id": "different-environment/en/adversarial",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 8.312053978443146e-05
    },
    {
      "id": "different-environment/pt/plain",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 8.288770914077759e-05
    },
    {
      "id": "different-environment/pt/adversarial",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 8.312053978443146e-05
    },
    {
      "id": "different-version/en/plain",
      "split": "heldout",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 0.00020908191800117493
    },
    {
      "id": "different-version/en/adversarial",
      "split": "heldout",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 8.288770914077759e-05
    },
    {
      "id": "different-version/pt/plain",
      "split": "heldout",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 8.405186235904694e-05
    },
    {
      "id": "different-version/pt/adversarial",
      "split": "heldout",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 4.21423465013504e-05
    },
    {
      "id": "retired-rule/en/plain",
      "split": "heldout",
      "label": "temporal_change",
      "prediction": "temporal_change",
      "status": "disjoint_validity",
      "score": {
        "relation": "temporal_change",
        "probability": null
      },
      "elapsed_ms": 8.405186235904694e-05
    },
    {
      "id": "retired-rule/en/adversarial",
      "split": "heldout",
      "label": "temporal_change",
      "prediction": "temporal_change",
      "status": "disjoint_validity",
      "score": {
        "relation": "temporal_change",
        "probability": null
      },
      "elapsed_ms": 4.190951585769653e-05
    },
    {
      "id": "retired-rule/pt/plain",
      "split": "heldout",
      "label": "temporal_change",
      "prediction": "temporal_change",
      "status": "disjoint_validity",
      "score": {
        "relation": "temporal_change",
        "probability": null
      },
      "elapsed_ms": 8.288770914077759e-05
    },
    {
      "id": "retired-rule/pt/adversarial",
      "split": "heldout",
      "label": "temporal_change",
      "prediction": "temporal_change",
      "status": "disjoint_validity",
      "score": {
        "relation": "temporal_change",
        "probability": null
      },
      "elapsed_ms": 8.288770914077759e-05
    },
    {
      "id": "scope-private/en/plain",
      "split": "heldout",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "scope_denied",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 8.288770914077759e-05
    },
    {
      "id": "scope-private/en/adversarial",
      "split": "heldout",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "scope_denied",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.000125030055642128
    },
    {
      "id": "scope-private/pt/plain",
      "split": "heldout",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "scope_denied",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 8.288770914077759e-05
    },
    {
      "id": "scope-private/pt/adversarial",
      "split": "heldout",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "scope_denied",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00012479722499847412
    },
    {
      "id": "long-truncation/en/plain",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "entails",
      "status": "identical_supplied_text",
      "score": {
        "relation": "entails",
        "probability": null
      },
      "elapsed_ms": 0.008875038474798203
    },
    {
      "id": "long-truncation/en/adversarial",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "entails",
      "status": "identical_supplied_text",
      "score": {
        "relation": "entails",
        "probability": null
      },
      "elapsed_ms": 0.012292061001062393
    },
    {
      "id": "long-truncation/pt/plain",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "entails",
      "status": "identical_supplied_text",
      "score": {
        "relation": "entails",
        "probability": null
      },
      "elapsed_ms": 0.03300001844763756
    },
    {
      "id": "long-truncation/pt/adversarial",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "entails",
      "status": "identical_supplied_text",
      "score": {
        "relation": "entails",
        "probability": null
      },
      "elapsed_ms": 0.06095808930695057
    },
    {
      "id": "distractor/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00045797787606716156
    },
    {
      "id": "distractor/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0005420297384262085
    },
    {
      "id": "distractor/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0009580980986356735
    },
    {
      "id": "distractor/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "rules_abstained_from_general_prose",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0011248048394918442
    }
  ]
}