{
  "cases": 64,
  "tp": 0,
  "fp": 0,
  "fn": 36,
  "alert_precision": null,
  "conflict_recall": 0.0,
  "abstention": 0.875,
  "review_load": 0.875,
  "selective_accuracy": 1,
  "bootstrap_95": {
    "alert_precision": null,
    "conflict_recall": [
      0.0,
      0.0
    ],
    "selective_accuracy": [
      1,
      1
    ],
    "unit": "whole heldout topic families, including language/adversarial variants"
  },
  "languages": {
    "en": {
      "cases": 32,
      "tp": 0,
      "fp": 0,
      "fn": 18,
      "alert_precision": null,
      "conflict_recall": 0.0,
      "abstention": 0.875,
      "review_load": 0.875,
      "selective_accuracy": 1,
      "bootstrap_95": {
        "alert_precision": null,
        "conflict_recall": [
          0.0,
          0.0
        ],
        "selective_accuracy": [
          1,
          1
        ],
        "unit": "whole heldout topic families, including language/adversarial variants"
      }
    },
    "pt": {
      "cases": 32,
      "tp": 0,
      "fp": 0,
      "fn": 18,
      "alert_precision": null,
      "conflict_recall": 0.0,
      "abstention": 0.875,
      "review_load": 0.875,
      "selective_accuracy": 1,
      "bootstrap_95": {
        "alert_precision": null,
        "conflict_recall": [
          0.0,
          0.0
        ],
        "selective_accuracy": [
          1,
          1
        ],
        "unit": "whole heldout topic families, including language/adversarial variants"
      }
    }
  },
  "candidate_conflict_recall": 1.0,
  "threshold_selected_on_train": 1.01,
  "p95_cpu_ms": 45.7567498087883,
  "rss_max_mib": 931.859375,
  "gate_passed": false,
  "model_load_cpu_ms": 4183.927458943799,
  "model_only_p95_cpu_ms": 47.785083996132016,
  "raw_score_reliability": [
    {
      "range": [
        0.4,
        0.6000000000000001
      ],
      "n": 1,
      "mean_raw_score": 0.5438234210014343,
      "accuracy": 0
    },
    {
      "range": [
        0.6,
        0.8
      ],
      "n": 1,
      "mean_raw_score": 0.6875956058502197,
      "accuracy": 1
    },
    {
      "range": [
        0.8,
        1.0
      ],
      "n": 50,
      "mean_raw_score": 0.9811117124557495,
      "accuracy": 0.78
    }
  ],
  "expected_calibration_error": 0.20984256611420551,
  "calibration_claim": "unknown; empirical synthetic reliability only, no fitted confidence exported",
  "schema": "synapse.decision-benchmark/1",
  "provider": "english",
  "revision": "fa2804872c3b4bd748f38c0185cc85775361e735",
  "corpus_sha256": "91731e40234a85b2df545eb69916d01cfdfe2c084f643dde7535861587b5dcb5",
  "confidence_status": "uncalibrated_softmax_or_heuristic; not probability of source truth",
  "model_artifacts": {
    "schema": "synapse.local-decision-model/1",
    "repository": "cross-encoder/nli-deberta-v3-small",
    "revision": "fa2804872c3b4bd748f38c0185cc85775361e735",
    "license": "apache-2.0",
    "files": {
      "README.md": {
        "sha256": "a9aa108025ad1984374c5e80365406bafe331aea5abc30239e1796ec0a7c267d",
        "bytes": 2864
      },
      "config.json": {
        "sha256": "885d0dceae8fa5c136da9209121ec9eb11160488e840de3bc1f29353674e5712",
        "bytes": 1052
      },
      "model.safetensors": {
        "sha256": "ebc79588dd73ccfb6a3f6078519cfbf512c5305384c5ea1845bc71cd32216e86",
        "bytes": 567605820
      },
      "special_tokens_map.json": {
        "sha256": "ed7c099c988dbb414b18a6980d20cb57b91b7cd119f6f6941eb364b0e892e712",
        "bytes": 301
      },
      "spm.model": {
        "sha256": "c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd",
        "bytes": 2464616
      },
      "tokenizer.json": {
        "sha256": "5124ef2ead1a10a717703bc436de7f353da76d6340e4587719b42b1693707964",
        "bytes": 8656624
      },
      "tokenizer_config.json": {
        "sha256": "f3eecd07c370ef0bf7dd3780d3cd68cf9c8b00c267e21a208ddcd8f82bfec1a6",
        "bytes": 1346
      },
      "onnx/model_qint8_arm64.onnx": {
        "sha256": "a1da6815c8da542d16381c7d84892d409b4b6604d655bb3054ecb15dd8714e6e",
        "bytes": 172503006
      }
    }
  },
  "general_semantic_proof": false,
  "default_enabled": false,
  "raw": [
    {
      "id": "tokens/en/plain",
      "split": "train",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9993688464164734,
        "probabilities": {
          "contradiction": 0.9993688464164734,
          "entails": 0.00035717865102924407,
          "independent": 0.00027399513055570424
        }
      },
      "elapsed_ms": 97.22833288833499
    },
    {
      "id": "tokens/en/adversarial",
      "split": "train",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9981992840766907,
        "probabilities": {
          "contradiction": 0.9981992840766907,
          "entails": 0.0011908903252333403,
          "independent": 0.0006098951562307775
        }
      },
      "elapsed_ms": 34.18979188427329
    },
    {
      "id": "tokens/pt/plain",
      "split": "train",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9984228610992432,
        "probabilities": {
          "contradiction": 0.9984228610992432,
          "entails": 0.001184130902402103,
          "independent": 0.0003930196398869157
        }
      },
      "elapsed_ms": 31.303833005949855
    },
    {
      "id": "tokens/pt/adversarial",
      "split": "train",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9974664449691772,
        "probabilities": {
          "contradiction": 0.9974664449691772,
          "entails": 0.0021192573476582766,
          "independent": 0.0004143265832681209
        }
      },
      "elapsed_ms": 36.07774991542101
    },
    {
      "id": "tls/en/plain",
      "split": "train",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9890823364257812,
        "probabilities": {
          "contradiction": 0.9890823364257812,
          "entails": 0.0017821511719375849,
          "independent": 0.009135532192885876
        }
      },
      "elapsed_ms": 29.730582842603326
    },
    {
      "id": "tls/en/adversarial",
      "split": "train",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9920952916145325,
        "probabilities": {
          "contradiction": 0.9920952916145325,
          "entails": 0.0010848499368876219,
          "independent": 0.006819861475378275
        }
      },
      "elapsed_ms": 29.738500015810132
    },
    {
      "id": "tls/pt/plain",
      "split": "train",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9983471632003784,
        "probabilities": {
          "contradiction": 0.9983471632003784,
          "entails": 0.00019425393838901073,
          "independent": 0.0014586520846933126
        }
      },
      "elapsed_ms": 30.44966608285904
    },
    {
      "id": "tls/pt/adversarial",
      "split": "train",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9937441945075989,
        "probabilities": {
          "contradiction": 0.9937441945075989,
          "entails": 0.00031849401420913637,
          "independent": 0.005937330424785614
        }
      },
      "elapsed_ms": 36.89558291807771
    },
    {
      "id": "database/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9995094537734985,
        "probabilities": {
          "contradiction": 0.9995094537734985,
          "entails": 3.421550354687497e-05,
          "independent": 0.0004563533584587276
        }
      },
      "elapsed_ms": 24.24858300946653
    },
    {
      "id": "database/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9991996884346008,
        "probabilities": {
          "contradiction": 0.9991996884346008,
          "entails": 4.721797813544981e-05,
          "independent": 0.0007531464798375964
        }
      },
      "elapsed_ms": 34.36116687953472
    },
    {
      "id": "database/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9989495873451233,
        "probabilities": {
          "contradiction": 0.9989495873451233,
          "entails": 5.6893190048867837e-05,
          "independent": 0.0009935367852449417
        }
      },
      "elapsed_ms": 30.93345882371068
    },
    {
      "id": "database/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9982064962387085,
        "probabilities": {
          "contradiction": 0.9982064962387085,
          "entails": 8.476645598420873e-05,
          "independent": 0.0017088173190131783
        }
      },
      "elapsed_ms": 47.785083996132016
    },
    {
      "id": "backend-access/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "independent",
        "probability": 0.5438234210014343,
        "probabilities": {
          "contradiction": 0.4356834888458252,
          "entails": 0.020493026822805405,
          "independent": 0.5438234210014343
        }
      },
      "elapsed_ms": 32.61458408087492
    },
    {
      "id": "backend-access/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.8215413093566895,
        "probabilities": {
          "contradiction": 0.8215413093566895,
          "entails": 0.0073280977085232735,
          "independent": 0.17113058269023895
        }
      },
      "elapsed_ms": 36.78866685368121
    },
    {
      "id": "backend-access/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9736770987510681,
        "probabilities": {
          "contradiction": 0.9736770987510681,
          "entails": 0.0019635511562228203,
          "independent": 0.02435937337577343
        }
      },
      "elapsed_ms": 51.70245794579387
    },
    {
      "id": "backend-access/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9192724823951721,
        "probabilities": {
          "contradiction": 0.9192724823951721,
          "entails": 0.012021178379654884,
          "independent": 0.06870636343955994
        }
      },
      "elapsed_ms": 50.86487508378923
    },
    {
      "id": "writes/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9705396294593811,
        "probabilities": {
          "contradiction": 0.9705396294593811,
          "entails": 0.00015681851073168218,
          "independent": 0.02930355817079544
        }
      },
      "elapsed_ms": 32.94937498867512
    },
    {
      "id": "writes/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9848849773406982,
        "probabilities": {
          "contradiction": 0.9848849773406982,
          "entails": 0.0001552325556986034,
          "independent": 0.014959780499339104
        }
      },
      "elapsed_ms": 34.811208955943584
    },
    {
      "id": "writes/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "independent",
        "probability": 0.9578053951263428,
        "probabilities": {
          "contradiction": 0.023496005684137344,
          "entails": 0.01869860664010048,
          "independent": 0.9578053951263428
        }
      },
      "elapsed_ms": 34.1839580796659
    },
    {
      "id": "writes/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "independent",
        "probability": 0.9297319650650024,
        "probabilities": {
          "contradiction": 0.05089142173528671,
          "entails": 0.019376583397388458,
          "independent": 0.9297319650650024
        }
      },
      "elapsed_ms": 45.7567498087883
    },
    {
      "id": "approval/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.99882572889328,
        "probabilities": {
          "contradiction": 0.99882572889328,
          "entails": 0.00042223441414535046,
          "independent": 0.000752015330363065
        }
      },
      "elapsed_ms": 36.632333882153034
    },
    {
      "id": "approval/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9989781379699707,
        "probabilities": {
          "contradiction": 0.9989781379699707,
          "entails": 0.000469132763100788,
          "independent": 0.0005528008914552629
        }
      },
      "elapsed_ms": 40.81712500192225
    },
    {
      "id": "approval/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.999538779258728,
        "probabilities": {
          "contradiction": 0.999538779258728,
          "entails": 0.00021118059521541,
          "independent": 0.00025014180573634803
        }
      },
      "elapsed_ms": 36.05408291332424
    },
    {
      "id": "approval/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9992765784263611,
        "probabilities": {
          "contradiction": 0.9992765784263611,
          "entails": 0.0003213624586351216,
          "independent": 0.0004020932537969202
        }
      },
      "elapsed_ms": 36.66949993930757
    },
    {
      "id": "environment/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "independent",
        "probability": 0.9863250851631165,
        "probabilities": {
          "contradiction": 0.013612557202577591,
          "entails": 6.230642611626536e-05,
          "independent": 0.9863250851631165
        }
      },
      "elapsed_ms": 26.21012507006526
    },
    {
      "id": "environment/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "independent",
        "probability": 0.9765372276306152,
        "probabilities": {
          "contradiction": 0.023311302065849304,
          "entails": 0.00015145195357035846,
          "independent": 0.9765372276306152
        }
      },
      "elapsed_ms": 29.324499890208244
    },
    {
      "id": "environment/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "independent",
        "probability": 0.9992725253105164,
        "probabilities": {
          "contradiction": 0.0005570257781073451,
          "entails": 0.00017045142885763198,
          "independent": 0.9992725253105164
        }
      },
      "elapsed_ms": 22.90983311831951
    },
    {
      "id": "environment/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "independent",
        "probability": 0.9994396567344666,
        "probabilities": {
          "contradiction": 0.0004037284816149622,
          "entails": 0.00015662270016036928,
          "independent": 0.9994396567344666
        }
      },
      "elapsed_ms": 35.19112500362098
    },
    {
      "id": "network/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9813131093978882,
        "probabilities": {
          "contradiction": 0.9813131093978882,
          "entails": 0.004565814975649118,
          "independent": 0.01412112731486559
        }
      },
      "elapsed_ms": 24.964499985799193
    },
    {
      "id": "network/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.996907651424408,
        "probabilities": {
          "contradiction": 0.996907651424408,
          "entails": 0.0013568424619734287,
          "independent": 0.0017355397576466203
        }
      },
      "elapsed_ms": 32.237250125035644
    },
    {
      "id": "network/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9956166744232178,
        "probabilities": {
          "contradiction": 0.9956166744232178,
          "entails": 0.0018010492203757167,
          "independent": 0.0025823048781603575
        }
      },
      "elapsed_ms": 28.432375052943826
    },
    {
      "id": "network/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.8982254266738892,
        "probabilities": {
          "contradiction": 0.8982254266738892,
          "entails": 0.05032971501350403,
          "independent": 0.051444802433252335
        }
      },
      "elapsed_ms": 37.07012510858476
    },
    {
      "id": "retention/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9863944053649902,
        "probabilities": {
          "contradiction": 0.9863944053649902,
          "entails": 7.321094017243013e-05,
          "independent": 0.01353232841938734
        }
      },
      "elapsed_ms": 26.19966701604426
    },
    {
      "id": "retention/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9685056209564209,
        "probabilities": {
          "contradiction": 0.9685056209564209,
          "entails": 0.0001433992583770305,
          "independent": 0.03135092556476593
        }
      },
      "elapsed_ms": 29.95108417235315
    },
    {
      "id": "retention/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9984424710273743,
        "probabilities": {
          "contradiction": 0.9984424710273743,
          "entails": 3.565495353541337e-05,
          "independent": 0.0015218150801956654
        }
      },
      "elapsed_ms": 28.296458069235086
    },
    {
      "id": "retention/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.994860827922821,
        "probabilities": {
          "contradiction": 0.994860827922821,
          "entails": 9.358016541227698e-05,
          "independent": 0.005045588128268719
        }
      },
      "elapsed_ms": 40.826624957844615
    },
    {
      "id": "account/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9996609687805176,
        "probabilities": {
          "contradiction": 0.9996609687805176,
          "entails": 6.947460497030988e-05,
          "independent": 0.000269538548309356
        }
      },
      "elapsed_ms": 26.147957891225815
    },
    {
      "id": "account/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9994668364524841,
        "probabilities": {
          "contradiction": 0.9994668364524841,
          "entails": 0.00017387958359904587,
          "independent": 0.00035936443600803614
        }
      },
      "elapsed_ms": 28.937499970197678
    },
    {
      "id": "account/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9995877146720886,
        "probabilities": {
          "contradiction": 0.9995877146720886,
          "entails": 0.00019027777307201177,
          "independent": 0.00022202255786396563
        }
      },
      "elapsed_ms": 28.281084029003978
    },
    {
      "id": "account/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9992582201957703,
        "probabilities": {
          "contradiction": 0.9992582201957703,
          "entails": 0.00015350601461250335,
          "independent": 0.0005882007535547018
        }
      },
      "elapsed_ms": 41.71824990771711
    },
    {
      "id": "encryption-exception/en/plain",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9949387311935425,
        "probabilities": {
          "contradiction": 0.00024285909603349864,
          "entails": 0.9949387311935425,
          "independent": 0.004818397108465433
        }
      },
      "elapsed_ms": 23.50033400580287
    },
    {
      "id": "encryption-exception/en/adversarial",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9959207773208618,
        "probabilities": {
          "contradiction": 0.0004493296437431127,
          "entails": 0.9959207773208618,
          "independent": 0.0036298297345638275
        }
      },
      "elapsed_ms": 28.114625019952655
    },
    {
      "id": "encryption-exception/pt/plain",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9934949278831482,
        "probabilities": {
          "contradiction": 0.000528222881257534,
          "entails": 0.9934949278831482,
          "independent": 0.005976890679448843
        }
      },
      "elapsed_ms": 33.325832802802324
    },
    {
      "id": "encryption-exception/pt/adversarial",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9941773414611816,
        "probabilities": {
          "contradiction": 0.0007040217751637101,
          "entails": 0.9941773414611816,
          "independent": 0.0051187206991016865
        }
      },
      "elapsed_ms": 40.75966612435877
    },
    {
      "id": "approval-exception/en/plain",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9196836352348328,
        "probabilities": {
          "contradiction": 0.9196836352348328,
          "entails": 0.03130095824599266,
          "independent": 0.04901544377207756
        }
      },
      "elapsed_ms": 24.76329216733575
    },
    {
      "id": "approval-exception/en/adversarial",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.8589076995849609,
        "probabilities": {
          "contradiction": 0.8589076995849609,
          "entails": 0.061403077095746994,
          "independent": 0.07968921214342117
        }
      },
      "elapsed_ms": 29.834916116669774
    },
    {
      "id": "approval-exception/pt/plain",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9816022515296936,
        "probabilities": {
          "contradiction": 0.0015466801123693585,
          "entails": 0.9816022515296936,
          "independent": 0.016851035878062248
        }
      },
      "elapsed_ms": 33.00124988891184
    },
    {
      "id": "approval-exception/pt/adversarial",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.6606845259666443,
        "probabilities": {
          "contradiction": 0.010877886787056923,
          "entails": 0.6606845259666443,
          "independent": 0.32843756675720215
        }
      },
      "elapsed_ms": 39.24287483096123
    },
    {
      "id": "identity/en/plain",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.832851767539978,
        "probabilities": {
          "contradiction": 0.832851767539978,
          "entails": 0.04046060889959335,
          "independent": 0.12668761610984802
        }
      },
      "elapsed_ms": 22.091541904956102
    },
    {
      "id": "identity/en/adversarial",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9237014651298523,
        "probabilities": {
          "contradiction": 0.9237014651298523,
          "entails": 0.019119910895824432,
          "independent": 0.05717857554554939
        }
      },
      "elapsed_ms": 28.90283288434148
    },
    {
      "id": "identity/pt/plain",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9911192059516907,
        "probabilities": {
          "contradiction": 0.9911192059516907,
          "entails": 0.005538369063287973,
          "independent": 0.003342337440699339
        }
      },
      "elapsed_ms": 23.755209054797888
    },
    {
      "id": "identity/pt/adversarial",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.6997140645980835,
        "probabilities": {
          "contradiction": 0.27932533621788025,
          "entails": 0.6997140645980835,
          "independent": 0.020960593596100807
        }
      },
      "elapsed_ms": 32.37124998122454
    },
    {
      "id": "key-storage/en/plain",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9958523511886597,
        "probabilities": {
          "contradiction": 0.000271933909971267,
          "entails": 0.9958523511886597,
          "independent": 0.0038756916765123606
        }
      },
      "elapsed_ms": 21.490709157660604
    },
    {
      "id": "key-storage/en/adversarial",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9967514276504517,
        "probabilities": {
          "contradiction": 0.00045202436740510166,
          "entails": 0.9967514276504517,
          "independent": 0.002796576824039221
        }
      },
      "elapsed_ms": 29.791584005579352
    },
    {
      "id": "key-storage/pt/plain",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9951701760292053,
        "probabilities": {
          "contradiction": 0.0009259720100089908,
          "entails": 0.9951701760292053,
          "independent": 0.0039039235562086105
        }
      },
      "elapsed_ms": 25.719957891851664
    },
    {
      "id": "key-storage/pt/adversarial",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9940996170043945,
        "probabilities": {
          "contradiction": 0.0017866228008642793,
          "entails": 0.9940996170043945,
          "independent": 0.004113814327865839
        }
      },
      "elapsed_ms": 33.343249931931496
    },
    {
      "id": "scope-topic/en/plain",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 0.000250060111284256
    },
    {
      "id": "scope-topic/en/adversarial",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 0.000125030055642128
    },
    {
      "id": "scope-topic/pt/plain",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 8.405186235904694e-05
    },
    {
      "id": "scope-topic/pt/adversarial",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 4.190951585769653e-05
    },
    {
      "id": "exception-optional/en/plain",
      "split": "heldout",
      "label": "independent",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9885618686676025,
        "probabilities": {
          "contradiction": 0.0010019660694524646,
          "entails": 0.9885618686676025,
          "independent": 0.010436177253723145
        }
      },
      "elapsed_ms": 21.434209076687694
    },
    {
      "id": "exception-optional/en/adversarial",
      "split": "heldout",
      "label": "independent",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.989370584487915,
        "probabilities": {
          "contradiction": 0.003909021150320768,
          "entails": 0.989370584487915,
          "independent": 0.006720316130667925
        }
      },
      "elapsed_ms": 27.75833406485617
    },
    {
      "id": "exception-optional/pt/plain",
      "split": "heldout",
      "label": "independent",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9859570264816284,
        "probabilities": {
          "contradiction": 0.0016279177507385612,
          "entails": 0.9859570264816284,
          "independent": 0.012415179051458836
        }
      },
      "elapsed_ms": 23.591042030602694
    },
    {
      "id": "exception-optional/pt/adversarial",
      "split": "heldout",
      "label": "independent",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9932165741920471,
        "probabilities": {
          "contradiction": 0.0016730466159060597,
          "entails": 0.9932165741920471,
          "independent": 0.005110424477607012
        }
      },
      "elapsed_ms": 34.96779198758304
    },
    {
      "id": "option-renamed/en/plain",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9816794395446777,
        "probabilities": {
          "contradiction": 0.0010112124728038907,
          "entails": 0.9816794395446777,
          "independent": 0.01730935461819172
        }
      },
      "elapsed_ms": 23.917707847431302
    },
    {
      "id": "option-renamed/en/adversarial",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9863255620002747,
        "probabilities": {
          "contradiction": 0.002203129231929779,
          "entails": 0.9863255620002747,
          "independent": 0.011471365578472614
        }
      },
      "elapsed_ms": 27.111874893307686
    },
    {
      "id": "option-renamed/pt/plain",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.8773613572120667,
        "probabilities": {
          "contradiction": 0.8773613572120667,
          "entails": 0.09529270231723785,
          "independent": 0.027346037328243256
        }
      },
      "elapsed_ms": 25.15820786356926
    },
    {
      "id": "option-renamed/pt/adversarial",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.6875956058502197,
        "probabilities": {
          "contradiction": 0.29306814074516296,
          "entails": 0.6875956058502197,
          "independent": 0.01933627389371395
        }
      },
      "elapsed_ms": 31.829624902457
    },
    {
      "id": "unknown-pronoun/en/plain",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "entity_unknown",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0006670597940683365
    },
    {
      "id": "unknown-pronoun/en/adversarial",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "entity_unknown",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.000125030055642128
    },
    {
      "id": "unknown-pronoun/pt/plain",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "entity_unknown",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 4.190951585769653e-05
    },
    {
      "id": "unknown-pronoun/pt/adversarial",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "entity_unknown",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 8.312053978443146e-05
    },
    {
      "id": "different-environment/en/plain",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 0.00016600824892520905
    },
    {
      "id": "different-environment/en/adversarial",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 8.288770914077759e-05
    },
    {
      "id": "different-environment/pt/plain",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 8.381903171539307e-05
    },
    {
      "id": "different-environment/pt/adversarial",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 4.190951585769653e-05
    },
    {
      "id": "different-version/en/plain",
      "split": "heldout",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 4.190951585769653e-05
    },
    {
      "id": "different-version/en/adversarial",
      "split": "heldout",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 4.190951585769653e-05
    },
    {
      "id": "different-version/pt/plain",
      "split": "heldout",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 0.000250060111284256
    },
    {
      "id": "different-version/pt/adversarial",
      "split": "heldout",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 4.21423465013504e-05
    },
    {
      "id": "retired-rule/en/plain",
      "split": "heldout",
      "label": "temporal_change",
      "prediction": "temporal_change",
      "status": "disjoint_validity",
      "score": {
        "relation": "temporal_change",
        "probability": null
      },
      "elapsed_ms": 8.405186235904694e-05
    },
    {
      "id": "retired-rule/en/adversarial",
      "split": "heldout",
      "label": "temporal_change",
      "prediction": "temporal_change",
      "status": "disjoint_validity",
      "score": {
        "relation": "temporal_change",
        "probability": null
      },
      "elapsed_ms": 8.312053978443146e-05
    },
    {
      "id": "retired-rule/pt/plain",
      "split": "heldout",
      "label": "temporal_change",
      "prediction": "temporal_change",
      "status": "disjoint_validity",
      "score": {
        "relation": "temporal_change",
        "probability": null
      },
      "elapsed_ms": 8.405186235904694e-05
    },
    {
      "id": "retired-rule/pt/adversarial",
      "split": "heldout",
      "label": "temporal_change",
      "prediction": "temporal_change",
      "status": "disjoint_validity",
      "score": {
        "relation": "temporal_change",
        "probability": null
      },
      "elapsed_ms": 4.0978193283081055e-05
    },
    {
      "id": "scope-private/en/plain",
      "split": "heldout",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "scope_denied",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 4.21423465013504e-05
    },
    {
      "id": "scope-private/en/adversarial",
      "split": "heldout",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "scope_denied",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 8.405186235904694e-05
    },
    {
      "id": "scope-private/pt/plain",
      "split": "heldout",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "scope_denied",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00029103830456733704
    },
    {
      "id": "scope-private/pt/adversarial",
      "split": "heldout",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "scope_denied",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 8.405186235904694e-05
    },
    {
      "id": "long-truncation/en/plain",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "input_window_abstention",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 2.727875020354986
    },
    {
      "id": "long-truncation/en/adversarial",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "input_window_abstention",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 3.984207985922694
    },
    {
      "id": "long-truncation/pt/plain",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "input_window_abstention",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 2.9798748437315226
    },
    {
      "id": "long-truncation/pt/adversarial",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "input_window_abstention",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 5.692540900781751
    },
    {
      "id": "distractor/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9985419511795044,
        "probabilities": {
          "contradiction": 0.9985419511795044,
          "entails": 0.0006110137328505516,
          "independent": 0.0008471084875054657
        }
      },
      "elapsed_ms": 27.516167145222425
    },
    {
      "id": "distractor/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9985957741737366,
        "probabilities": {
          "contradiction": 0.9985957741737366,
          "entails": 0.0005776582402177155,
          "independent": 0.0008265053038485348
        }
      },
      "elapsed_ms": 28.012959053739905
    },
    {
      "id": "distractor/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9936962723731995,
        "probabilities": {
          "contradiction": 0.9936962723731995,
          "entails": 0.0039131538942456245,
          "independent": 0.0023905725684016943
        }
      },
      "elapsed_ms": 29.25541694276035
    },
    {
      "id": "distractor/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9721221327781677,
        "probabilities": {
          "contradiction": 0.9721221327781677,
          "entails": 0.020980628207325935,
          "independent": 0.006897277198731899
        }
      },
      "elapsed_ms": 37.506625056266785
    }
  ]
}