{
  "cases": 64,
  "tp": 9,
  "fp": 0,
  "fn": 27,
  "alert_precision": 1.0,
  "conflict_recall": 0.25,
  "abstention": 0.5,
  "review_load": 0.640625,
  "selective_accuracy": 0.84375,
  "bootstrap_95": {
    "alert_precision": [
      1.0,
      1.0
    ],
    "conflict_recall": [
      0.07142857142857142,
      0.4642857142857143
    ],
    "selective_accuracy": [
      0.6521739130434783,
      0.9722222222222222
    ],
    "unit": "whole heldout topic families, including language/adversarial variants"
  },
  "languages": {
    "en": {
      "cases": 32,
      "tp": 4,
      "fp": 0,
      "fn": 14,
      "alert_precision": 1.0,
      "conflict_recall": 0.2222222222222222,
      "abstention": 0.53125,
      "review_load": 0.65625,
      "selective_accuracy": 0.9333333333333333,
      "bootstrap_95": {
        "alert_precision": [
          1.0,
          1.0
        ],
        "conflict_recall": [
          0.0,
          0.5
        ],
        "selective_accuracy": [
          0.75,
          1
        ],
        "unit": "whole heldout topic families, including language/adversarial variants"
      }
    },
    "pt": {
      "cases": 32,
      "tp": 5,
      "fp": 0,
      "fn": 13,
      "alert_precision": 1.0,
      "conflict_recall": 0.2777777777777778,
      "abstention": 0.46875,
      "review_load": 0.625,
      "selective_accuracy": 0.7647058823529411,
      "bootstrap_95": {
        "alert_precision": [
          1.0,
          1.0
        ],
        "conflict_recall": [
          0.07142857142857142,
          0.5
        ],
        "selective_accuracy": [
          0.5555555555555556,
          0.9444444444444444
        ],
        "unit": "whole heldout topic families, including language/adversarial variants"
      }
    }
  },
  "candidate_conflict_recall": 1.0,
  "threshold_selected_on_train": 0.9,
  "p95_cpu_ms": 10.748208034783602,
  "rss_max_mib": 1268.8125,
  "gate_passed": false,
  "model_load_cpu_ms": 4579.776332946494,
  "model_only_p95_cpu_ms": 10.89891605079174,
  "raw_score_reliability": [
    {
      "range": [
        0.4,
        0.6000000000000001
      ],
      "n": 9,
      "mean_raw_score": 0.5434648593266805,
      "accuracy": 0.4444444444444444
    },
    {
      "range": [
        0.6,
        0.8
      ],
      "n": 11,
      "mean_raw_score": 0.7132604067975824,
      "accuracy": 0.36363636363636365
    },
    {
      "range": [
        0.8,
        1.0
      ],
      "n": 32,
      "mean_raw_score": 0.9254468232393265,
      "accuracy": 0.8125
    }
  ],
  "expected_calibration_error": 0.16060281831484574,
  "calibration_claim": "unknown; empirical synthetic reliability only, no fitted confidence exported",
  "schema": "synapse.decision-benchmark/1",
  "provider": "multilingual",
  "revision": "0a71e92a985b6e1ad1828cf67ce9c459639c1dca",
  "corpus_sha256": "91731e40234a85b2df545eb69916d01cfdfe2c084f643dde7535861587b5dcb5",
  "confidence_status": "uncalibrated_softmax_or_heuristic; not probability of source truth",
  "model_artifacts": {
    "schema": "synapse.local-decision-model/1",
    "repository": "MoritzLaurer/multilingual-MiniLMv2-L6-mnli-xnli",
    "revision": "0a71e92a985b6e1ad1828cf67ce9c459639c1dca",
    "license": "mit",
    "files": {
      "README.md": {
        "sha256": "f28aebb1babf603b5c5c107670645f2b00e09a2e9a39132293ad0a9a7ab6fa34",
        "bytes": 6741
      },
      "config.json": {
        "sha256": "79862295be1538a947e0f56d495ef8d658c3eaebcfb42c7ad96229d195ca745d",
        "bytes": 1032
      },
      "model.safetensors": {
        "sha256": "91b323ccf247ec1e3b5925d566230bae7c52de8147e6062b42e250089a3fc80b",
        "bytes": 427997022
      },
      "sentencepiece.bpe.model": {
        "sha256": "cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865",
        "bytes": 5069051
      },
      "special_tokens_map.json": {
        "sha256": "06e405a36dfe4b9604f484f6a1e619af1a7f7d09e34a8555eb0b77b66318067f",
        "bytes": 280
      },
      "tokenizer.json": {
        "sha256": "098c131bb4423163db239755e309facaa6850059f850f9f3d88a78344a4b631c",
        "bytes": 17082758
      },
      "tokenizer_config.json": {
        "sha256": "86139ce1f39e814bcdb99a6fe30c0c4983911508aad07417e616d994fb74eb25",
        "bytes": 565
      },
      "onnx/model.onnx": {
        "sha256": "79f8cda2b1230585a95ea0514a6f1bd21c5c986ba0529bb3261213a3e195fa6e",
        "bytes": 428127016
      }
    }
  },
  "general_semantic_proof": false,
  "default_enabled": false,
  "raw": [
    {
      "id": "tokens/en/plain",
      "split": "train",
      "label": "contradiction",
      "prediction": "contradiction",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9896280765533447,
        "probabilities": {
          "entails": 0.006018033716827631,
          "independent": 0.004353885538876057,
          "contradiction": 0.9896280765533447
        }
      },
      "elapsed_ms": 9.815749945119023
    },
    {
      "id": "tokens/en/adversarial",
      "split": "train",
      "label": "contradiction",
      "prediction": "contradiction",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9770046472549438,
        "probabilities": {
          "entails": 0.0165863074362278,
          "independent": 0.006409002933651209,
          "contradiction": 0.9770046472549438
        }
      },
      "elapsed_ms": 8.869915967807174
    },
    {
      "id": "tokens/pt/plain",
      "split": "train",
      "label": "contradiction",
      "prediction": "contradiction",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9920143485069275,
        "probabilities": {
          "entails": 0.005306898150593042,
          "independent": 0.002678789896890521,
          "contradiction": 0.9920143485069275
        }
      },
      "elapsed_ms": 7.553542032837868
    },
    {
      "id": "tokens/pt/adversarial",
      "split": "train",
      "label": "contradiction",
      "prediction": "entails",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9805848002433777,
        "probabilities": {
          "entails": 0.9805848002433777,
          "independent": 0.0029803148936480284,
          "contradiction": 0.016434786841273308
        }
      },
      "elapsed_ms": 10.330416029319167
    },
    {
      "id": "tls/en/plain",
      "split": "train",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "independent",
        "probability": 0.7275297045707703,
        "probabilities": {
          "entails": 0.06928135454654694,
          "independent": 0.7275297045707703,
          "contradiction": 0.20318898558616638
        }
      },
      "elapsed_ms": 4.824874922633171
    },
    {
      "id": "tls/en/adversarial",
      "split": "train",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "independent",
        "probability": 0.6689406633377075,
        "probabilities": {
          "entails": 0.09769231826066971,
          "independent": 0.6689406633377075,
          "contradiction": 0.23336710035800934
        }
      },
      "elapsed_ms": 7.706083124503493
    },
    {
      "id": "tls/pt/plain",
      "split": "train",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.5090020895004272,
        "probabilities": {
          "entails": 0.005232653114944696,
          "independent": 0.4857652187347412,
          "contradiction": 0.5090020895004272
        }
      },
      "elapsed_ms": 6.48104096762836
    },
    {
      "id": "tls/pt/adversarial",
      "split": "train",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "independent",
        "probability": 0.86335289478302,
        "probabilities": {
          "entails": 0.010519430972635746,
          "independent": 0.86335289478302,
          "contradiction": 0.12612763047218323
        }
      },
      "elapsed_ms": 10.70683402940631
    },
    {
      "id": "database/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "contradiction",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9797614812850952,
        "probabilities": {
          "entails": 0.00881433766335249,
          "independent": 0.011424243450164795,
          "contradiction": 0.9797614812850952
        }
      },
      "elapsed_ms": 4.472666885703802
    },
    {
      "id": "database/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "contradiction",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9445672035217285,
        "probabilities": {
          "entails": 0.033780504018068314,
          "independent": 0.021652236580848694,
          "contradiction": 0.9445672035217285
        }
      },
      "elapsed_ms": 9.53795900568366
    },
    {
      "id": "database/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "contradiction",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.980459988117218,
        "probabilities": {
          "entails": 0.01187389437109232,
          "independent": 0.007666063029319048,
          "contradiction": 0.980459988117218
        }
      },
      "elapsed_ms": 5.574542097747326
    },
    {
      "id": "database/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.5720141530036926,
        "probabilities": {
          "entails": 0.41063085198402405,
          "independent": 0.0173549372702837,
          "contradiction": 0.5720141530036926
        }
      },
      "elapsed_ms": 10.061166947707534
    },
    {
      "id": "backend-access/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.5599903464317322,
        "probabilities": {
          "entails": 0.06452524662017822,
          "independent": 0.375484436750412,
          "contradiction": 0.5599903464317322
        }
      },
      "elapsed_ms": 5.879000062122941
    },
    {
      "id": "backend-access/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "independent",
        "probability": 0.5378039479255676,
        "probabilities": {
          "entails": 0.0907105803489685,
          "independent": 0.5378039479255676,
          "contradiction": 0.37148550152778625
        }
      },
      "elapsed_ms": 8.359750034287572
    },
    {
      "id": "backend-access/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "independent",
        "probability": 0.7412576079368591,
        "probabilities": {
          "entails": 0.09998197108507156,
          "independent": 0.7412576079368591,
          "contradiction": 0.15876039862632751
        }
      },
      "elapsed_ms": 5.089583108201623
    },
    {
      "id": "backend-access/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "independent",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "independent",
        "probability": 0.9288947582244873,
        "probabilities": {
          "entails": 0.02694389782845974,
          "independent": 0.9288947582244873,
          "contradiction": 0.044161416590213776
        }
      },
      "elapsed_ms": 10.639542015269399
    },
    {
      "id": "writes/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.614800751209259,
        "probabilities": {
          "entails": 0.026924770325422287,
          "independent": 0.3582744300365448,
          "contradiction": 0.614800751209259
        }
      },
      "elapsed_ms": 5.36891701631248
    },
    {
      "id": "writes/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "independent",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "independent",
        "probability": 0.9118965864181519,
        "probabilities": {
          "entails": 0.010894261300563812,
          "independent": 0.9118965864181519,
          "contradiction": 0.07720912992954254
        }
      },
      "elapsed_ms": 7.5474579352885485
    },
    {
      "id": "writes/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "independent",
        "probability": 0.5814465880393982,
        "probabilities": {
          "entails": 0.00725574092939496,
          "independent": 0.5814465880393982,
          "contradiction": 0.41129761934280396
        }
      },
      "elapsed_ms": 9.040959179401398
    },
    {
      "id": "writes/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "independent",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "independent",
        "probability": 0.9477695822715759,
        "probabilities": {
          "entails": 0.0038047146517783403,
          "independent": 0.9477695822715759,
          "contradiction": 0.04842568561434746
        }
      },
      "elapsed_ms": 12.580499984323978
    },
    {
      "id": "approval/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "contradiction",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9366011023521423,
        "probabilities": {
          "entails": 0.0490248017013073,
          "independent": 0.014374060556292534,
          "contradiction": 0.9366011023521423
        }
      },
      "elapsed_ms": 10.748208034783602
    },
    {
      "id": "approval/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.890693187713623,
        "probabilities": {
          "entails": 0.08552171289920807,
          "independent": 0.023785142228007317,
          "contradiction": 0.890693187713623
        }
      },
      "elapsed_ms": 9.98087483458221
    },
    {
      "id": "approval/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.8087446093559265,
        "probabilities": {
          "entails": 0.18328525125980377,
          "independent": 0.007970183156430721,
          "contradiction": 0.8087446093559265
        }
      },
      "elapsed_ms": 7.4772909283638
    },
    {
      "id": "approval/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.8601778149604797,
        "probabilities": {
          "entails": 0.13379314541816711,
          "independent": 0.006029040552675724,
          "contradiction": 0.8601778149604797
        }
      },
      "elapsed_ms": 10.066834045574069
    },
    {
      "id": "environment/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.6479319930076599,
        "probabilities": {
          "entails": 0.0423143170773983,
          "independent": 0.3097536563873291,
          "contradiction": 0.6479319930076599
        }
      },
      "elapsed_ms": 5.953833926469088
    },
    {
      "id": "environment/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "independent",
        "probability": 0.8339499831199646,
        "probabilities": {
          "entails": 0.025162281468510628,
          "independent": 0.8339499831199646,
          "contradiction": 0.14088764786720276
        }
      },
      "elapsed_ms": 8.560458896681666
    },
    {
      "id": "environment/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.5112430453300476,
        "probabilities": {
          "entails": 0.018202494829893112,
          "independent": 0.4705544412136078,
          "contradiction": 0.5112430453300476
        }
      },
      "elapsed_ms": 5.681500071659684
    },
    {
      "id": "environment/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "independent",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "independent",
        "probability": 0.9723244905471802,
        "probabilities": {
          "entails": 0.007979350164532661,
          "independent": 0.9723244905471802,
          "contradiction": 0.01969609223306179
        }
      },
      "elapsed_ms": 9.288540808483958
    },
    {
      "id": "network/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.7360091209411621,
        "probabilities": {
          "entails": 0.21121680736541748,
          "independent": 0.05277407541871071,
          "contradiction": 0.7360091209411621
        }
      },
      "elapsed_ms": 4.393417155370116
    },
    {
      "id": "network/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.6857118606567383,
        "probabilities": {
          "entails": 0.2774409055709839,
          "independent": 0.036847181618213654,
          "contradiction": 0.6857118606567383
        }
      },
      "elapsed_ms": 8.455165894702077
    },
    {
      "id": "network/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "contradiction",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9629459977149963,
        "probabilities": {
          "entails": 0.02322794310748577,
          "independent": 0.013826041482388973,
          "contradiction": 0.9629459977149963
        }
      },
      "elapsed_ms": 5.692333914339542
    },
    {
      "id": "network/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.7721022367477417,
        "probabilities": {
          "entails": 0.7721022367477417,
          "independent": 0.05872306227684021,
          "contradiction": 0.1691746860742569
        }
      },
      "elapsed_ms": 10.177584132179618
    },
    {
      "id": "retention/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.8069173097610474,
        "probabilities": {
          "entails": 0.021197175607085228,
          "independent": 0.17188550531864166,
          "contradiction": 0.8069173097610474
        }
      },
      "elapsed_ms": 5.9601671528071165
    },
    {
      "id": "retention/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "independent",
        "probability": 0.5997818112373352,
        "probabilities": {
          "entails": 0.009315124712884426,
          "independent": 0.5997818112373352,
          "contradiction": 0.390903115272522
        }
      },
      "elapsed_ms": 8.082417072728276
    },
    {
      "id": "retention/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "contradiction",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9037358164787292,
        "probabilities": {
          "entails": 0.03725142776966095,
          "independent": 0.0590127594769001,
          "contradiction": 0.9037358164787292
        }
      },
      "elapsed_ms": 6.641332991421223
    },
    {
      "id": "retention/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "independent",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "independent",
        "probability": 0.9028530120849609,
        "probabilities": {
          "entails": 0.0851806253194809,
          "independent": 0.9028530120849609,
          "contradiction": 0.011966346763074398
        }
      },
      "elapsed_ms": 9.33312508277595
    },
    {
      "id": "account/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "contradiction",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9749253988265991,
        "probabilities": {
          "entails": 0.017823051661252975,
          "independent": 0.007251514121890068,
          "contradiction": 0.9749253988265991
        }
      },
      "elapsed_ms": 4.650082904845476
    },
    {
      "id": "account/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.8381843566894531,
        "probabilities": {
          "entails": 0.11233066767454147,
          "independent": 0.0494849793612957,
          "contradiction": 0.8381843566894531
        }
      },
      "elapsed_ms": 7.774499943479896
    },
    {
      "id": "account/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "contradiction",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9746154546737671,
        "probabilities": {
          "entails": 0.02065696194767952,
          "independent": 0.0047275712713599205,
          "contradiction": 0.9746154546737671
        }
      },
      "elapsed_ms": 4.9502921756356955
    },
    {
      "id": "account/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "contradiction",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.9396381378173828,
        "probabilities": {
          "entails": 0.05247235670685768,
          "independent": 0.007889507338404655,
          "contradiction": 0.9396381378173828
        }
      },
      "elapsed_ms": 8.351665921509266
    },
    {
      "id": "encryption-exception/en/plain",
      "split": "heldout",
      "label": "entails",
      "prediction": "entails",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9795278310775757,
        "probabilities": {
          "entails": 0.9795278310775757,
          "independent": 0.01420065201818943,
          "contradiction": 0.006271519232541323
        }
      },
      "elapsed_ms": 5.040541989728808
    },
    {
      "id": "encryption-exception/en/adversarial",
      "split": "heldout",
      "label": "entails",
      "prediction": "entails",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9673538208007812,
        "probabilities": {
          "entails": 0.9673538208007812,
          "independent": 0.020536255091428757,
          "contradiction": 0.012109960429370403
        }
      },
      "elapsed_ms": 7.618832867592573
    },
    {
      "id": "encryption-exception/pt/plain",
      "split": "heldout",
      "label": "entails",
      "prediction": "entails",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9923728108406067,
        "probabilities": {
          "entails": 0.9923728108406067,
          "independent": 0.006163239944726229,
          "contradiction": 0.0014639829751104116
        }
      },
      "elapsed_ms": 7.035749964416027
    },
    {
      "id": "encryption-exception/pt/adversarial",
      "split": "heldout",
      "label": "entails",
      "prediction": "entails",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9840627908706665,
        "probabilities": {
          "entails": 0.9840627908706665,
          "independent": 0.012951381504535675,
          "contradiction": 0.002985734259709716
        }
      },
      "elapsed_ms": 8.615250000730157
    },
    {
      "id": "approval-exception/en/plain",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.50346440076828,
        "probabilities": {
          "entails": 0.50346440076828,
          "independent": 0.34621676802635193,
          "contradiction": 0.15031878650188446
        }
      },
      "elapsed_ms": 6.016749888658524
    },
    {
      "id": "approval-exception/en/adversarial",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "independent",
        "probability": 0.6035898923873901,
        "probabilities": {
          "entails": 0.35007724165916443,
          "independent": 0.6035898923873901,
          "contradiction": 0.046332892030477524
        }
      },
      "elapsed_ms": 7.20187509432435
    },
    {
      "id": "approval-exception/pt/plain",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.4828646183013916,
        "probabilities": {
          "entails": 0.4828646183013916,
          "independent": 0.08639773726463318,
          "contradiction": 0.43073761463165283
        }
      },
      "elapsed_ms": 6.536375032737851
    },
    {
      "id": "approval-exception/pt/adversarial",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.3877848982810974,
        "probabilities": {
          "entails": 0.2715272605419159,
          "independent": 0.3406878113746643,
          "contradiction": 0.3877848982810974
        }
      },
      "elapsed_ms": 8.223624899983406
    },
    {
      "id": "identity/en/plain",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.8048853874206543,
        "probabilities": {
          "entails": 0.12398174405097961,
          "independent": 0.0711328387260437,
          "contradiction": 0.8048853874206543
        }
      },
      "elapsed_ms": 6.0876659117639065
    },
    {
      "id": "identity/en/adversarial",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.6517003774642944,
        "probabilities": {
          "entails": 0.6517003774642944,
          "independent": 0.07135473936796188,
          "contradiction": 0.2769447863101959
        }
      },
      "elapsed_ms": 9.065540973097086
    },
    {
      "id": "identity/pt/plain",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "contradiction",
        "probability": 0.8595636487007141,
        "probabilities": {
          "entails": 0.1360124796628952,
          "independent": 0.004423896782100201,
          "contradiction": 0.8595636487007141
        }
      },
      "elapsed_ms": 6.06787484139204
    },
    {
      "id": "identity/pt/adversarial",
      "split": "train",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.8983575701713562,
        "probabilities": {
          "entails": 0.8983575701713562,
          "independent": 0.004518668632954359,
          "contradiction": 0.09712377935647964
        }
      },
      "elapsed_ms": 11.845750035718083
    },
    {
      "id": "key-storage/en/plain",
      "split": "heldout",
      "label": "entails",
      "prediction": "entails",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9259419441223145,
        "probabilities": {
          "entails": 0.9259419441223145,
          "independent": 0.035087473690509796,
          "contradiction": 0.038970645517110825
        }
      },
      "elapsed_ms": 6.719999946653843
    },
    {
      "id": "key-storage/en/adversarial",
      "split": "heldout",
      "label": "entails",
      "prediction": "entails",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9562668204307556,
        "probabilities": {
          "entails": 0.9562668204307556,
          "independent": 0.012264302000403404,
          "contradiction": 0.03146887198090553
        }
      },
      "elapsed_ms": 11.342166922986507
    },
    {
      "id": "key-storage/pt/plain",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.8565268516540527,
        "probabilities": {
          "entails": 0.8565268516540527,
          "independent": 0.0049177235923707485,
          "contradiction": 0.13855548202991486
        }
      },
      "elapsed_ms": 6.208208156749606
    },
    {
      "id": "key-storage/pt/adversarial",
      "split": "heldout",
      "label": "entails",
      "prediction": "entails",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9051088690757751,
        "probabilities": {
          "entails": 0.9051088690757751,
          "independent": 0.002602847758680582,
          "contradiction": 0.09228835254907608
        }
      },
      "elapsed_ms": 10.89891605079174
    },
    {
      "id": "scope-topic/en/plain",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 0.00062515027821064
    },
    {
      "id": "scope-topic/en/adversarial",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 0.00016693957149982452
    },
    {
      "id": "scope-topic/pt/plain",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 0.000250060111284256
    },
    {
      "id": "scope-topic/pt/adversarial",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 4.21423465013504e-05
    },
    {
      "id": "exception-optional/en/plain",
      "split": "heldout",
      "label": "independent",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "independent",
        "probability": 0.5027864575386047,
        "probabilities": {
          "entails": 0.25551849603652954,
          "independent": 0.5027864575386047,
          "contradiction": 0.2416951060295105
        }
      },
      "elapsed_ms": 6.2432498671114445
    },
    {
      "id": "exception-optional/en/adversarial",
      "split": "heldout",
      "label": "independent",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.4975336790084839,
        "probabilities": {
          "entails": 0.4975336790084839,
          "independent": 0.3557184934616089,
          "contradiction": 0.14674784243106842
        }
      },
      "elapsed_ms": 9.761916007846594
    },
    {
      "id": "exception-optional/pt/plain",
      "split": "heldout",
      "label": "independent",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.7240204811096191,
        "probabilities": {
          "entails": 0.7240204811096191,
          "independent": 0.25387856364250183,
          "contradiction": 0.022100964561104774
        }
      },
      "elapsed_ms": 7.3960828594863415
    },
    {
      "id": "exception-optional/pt/adversarial",
      "split": "heldout",
      "label": "independent",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.7531826496124268,
        "probabilities": {
          "entails": 0.7531826496124268,
          "independent": 0.16615961492061615,
          "contradiction": 0.08065774291753769
        }
      },
      "elapsed_ms": 9.398041991516948
    },
    {
      "id": "option-renamed/en/plain",
      "split": "heldout",
      "label": "entails",
      "prediction": "entails",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9155521988868713,
        "probabilities": {
          "entails": 0.9155521988868713,
          "independent": 0.017676224932074547,
          "contradiction": 0.06677157431840897
        }
      },
      "elapsed_ms": 5.530417198315263
    },
    {
      "id": "option-renamed/en/adversarial",
      "split": "heldout",
      "label": "entails",
      "prediction": "entails",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9637542963027954,
        "probabilities": {
          "entails": 0.9637542963027954,
          "independent": 0.009494941681623459,
          "contradiction": 0.026750802993774414
        }
      },
      "elapsed_ms": 10.194250149652362
    },
    {
      "id": "option-renamed/pt/plain",
      "split": "heldout",
      "label": "entails",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.8938669562339783,
        "probabilities": {
          "entails": 0.8938669562339783,
          "independent": 0.01321505755186081,
          "contradiction": 0.09291800856590271
        }
      },
      "elapsed_ms": 4.863457987084985
    },
    {
      "id": "option-renamed/pt/adversarial",
      "split": "heldout",
      "label": "entails",
      "prediction": "entails",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.9743068814277649,
        "probabilities": {
          "entails": 0.9743068814277649,
          "independent": 0.006451589521020651,
          "contradiction": 0.019241468980908394
        }
      },
      "elapsed_ms": 9.53245791606605
    },
    {
      "id": "unknown-pronoun/en/plain",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "entity_unknown",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0008749775588512421
    },
    {
      "id": "unknown-pronoun/en/adversarial",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "entity_unknown",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.000250060111284256
    },
    {
      "id": "unknown-pronoun/pt/plain",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "entity_unknown",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.0005830079317092896
    },
    {
      "id": "unknown-pronoun/pt/adversarial",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "entity_unknown",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.00037509016692638397
    },
    {
      "id": "different-environment/en/plain",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 0.0006658956408500671
    },
    {
      "id": "different-environment/en/adversarial",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 0.00012479722499847412
    },
    {
      "id": "different-environment/pt/plain",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 0.000125030055642128
    },
    {
      "id": "different-environment/pt/adversarial",
      "split": "train",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 4.21423465013504e-05
    },
    {
      "id": "different-version/en/plain",
      "split": "heldout",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 8.405186235904694e-05
    },
    {
      "id": "different-version/en/adversarial",
      "split": "heldout",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 0.00041583552956581116
    },
    {
      "id": "different-version/pt/plain",
      "split": "heldout",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 8.288770914077759e-05
    },
    {
      "id": "different-version/pt/adversarial",
      "split": "heldout",
      "label": "independent",
      "prediction": "independent",
      "status": "different_subject_version_or_environment",
      "score": {
        "relation": "independent",
        "probability": null
      },
      "elapsed_ms": 4.21423465013504e-05
    },
    {
      "id": "retired-rule/en/plain",
      "split": "heldout",
      "label": "temporal_change",
      "prediction": "temporal_change",
      "status": "disjoint_validity",
      "score": {
        "relation": "temporal_change",
        "probability": null
      },
      "elapsed_ms": 0.000125030055642128
    },
    {
      "id": "retired-rule/en/adversarial",
      "split": "heldout",
      "label": "temporal_change",
      "prediction": "temporal_change",
      "status": "disjoint_validity",
      "score": {
        "relation": "temporal_change",
        "probability": null
      },
      "elapsed_ms": 0.000125030055642128
    },
    {
      "id": "retired-rule/pt/plain",
      "split": "heldout",
      "label": "temporal_change",
      "prediction": "temporal_change",
      "status": "disjoint_validity",
      "score": {
        "relation": "temporal_change",
        "probability": null
      },
      "elapsed_ms": 4.190951585769653e-05
    },
    {
      "id": "retired-rule/pt/adversarial",
      "split": "heldout",
      "label": "temporal_change",
      "prediction": "temporal_change",
      "status": "disjoint_validity",
      "score": {
        "relation": "temporal_change",
        "probability": null
      },
      "elapsed_ms": 8.288770914077759e-05
    },
    {
      "id": "scope-private/en/plain",
      "split": "heldout",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "scope_denied",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.000250060111284256
    },
    {
      "id": "scope-private/en/adversarial",
      "split": "heldout",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "scope_denied",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 0.000125030055642128
    },
    {
      "id": "scope-private/pt/plain",
      "split": "heldout",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "scope_denied",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 8.288770914077759e-05
    },
    {
      "id": "scope-private/pt/adversarial",
      "split": "heldout",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "scope_denied",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 8.288770914077759e-05
    },
    {
      "id": "long-truncation/en/plain",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "input_window_abstention",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 3.93499992787838
    },
    {
      "id": "long-truncation/en/adversarial",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "input_window_abstention",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 6.58445805311203
    },
    {
      "id": "long-truncation/pt/plain",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "input_window_abstention",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 4.27825003862381
    },
    {
      "id": "long-truncation/pt/adversarial",
      "split": "train",
      "label": "insufficient_context",
      "prediction": "insufficient_context",
      "status": "input_window_abstention",
      "score": {
        "relation": "insufficient_context",
        "probability": null
      },
      "elapsed_ms": 8.148290915414691
    },
    {
      "id": "distractor/en/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.6931219100952148,
        "probabilities": {
          "entails": 0.6931219100952148,
          "independent": 0.044040828943252563,
          "contradiction": 0.2628372311592102
        }
      },
      "elapsed_ms": 7.010875036939979
    },
    {
      "id": "distractor/en/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.7946034073829651,
        "probabilities": {
          "entails": 0.7946034073829651,
          "independent": 0.03479544445872307,
          "contradiction": 0.17060117423534393
        }
      },
      "elapsed_ms": 8.400333113968372
    },
    {
      "id": "distractor/pt/plain",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.5285837054252625,
        "probabilities": {
          "entails": 0.5285837054252625,
          "independent": 0.010806086473166943,
          "contradiction": 0.46061021089553833
        }
      },
      "elapsed_ms": 7.022875128313899
    },
    {
      "id": "distractor/pt/adversarial",
      "split": "heldout",
      "label": "contradiction",
      "prediction": "insufficient_context",
      "status": "uncalibrated_model_softmax",
      "score": {
        "relation": "entails",
        "probability": 0.683122456073761,
        "probabilities": {
          "entails": 0.683122456073761,
          "independent": 0.006553929764777422,
          "contradiction": 0.3103235960006714
        }
      },
      "elapsed_ms": 9.413083083927631
    }
  ]
}