{ "model": "Qwen/Qwen3-32B", "probe_layers": [ 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 63 ], "design": "lexical-swap control + cross-transfer", "n_predicates": 40, "axes": { "gender": { "n_control_pairs": 200, "layers": { "0": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/gender": { "benchmark_acc": 0.3796, "benchmark_ci95": 0.0324, "transfer_control_to_bench_sep": 0.6221, "transfer_bench_to_control_sep": 0.595, "stereotype_excess_over_lexical": -0.2425 }, "BBQ/Gender_identity": { "benchmark_acc": 0.7777, "benchmark_ci95": 0.0079, "transfer_control_to_bench_sep": 0.5148, "transfer_bench_to_control_sep": 0.545, "stereotype_excess_over_lexical": 0.263 } } }, "4": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/gender": { "benchmark_acc": 0.4401, "benchmark_ci95": 0.024, "transfer_control_to_bench_sep": 0.7023, "transfer_bench_to_control_sep": 0.615, "stereotype_excess_over_lexical": -0.2622 }, "BBQ/Gender_identity": { "benchmark_acc": 0.9583, "benchmark_ci95": 0.0042, "transfer_control_to_bench_sep": 0.5192, "transfer_bench_to_control_sep": 0.55, "stereotype_excess_over_lexical": 0.4391 } } }, "8": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/gender": { "benchmark_acc": 0.4446, "benchmark_ci95": 0.016, "transfer_control_to_bench_sep": 0.7118, "transfer_bench_to_control_sep": 0.6875, "stereotype_excess_over_lexical": -0.2672 }, "BBQ/Gender_identity": { "benchmark_acc": 0.9906, "benchmark_ci95": 0.0028, "transfer_control_to_bench_sep": 0.501, "transfer_bench_to_control_sep": 0.53, "stereotype_excess_over_lexical": 0.4896 } } }, "12": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/gender": { "benchmark_acc": 0.407, "benchmark_ci95": 0.0312, "transfer_control_to_bench_sep": 0.7328, "transfer_bench_to_control_sep": 0.71, "stereotype_excess_over_lexical": -0.3258 }, "BBQ/Gender_identity": { "benchmark_acc": 0.9876, "benchmark_ci95": 0.0034, "transfer_control_to_bench_sep": 0.509, "transfer_bench_to_control_sep": 0.565, "stereotype_excess_over_lexical": 0.4786 } } }, "16": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/gender": { "benchmark_acc": 0.4096, "benchmark_ci95": 0.0314, "transfer_control_to_bench_sep": 0.7366, "transfer_bench_to_control_sep": 0.635, "stereotype_excess_over_lexical": -0.3271 }, "BBQ/Gender_identity": { "benchmark_acc": 0.9872, "benchmark_ci95": 0.0028, "transfer_control_to_bench_sep": 0.5097, "transfer_bench_to_control_sep": 0.5075, "stereotype_excess_over_lexical": 0.4774 } } }, "20": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/gender": { "benchmark_acc": 0.4255, "benchmark_ci95": 0.0338, "transfer_control_to_bench_sep": 0.7366, "transfer_bench_to_control_sep": 0.6375, "stereotype_excess_over_lexical": -0.3112 }, "BBQ/Gender_identity": { "benchmark_acc": 0.9897, "benchmark_ci95": 0.0018, "transfer_control_to_bench_sep": 0.5363, "transfer_bench_to_control_sep": 0.515, "stereotype_excess_over_lexical": 0.4534 } } }, "24": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/gender": { "benchmark_acc": 0.4293, "benchmark_ci95": 0.0229, "transfer_control_to_bench_sep": 0.7233, "transfer_bench_to_control_sep": 0.6325, "stereotype_excess_over_lexical": -0.294 }, "BBQ/Gender_identity": { "benchmark_acc": 0.9862, "benchmark_ci95": 0.0026, "transfer_control_to_bench_sep": 0.5315, "transfer_bench_to_control_sep": 0.5, "stereotype_excess_over_lexical": 0.4547 } } }, "28": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/gender": { "benchmark_acc": 0.4337, "benchmark_ci95": 0.0276, "transfer_control_to_bench_sep": 0.75, "transfer_bench_to_control_sep": 0.6125, "stereotype_excess_over_lexical": -0.3163 }, "BBQ/Gender_identity": { "benchmark_acc": 0.9797, "benchmark_ci95": 0.004, "transfer_control_to_bench_sep": 0.5092, "transfer_bench_to_control_sep": 0.5325, "stereotype_excess_over_lexical": 0.4704 } } }, "32": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/gender": { "benchmark_acc": 0.4382, "benchmark_ci95": 0.0317, "transfer_control_to_bench_sep": 0.7481, "transfer_bench_to_control_sep": 0.665, "stereotype_excess_over_lexical": -0.3099 }, "BBQ/Gender_identity": { "benchmark_acc": 0.9738, "benchmark_ci95": 0.0051, "transfer_control_to_bench_sep": 0.5308, "transfer_bench_to_control_sep": 0.535, "stereotype_excess_over_lexical": 0.4431 } } }, "36": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/gender": { "benchmark_acc": 0.4535, "benchmark_ci95": 0.0257, "transfer_control_to_bench_sep": 0.7405, "transfer_bench_to_control_sep": 0.74, "stereotype_excess_over_lexical": -0.2869 }, "BBQ/Gender_identity": { "benchmark_acc": 0.9618, "benchmark_ci95": 0.0033, "transfer_control_to_bench_sep": 0.5323, "transfer_bench_to_control_sep": 0.515, "stereotype_excess_over_lexical": 0.4296 } } }, "40": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/gender": { "benchmark_acc": 0.4701, "benchmark_ci95": 0.0332, "transfer_control_to_bench_sep": 0.7347, "transfer_bench_to_control_sep": 0.7525, "stereotype_excess_over_lexical": -0.2647 }, "BBQ/Gender_identity": { "benchmark_acc": 0.9449, "benchmark_ci95": 0.0068, "transfer_control_to_bench_sep": 0.5277, "transfer_bench_to_control_sep": 0.545, "stereotype_excess_over_lexical": 0.4172 } } }, "44": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/gender": { "benchmark_acc": 0.472, "benchmark_ci95": 0.0243, "transfer_control_to_bench_sep": 0.7424, "transfer_bench_to_control_sep": 0.785, "stereotype_excess_over_lexical": -0.2704 }, "BBQ/Gender_identity": { "benchmark_acc": 0.9591, "benchmark_ci95": 0.0056, "transfer_control_to_bench_sep": 0.5177, "transfer_bench_to_control_sep": 0.505, "stereotype_excess_over_lexical": 0.4413 } } }, "48": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/gender": { "benchmark_acc": 0.4567, "benchmark_ci95": 0.0222, "transfer_control_to_bench_sep": 0.7366, "transfer_bench_to_control_sep": 0.755, "stereotype_excess_over_lexical": -0.2799 }, "BBQ/Gender_identity": { "benchmark_acc": 0.9377, "benchmark_ci95": 0.0065, "transfer_control_to_bench_sep": 0.5005, "transfer_bench_to_control_sep": 0.5775, "stereotype_excess_over_lexical": 0.4372 } } }, "52": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/gender": { "benchmark_acc": 0.4898, "benchmark_ci95": 0.0228, "transfer_control_to_bench_sep": 0.7405, "transfer_bench_to_control_sep": 0.755, "stereotype_excess_over_lexical": -0.2507 }, "BBQ/Gender_identity": { "benchmark_acc": 0.9278, "benchmark_ci95": 0.0061, "transfer_control_to_bench_sep": 0.5172, "transfer_bench_to_control_sep": 0.5125, "stereotype_excess_over_lexical": 0.4106 } } }, "56": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/gender": { "benchmark_acc": 0.4739, "benchmark_ci95": 0.0197, "transfer_control_to_bench_sep": 0.7309, "transfer_bench_to_control_sep": 0.665, "stereotype_excess_over_lexical": -0.257 }, "BBQ/Gender_identity": { "benchmark_acc": 0.9245, "benchmark_ci95": 0.0061, "transfer_control_to_bench_sep": 0.516, "transfer_bench_to_control_sep": 0.57, "stereotype_excess_over_lexical": 0.4085 } } }, "60": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/gender": { "benchmark_acc": 0.4732, "benchmark_ci95": 0.0169, "transfer_control_to_bench_sep": 0.7233, "transfer_bench_to_control_sep": 0.7525, "stereotype_excess_over_lexical": -0.25 }, "BBQ/Gender_identity": { "benchmark_acc": 0.9078, "benchmark_ci95": 0.0068, "transfer_control_to_bench_sep": 0.5135, "transfer_bench_to_control_sep": 0.51, "stereotype_excess_over_lexical": 0.3942 } } }, "63": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/gender": { "benchmark_acc": 0.4758, "benchmark_ci95": 0.0195, "transfer_control_to_bench_sep": 0.7214, "transfer_bench_to_control_sep": 0.7625, "stereotype_excess_over_lexical": -0.2456 }, "BBQ/Gender_identity": { "benchmark_acc": 0.89, "benchmark_ci95": 0.0064, "transfer_control_to_bench_sep": 0.5105, "transfer_bench_to_control_sep": 0.5175, "stereotype_excess_over_lexical": 0.3795 } } } }, "final_layer_verdict": { "control_acc": 1.0, "max_stereotype_excess": 0.3795, "lexical_explains_benchmark": false } }, "race-color": { "n_control_pairs": 200, "layers": { "0": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/race-color": { "benchmark_acc": 0.5943, "benchmark_ci95": 0.0228, "transfer_control_to_bench_sep": 0.7471, "transfer_bench_to_control_sep": 0.8125, "stereotype_excess_over_lexical": -0.1528 }, "BBQ/Race_ethnicity": { "benchmark_acc": 0.702, "benchmark_ci95": 0.0098, "transfer_control_to_bench_sep": 0.5213, "transfer_bench_to_control_sep": 0.715, "stereotype_excess_over_lexical": 0.1807 } } }, "4": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/race-color": { "benchmark_acc": 0.6848, "benchmark_ci95": 0.0178, "transfer_control_to_bench_sep": 0.7578, "transfer_bench_to_control_sep": 0.89, "stereotype_excess_over_lexical": -0.073 }, "BBQ/Race_ethnicity": { "benchmark_acc": 0.95, "benchmark_ci95": 0.0039, "transfer_control_to_bench_sep": 0.5069, "transfer_bench_to_control_sep": 0.5075, "stereotype_excess_over_lexical": 0.4431 } } }, "8": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/race-color": { "benchmark_acc": 0.7206, "benchmark_ci95": 0.0132, "transfer_control_to_bench_sep": 0.7645, "transfer_bench_to_control_sep": 0.935, "stereotype_excess_over_lexical": -0.044 }, "BBQ/Race_ethnicity": { "benchmark_acc": 0.9744, "benchmark_ci95": 0.0039, "transfer_control_to_bench_sep": 0.5085, "transfer_bench_to_control_sep": 0.5175, "stereotype_excess_over_lexical": 0.4659 } } }, "12": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/race-color": { "benchmark_acc": 0.7158, "benchmark_ci95": 0.0134, "transfer_control_to_bench_sep": 0.8004, "transfer_bench_to_control_sep": 0.9425, "stereotype_excess_over_lexical": -0.0846 }, "BBQ/Race_ethnicity": { "benchmark_acc": 0.9768, "benchmark_ci95": 0.0034, "transfer_control_to_bench_sep": 0.5173, "transfer_bench_to_control_sep": 0.6475, "stereotype_excess_over_lexical": 0.4595 } } }, "16": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/race-color": { "benchmark_acc": 0.7042, "benchmark_ci95": 0.0185, "transfer_control_to_bench_sep": 0.7742, "transfer_bench_to_control_sep": 0.93, "stereotype_excess_over_lexical": -0.0701 }, "BBQ/Race_ethnicity": { "benchmark_acc": 0.9736, "benchmark_ci95": 0.0043, "transfer_control_to_bench_sep": 0.5492, "transfer_bench_to_control_sep": 0.6925, "stereotype_excess_over_lexical": 0.4244 } } }, "20": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/race-color": { "benchmark_acc": 0.7306, "benchmark_ci95": 0.0125, "transfer_control_to_bench_sep": 0.7994, "transfer_bench_to_control_sep": 0.9675, "stereotype_excess_over_lexical": -0.0688 }, "BBQ/Race_ethnicity": { "benchmark_acc": 0.9715, "benchmark_ci95": 0.0045, "transfer_control_to_bench_sep": 0.5327, "transfer_bench_to_control_sep": 0.685, "stereotype_excess_over_lexical": 0.4388 } } }, "24": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/race-color": { "benchmark_acc": 0.7113, "benchmark_ci95": 0.0143, "transfer_control_to_bench_sep": 0.7829, "transfer_bench_to_control_sep": 0.9625, "stereotype_excess_over_lexical": -0.0717 }, "BBQ/Race_ethnicity": { "benchmark_acc": 0.9685, "benchmark_ci95": 0.0043, "transfer_control_to_bench_sep": 0.516, "transfer_bench_to_control_sep": 0.8425, "stereotype_excess_over_lexical": 0.4526 } } }, "28": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/race-color": { "benchmark_acc": 0.7187, "benchmark_ci95": 0.0151, "transfer_control_to_bench_sep": 0.7888, "transfer_bench_to_control_sep": 0.9525, "stereotype_excess_over_lexical": -0.0701 }, "BBQ/Race_ethnicity": { "benchmark_acc": 0.9617, "benchmark_ci95": 0.0048, "transfer_control_to_bench_sep": 0.5207, "transfer_bench_to_control_sep": 0.8275, "stereotype_excess_over_lexical": 0.441 } } }, "32": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/race-color": { "benchmark_acc": 0.7355, "benchmark_ci95": 0.0168, "transfer_control_to_bench_sep": 0.8178, "transfer_bench_to_control_sep": 0.985, "stereotype_excess_over_lexical": -0.0823 }, "BBQ/Race_ethnicity": { "benchmark_acc": 0.9595, "benchmark_ci95": 0.0053, "transfer_control_to_bench_sep": 0.5223, "transfer_bench_to_control_sep": 0.95, "stereotype_excess_over_lexical": 0.4371 } } }, "36": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/race-color": { "benchmark_acc": 0.7484, "benchmark_ci95": 0.0146, "transfer_control_to_bench_sep": 0.7984, "transfer_bench_to_control_sep": 0.93, "stereotype_excess_over_lexical": -0.05 }, "BBQ/Race_ethnicity": { "benchmark_acc": 0.9535, "benchmark_ci95": 0.0051, "transfer_control_to_bench_sep": 0.5295, "transfer_bench_to_control_sep": 0.8475, "stereotype_excess_over_lexical": 0.424 } } }, "40": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/race-color": { "benchmark_acc": 0.7403, "benchmark_ci95": 0.0137, "transfer_control_to_bench_sep": 0.7742, "transfer_bench_to_control_sep": 0.955, "stereotype_excess_over_lexical": -0.0339 }, "BBQ/Race_ethnicity": { "benchmark_acc": 0.9492, "benchmark_ci95": 0.0035, "transfer_control_to_bench_sep": 0.5184, "transfer_bench_to_control_sep": 0.795, "stereotype_excess_over_lexical": 0.4309 } } }, "44": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/race-color": { "benchmark_acc": 0.7342, "benchmark_ci95": 0.0123, "transfer_control_to_bench_sep": 0.7355, "transfer_bench_to_control_sep": 0.8675, "stereotype_excess_over_lexical": -0.0013 }, "BBQ/Race_ethnicity": { "benchmark_acc": 0.9363, "benchmark_ci95": 0.0051, "transfer_control_to_bench_sep": 0.517, "transfer_bench_to_control_sep": 0.7625, "stereotype_excess_over_lexical": 0.4193 } } }, "48": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/race-color": { "benchmark_acc": 0.7348, "benchmark_ci95": 0.0187, "transfer_control_to_bench_sep": 0.6473, "transfer_bench_to_control_sep": 0.8775, "stereotype_excess_over_lexical": 0.0875 }, "BBQ/Race_ethnicity": { "benchmark_acc": 0.9098, "benchmark_ci95": 0.0052, "transfer_control_to_bench_sep": 0.5098, "transfer_bench_to_control_sep": 0.705, "stereotype_excess_over_lexical": 0.4 } } }, "52": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/race-color": { "benchmark_acc": 0.7313, "benchmark_ci95": 0.0152, "transfer_control_to_bench_sep": 0.6647, "transfer_bench_to_control_sep": 0.9125, "stereotype_excess_over_lexical": 0.0665 }, "BBQ/Race_ethnicity": { "benchmark_acc": 0.892, "benchmark_ci95": 0.0054, "transfer_control_to_bench_sep": 0.5165, "transfer_bench_to_control_sep": 0.8175, "stereotype_excess_over_lexical": 0.3755 } } }, "56": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/race-color": { "benchmark_acc": 0.751, "benchmark_ci95": 0.0128, "transfer_control_to_bench_sep": 0.6589, "transfer_bench_to_control_sep": 0.925, "stereotype_excess_over_lexical": 0.0921 }, "BBQ/Race_ethnicity": { "benchmark_acc": 0.8799, "benchmark_ci95": 0.0059, "transfer_control_to_bench_sep": 0.5205, "transfer_bench_to_control_sep": 0.82, "stereotype_excess_over_lexical": 0.3594 } } }, "60": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/race-color": { "benchmark_acc": 0.751, "benchmark_ci95": 0.0145, "transfer_control_to_bench_sep": 0.6734, "transfer_bench_to_control_sep": 0.8925, "stereotype_excess_over_lexical": 0.0775 }, "BBQ/Race_ethnicity": { "benchmark_acc": 0.8806, "benchmark_ci95": 0.0068, "transfer_control_to_bench_sep": 0.5234, "transfer_bench_to_control_sep": 0.7725, "stereotype_excess_over_lexical": 0.3572 } } }, "63": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/race-color": { "benchmark_acc": 0.74, "benchmark_ci95": 0.014, "transfer_control_to_bench_sep": 0.6337, "transfer_bench_to_control_sep": 0.8925, "stereotype_excess_over_lexical": 0.1063 }, "BBQ/Race_ethnicity": { "benchmark_acc": 0.8706, "benchmark_ci95": 0.0055, "transfer_control_to_bench_sep": 0.5088, "transfer_bench_to_control_sep": 0.7625, "stereotype_excess_over_lexical": 0.3618 } } } }, "final_layer_verdict": { "control_acc": 1.0, "max_stereotype_excess": 0.3618, "lexical_explains_benchmark": false } }, "religion": { "n_control_pairs": 200, "layers": { "0": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/religion": { "benchmark_acc": 0.6556, "benchmark_ci95": 0.0372, "transfer_control_to_bench_sep": 0.6095, "transfer_bench_to_control_sep": 0.8675, "stereotype_excess_over_lexical": 0.046 }, "BBQ/Religion": { "benchmark_acc": 0.5386, "benchmark_ci95": 0.0077, "transfer_control_to_bench_sep": 0.5217, "transfer_bench_to_control_sep": 0.69, "stereotype_excess_over_lexical": 0.0169 } } }, "4": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/religion": { "benchmark_acc": 0.7333, "benchmark_ci95": 0.0379, "transfer_control_to_bench_sep": 0.8571, "transfer_bench_to_control_sep": 0.8425, "stereotype_excess_over_lexical": -0.1238 }, "BBQ/Religion": { "benchmark_acc": 0.9064, "benchmark_ci95": 0.0056, "transfer_control_to_bench_sep": 0.5088, "transfer_bench_to_control_sep": 0.56, "stereotype_excess_over_lexical": 0.3976 } } }, "8": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/religion": { "benchmark_acc": 0.7667, "benchmark_ci95": 0.0289, "transfer_control_to_bench_sep": 0.6762, "transfer_bench_to_control_sep": 0.9075, "stereotype_excess_over_lexical": 0.0905 }, "BBQ/Religion": { "benchmark_acc": 0.9593, "benchmark_ci95": 0.0051, "transfer_control_to_bench_sep": 0.5121, "transfer_bench_to_control_sep": 0.6125, "stereotype_excess_over_lexical": 0.4472 } } }, "12": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/religion": { "benchmark_acc": 0.7571, "benchmark_ci95": 0.0313, "transfer_control_to_bench_sep": 0.6524, "transfer_bench_to_control_sep": 0.9275, "stereotype_excess_over_lexical": 0.1048 }, "BBQ/Religion": { "benchmark_acc": 0.9726, "benchmark_ci95": 0.0043, "transfer_control_to_bench_sep": 0.5033, "transfer_bench_to_control_sep": 0.565, "stereotype_excess_over_lexical": 0.4693 } } }, "16": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/religion": { "benchmark_acc": 0.7571, "benchmark_ci95": 0.0258, "transfer_control_to_bench_sep": 0.7, "transfer_bench_to_control_sep": 0.9675, "stereotype_excess_over_lexical": 0.0571 }, "BBQ/Religion": { "benchmark_acc": 0.9649, "benchmark_ci95": 0.0033, "transfer_control_to_bench_sep": 0.5067, "transfer_bench_to_control_sep": 0.565, "stereotype_excess_over_lexical": 0.4582 } } }, "20": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/religion": { "benchmark_acc": 0.746, "benchmark_ci95": 0.0414, "transfer_control_to_bench_sep": 0.6286, "transfer_bench_to_control_sep": 0.89, "stereotype_excess_over_lexical": 0.1175 }, "BBQ/Religion": { "benchmark_acc": 0.9624, "benchmark_ci95": 0.0033, "transfer_control_to_bench_sep": 0.5008, "transfer_bench_to_control_sep": 0.5125, "stereotype_excess_over_lexical": 0.4615 } } }, "24": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/religion": { "benchmark_acc": 0.7254, "benchmark_ci95": 0.0334, "transfer_control_to_bench_sep": 0.581, "transfer_bench_to_control_sep": 0.96, "stereotype_excess_over_lexical": 0.1444 }, "BBQ/Religion": { "benchmark_acc": 0.9494, "benchmark_ci95": 0.0051, "transfer_control_to_bench_sep": 0.5004, "transfer_bench_to_control_sep": 0.5075, "stereotype_excess_over_lexical": 0.449 } } }, "28": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/religion": { "benchmark_acc": 0.7333, "benchmark_ci95": 0.0323, "transfer_control_to_bench_sep": 0.6571, "transfer_bench_to_control_sep": 0.9625, "stereotype_excess_over_lexical": 0.0762 }, "BBQ/Religion": { "benchmark_acc": 0.9303, "benchmark_ci95": 0.0078, "transfer_control_to_bench_sep": 0.5029, "transfer_bench_to_control_sep": 0.5025, "stereotype_excess_over_lexical": 0.4274 } } }, "32": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/religion": { "benchmark_acc": 0.7413, "benchmark_ci95": 0.0364, "transfer_control_to_bench_sep": 0.6333, "transfer_bench_to_control_sep": 0.955, "stereotype_excess_over_lexical": 0.1079 }, "BBQ/Religion": { "benchmark_acc": 0.9203, "benchmark_ci95": 0.0088, "transfer_control_to_bench_sep": 0.5017, "transfer_bench_to_control_sep": 0.5, "stereotype_excess_over_lexical": 0.4186 } } }, "36": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/religion": { "benchmark_acc": 0.7302, "benchmark_ci95": 0.0404, "transfer_control_to_bench_sep": 0.6286, "transfer_bench_to_control_sep": 0.9825, "stereotype_excess_over_lexical": 0.1016 }, "BBQ/Religion": { "benchmark_acc": 0.9136, "benchmark_ci95": 0.0082, "transfer_control_to_bench_sep": 0.5054, "transfer_bench_to_control_sep": 0.5025, "stereotype_excess_over_lexical": 0.4082 } } }, "40": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/religion": { "benchmark_acc": 0.7063, "benchmark_ci95": 0.0388, "transfer_control_to_bench_sep": 0.5952, "transfer_bench_to_control_sep": 0.955, "stereotype_excess_over_lexical": 0.1111 }, "BBQ/Religion": { "benchmark_acc": 0.8942, "benchmark_ci95": 0.0082, "transfer_control_to_bench_sep": 0.5054, "transfer_bench_to_control_sep": 0.5225, "stereotype_excess_over_lexical": 0.3888 } } }, "44": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/religion": { "benchmark_acc": 0.7143, "benchmark_ci95": 0.0338, "transfer_control_to_bench_sep": 0.6, "transfer_bench_to_control_sep": 0.8825, "stereotype_excess_over_lexical": 0.1143 }, "BBQ/Religion": { "benchmark_acc": 0.8857, "benchmark_ci95": 0.0066, "transfer_control_to_bench_sep": 0.5004, "transfer_bench_to_control_sep": 0.5325, "stereotype_excess_over_lexical": 0.3853 } } }, "48": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/religion": { "benchmark_acc": 0.6492, "benchmark_ci95": 0.0398, "transfer_control_to_bench_sep": 0.5905, "transfer_bench_to_control_sep": 0.905, "stereotype_excess_over_lexical": 0.0587 }, "BBQ/Religion": { "benchmark_acc": 0.8486, "benchmark_ci95": 0.0079, "transfer_control_to_bench_sep": 0.5008, "transfer_bench_to_control_sep": 0.5525, "stereotype_excess_over_lexical": 0.3478 } } }, "52": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/religion": { "benchmark_acc": 0.6524, "benchmark_ci95": 0.0407, "transfer_control_to_bench_sep": 0.5952, "transfer_bench_to_control_sep": 0.95, "stereotype_excess_over_lexical": 0.0571 }, "BBQ/Religion": { "benchmark_acc": 0.8111, "benchmark_ci95": 0.0064, "transfer_control_to_bench_sep": 0.5025, "transfer_bench_to_control_sep": 0.5775, "stereotype_excess_over_lexical": 0.3086 } } }, "56": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/religion": { "benchmark_acc": 0.7127, "benchmark_ci95": 0.0333, "transfer_control_to_bench_sep": 0.6619, "transfer_bench_to_control_sep": 0.9175, "stereotype_excess_over_lexical": 0.0508 }, "BBQ/Religion": { "benchmark_acc": 0.7861, "benchmark_ci95": 0.0061, "transfer_control_to_bench_sep": 0.5046, "transfer_bench_to_control_sep": 0.68, "stereotype_excess_over_lexical": 0.2815 } } }, "60": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/religion": { "benchmark_acc": 0.727, "benchmark_ci95": 0.0322, "transfer_control_to_bench_sep": 0.7762, "transfer_bench_to_control_sep": 0.9225, "stereotype_excess_over_lexical": -0.0492 }, "BBQ/Religion": { "benchmark_acc": 0.7775, "benchmark_ci95": 0.0055, "transfer_control_to_bench_sep": 0.5033, "transfer_bench_to_control_sep": 0.63, "stereotype_excess_over_lexical": 0.2742 } } }, "63": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/religion": { "benchmark_acc": 0.719, "benchmark_ci95": 0.0316, "transfer_control_to_bench_sep": 0.7524, "transfer_bench_to_control_sep": 0.9575, "stereotype_excess_over_lexical": -0.0333 }, "BBQ/Religion": { "benchmark_acc": 0.7719, "benchmark_ci95": 0.0093, "transfer_control_to_bench_sep": 0.5017, "transfer_bench_to_control_sep": 0.53, "stereotype_excess_over_lexical": 0.2703 } } } }, "final_layer_verdict": { "control_acc": 1.0, "max_stereotype_excess": 0.2703, "lexical_explains_benchmark": false } }, "age": { "n_control_pairs": 200, "layers": { "0": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/age": { "benchmark_acc": 0.3678, "benchmark_ci95": 0.0376, "transfer_control_to_bench_sep": 0.6724, "transfer_bench_to_control_sep": 0.64, "stereotype_excess_over_lexical": -0.3046 }, "BBQ/Age": { "benchmark_acc": 0.8442, "benchmark_ci95": 0.0068, "transfer_control_to_bench_sep": 0.5202, "transfer_bench_to_control_sep": 0.5225, "stereotype_excess_over_lexical": 0.3239 } } }, "4": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/age": { "benchmark_acc": 0.4809, "benchmark_ci95": 0.0452, "transfer_control_to_bench_sep": 0.6437, "transfer_bench_to_control_sep": 0.7775, "stereotype_excess_over_lexical": -0.1628 }, "BBQ/Age": { "benchmark_acc": 0.9922, "benchmark_ci95": 0.0024, "transfer_control_to_bench_sep": 0.5222, "transfer_bench_to_control_sep": 0.55, "stereotype_excess_over_lexical": 0.4699 } } }, "8": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/age": { "benchmark_acc": 0.5176, "benchmark_ci95": 0.0436, "transfer_control_to_bench_sep": 0.6552, "transfer_bench_to_control_sep": 0.7575, "stereotype_excess_over_lexical": -0.1376 }, "BBQ/Age": { "benchmark_acc": 0.9962, "benchmark_ci95": 0.0013, "transfer_control_to_bench_sep": 0.5625, "transfer_bench_to_control_sep": 0.505, "stereotype_excess_over_lexical": 0.4337 } } }, "12": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/age": { "benchmark_acc": 0.5268, "benchmark_ci95": 0.0513, "transfer_control_to_bench_sep": 0.6782, "transfer_bench_to_control_sep": 0.865, "stereotype_excess_over_lexical": -0.1514 }, "BBQ/Age": { "benchmark_acc": 0.9969, "benchmark_ci95": 0.001, "transfer_control_to_bench_sep": 0.5443, "transfer_bench_to_control_sep": 0.535, "stereotype_excess_over_lexical": 0.4527 } } }, "16": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/age": { "benchmark_acc": 0.515, "benchmark_ci95": 0.0442, "transfer_control_to_bench_sep": 0.6782, "transfer_bench_to_control_sep": 0.8425, "stereotype_excess_over_lexical": -0.1631 }, "BBQ/Age": { "benchmark_acc": 0.9972, "benchmark_ci95": 0.0011, "transfer_control_to_bench_sep": 0.556, "transfer_bench_to_control_sep": 0.5575, "stereotype_excess_over_lexical": 0.4412 } } }, "20": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/age": { "benchmark_acc": 0.5038, "benchmark_ci95": 0.0396, "transfer_control_to_bench_sep": 0.6494, "transfer_bench_to_control_sep": 0.8575, "stereotype_excess_over_lexical": -0.1457 }, "BBQ/Age": { "benchmark_acc": 0.9976, "benchmark_ci95": 0.001, "transfer_control_to_bench_sep": 0.552, "transfer_bench_to_control_sep": 0.5325, "stereotype_excess_over_lexical": 0.4456 } } }, "24": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/age": { "benchmark_acc": 0.4885, "benchmark_ci95": 0.0436, "transfer_control_to_bench_sep": 0.6437, "transfer_bench_to_control_sep": 0.8775, "stereotype_excess_over_lexical": -0.1552 }, "BBQ/Age": { "benchmark_acc": 0.9975, "benchmark_ci95": 0.0011, "transfer_control_to_bench_sep": 0.5813, "transfer_bench_to_control_sep": 0.5675, "stereotype_excess_over_lexical": 0.4162 } } }, "28": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/age": { "benchmark_acc": 0.4944, "benchmark_ci95": 0.0387, "transfer_control_to_bench_sep": 0.6609, "transfer_bench_to_control_sep": 0.8475, "stereotype_excess_over_lexical": -0.1665 }, "BBQ/Age": { "benchmark_acc": 0.9968, "benchmark_ci95": 0.001, "transfer_control_to_bench_sep": 0.546, "transfer_bench_to_control_sep": 0.6, "stereotype_excess_over_lexical": 0.4507 } } }, "32": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/age": { "benchmark_acc": 0.523, "benchmark_ci95": 0.0331, "transfer_control_to_bench_sep": 0.6667, "transfer_bench_to_control_sep": 0.86, "stereotype_excess_over_lexical": -0.1437 }, "BBQ/Age": { "benchmark_acc": 0.9969, "benchmark_ci95": 0.0014, "transfer_control_to_bench_sep": 0.5493, "transfer_bench_to_control_sep": 0.5575, "stereotype_excess_over_lexical": 0.4477 } } }, "36": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/age": { "benchmark_acc": 0.5341, "benchmark_ci95": 0.0385, "transfer_control_to_bench_sep": 0.6494, "transfer_bench_to_control_sep": 0.8675, "stereotype_excess_over_lexical": -0.1153 }, "BBQ/Age": { "benchmark_acc": 0.9964, "benchmark_ci95": 0.0009, "transfer_control_to_bench_sep": 0.5553, "transfer_bench_to_control_sep": 0.585, "stereotype_excess_over_lexical": 0.4412 } } }, "40": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/age": { "benchmark_acc": 0.5208, "benchmark_ci95": 0.0331, "transfer_control_to_bench_sep": 0.6724, "transfer_bench_to_control_sep": 0.8425, "stereotype_excess_over_lexical": -0.1516 }, "BBQ/Age": { "benchmark_acc": 0.9948, "benchmark_ci95": 0.0016, "transfer_control_to_bench_sep": 0.5615, "transfer_bench_to_control_sep": 0.6225, "stereotype_excess_over_lexical": 0.4333 } } }, "44": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/age": { "benchmark_acc": 0.4982, "benchmark_ci95": 0.0337, "transfer_control_to_bench_sep": 0.6724, "transfer_bench_to_control_sep": 0.8675, "stereotype_excess_over_lexical": -0.1743 }, "BBQ/Age": { "benchmark_acc": 0.9939, "benchmark_ci95": 0.0017, "transfer_control_to_bench_sep": 0.576, "transfer_bench_to_control_sep": 0.5225, "stereotype_excess_over_lexical": 0.4179 } } }, "48": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/age": { "benchmark_acc": 0.4885, "benchmark_ci95": 0.0448, "transfer_control_to_bench_sep": 0.6207, "transfer_bench_to_control_sep": 0.88, "stereotype_excess_over_lexical": -0.1322 }, "BBQ/Age": { "benchmark_acc": 0.9905, "benchmark_ci95": 0.0026, "transfer_control_to_bench_sep": 0.5615, "transfer_bench_to_control_sep": 0.605, "stereotype_excess_over_lexical": 0.429 } } }, "52": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/age": { "benchmark_acc": 0.4924, "benchmark_ci95": 0.0387, "transfer_control_to_bench_sep": 0.6494, "transfer_bench_to_control_sep": 0.855, "stereotype_excess_over_lexical": -0.157 }, "BBQ/Age": { "benchmark_acc": 0.988, "benchmark_ci95": 0.003, "transfer_control_to_bench_sep": 0.5445, "transfer_bench_to_control_sep": 0.51, "stereotype_excess_over_lexical": 0.4435 } } }, "56": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/age": { "benchmark_acc": 0.4791, "benchmark_ci95": 0.0521, "transfer_control_to_bench_sep": 0.6437, "transfer_bench_to_control_sep": 0.81, "stereotype_excess_over_lexical": -0.1646 }, "BBQ/Age": { "benchmark_acc": 0.9878, "benchmark_ci95": 0.0017, "transfer_control_to_bench_sep": 0.5202, "transfer_bench_to_control_sep": 0.5, "stereotype_excess_over_lexical": 0.4675 } } }, "60": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/age": { "benchmark_acc": 0.4715, "benchmark_ci95": 0.047, "transfer_control_to_bench_sep": 0.6494, "transfer_bench_to_control_sep": 0.7725, "stereotype_excess_over_lexical": -0.1779 }, "BBQ/Age": { "benchmark_acc": 0.9838, "benchmark_ci95": 0.0024, "transfer_control_to_bench_sep": 0.5208, "transfer_bench_to_control_sep": 0.545, "stereotype_excess_over_lexical": 0.463 } } }, "63": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/age": { "benchmark_acc": 0.4618, "benchmark_ci95": 0.0536, "transfer_control_to_bench_sep": 0.6322, "transfer_bench_to_control_sep": 0.71, "stereotype_excess_over_lexical": -0.1704 }, "BBQ/Age": { "benchmark_acc": 0.9811, "benchmark_ci95": 0.0023, "transfer_control_to_bench_sep": 0.517, "transfer_bench_to_control_sep": 0.5275, "stereotype_excess_over_lexical": 0.4641 } } } }, "final_layer_verdict": { "control_acc": 1.0, "max_stereotype_excess": 0.4641, "lexical_explains_benchmark": false } }, "disability": { "n_control_pairs": 200, "layers": { "0": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/disability": { "benchmark_acc": 0.5083, "benchmark_ci95": 0.0473, "transfer_control_to_bench_sep": 0.5417, "transfer_bench_to_control_sep": 0.5975, "stereotype_excess_over_lexical": -0.0333 }, "BBQ/Disability_status": { "benchmark_acc": 0.9951, "benchmark_ci95": 0.0014, "transfer_control_to_bench_sep": 0.5315, "transfer_bench_to_control_sep": 0.8925, "stereotype_excess_over_lexical": 0.4636 } } }, "4": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/disability": { "benchmark_acc": 0.6167, "benchmark_ci95": 0.04, "transfer_control_to_bench_sep": 0.6167, "transfer_bench_to_control_sep": 0.86, "stereotype_excess_over_lexical": -0.0 }, "BBQ/Disability_status": { "benchmark_acc": 0.9986, "benchmark_ci95": 0.001, "transfer_control_to_bench_sep": 0.5382, "transfer_bench_to_control_sep": 0.9225, "stereotype_excess_over_lexical": 0.4604 } } }, "8": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/disability": { "benchmark_acc": 0.6611, "benchmark_ci95": 0.0477, "transfer_control_to_bench_sep": 0.625, "transfer_bench_to_control_sep": 0.855, "stereotype_excess_over_lexical": 0.0361 }, "BBQ/Disability_status": { "benchmark_acc": 0.9993, "benchmark_ci95": 0.0011, "transfer_control_to_bench_sep": 0.5209, "transfer_bench_to_control_sep": 0.8675, "stereotype_excess_over_lexical": 0.4784 } } }, "12": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/disability": { "benchmark_acc": 0.6694, "benchmark_ci95": 0.0447, "transfer_control_to_bench_sep": 0.6417, "transfer_bench_to_control_sep": 0.9775, "stereotype_excess_over_lexical": 0.0278 }, "BBQ/Disability_status": { "benchmark_acc": 0.9993, "benchmark_ci95": 0.0011, "transfer_control_to_bench_sep": 0.5418, "transfer_bench_to_control_sep": 0.9025, "stereotype_excess_over_lexical": 0.4575 } } }, "16": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/disability": { "benchmark_acc": 0.6778, "benchmark_ci95": 0.0507, "transfer_control_to_bench_sep": 0.65, "transfer_bench_to_control_sep": 0.9875, "stereotype_excess_over_lexical": 0.0278 }, "BBQ/Disability_status": { "benchmark_acc": 0.9998, "benchmark_ci95": 0.0004, "transfer_control_to_bench_sep": 0.5681, "transfer_bench_to_control_sep": 0.855, "stereotype_excess_over_lexical": 0.4317 } } }, "20": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/disability": { "benchmark_acc": 0.725, "benchmark_ci95": 0.0389, "transfer_control_to_bench_sep": 0.6667, "transfer_bench_to_control_sep": 0.995, "stereotype_excess_over_lexical": 0.0583 }, "BBQ/Disability_status": { "benchmark_acc": 1.0, "benchmark_ci95": 0.0, "transfer_control_to_bench_sep": 0.5701, "transfer_bench_to_control_sep": 0.6325, "stereotype_excess_over_lexical": 0.4299 } } }, "24": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/disability": { "benchmark_acc": 0.7111, "benchmark_ci95": 0.0369, "transfer_control_to_bench_sep": 0.6583, "transfer_bench_to_control_sep": 0.9925, "stereotype_excess_over_lexical": 0.0528 }, "BBQ/Disability_status": { "benchmark_acc": 0.9997, "benchmark_ci95": 0.0005, "transfer_control_to_bench_sep": 0.5684, "transfer_bench_to_control_sep": 0.8275, "stereotype_excess_over_lexical": 0.4312 } } }, "28": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/disability": { "benchmark_acc": 0.7167, "benchmark_ci95": 0.0512, "transfer_control_to_bench_sep": 0.625, "transfer_bench_to_control_sep": 0.99, "stereotype_excess_over_lexical": 0.0917 }, "BBQ/Disability_status": { "benchmark_acc": 0.9996, "benchmark_ci95": 0.0008, "transfer_control_to_bench_sep": 0.546, "transfer_bench_to_control_sep": 0.9475, "stereotype_excess_over_lexical": 0.4536 } } }, "32": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/disability": { "benchmark_acc": 0.6889, "benchmark_ci95": 0.049, "transfer_control_to_bench_sep": 0.6583, "transfer_bench_to_control_sep": 0.965, "stereotype_excess_over_lexical": 0.0306 }, "BBQ/Disability_status": { "benchmark_acc": 0.9996, "benchmark_ci95": 0.0008, "transfer_control_to_bench_sep": 0.5511, "transfer_bench_to_control_sep": 0.92, "stereotype_excess_over_lexical": 0.4485 } } }, "36": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/disability": { "benchmark_acc": 0.675, "benchmark_ci95": 0.0542, "transfer_control_to_bench_sep": 0.6333, "transfer_bench_to_control_sep": 0.9725, "stereotype_excess_over_lexical": 0.0417 }, "BBQ/Disability_status": { "benchmark_acc": 0.9995, "benchmark_ci95": 0.0006, "transfer_control_to_bench_sep": 0.5688, "transfer_bench_to_control_sep": 0.94, "stereotype_excess_over_lexical": 0.4307 } } }, "40": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/disability": { "benchmark_acc": 0.675, "benchmark_ci95": 0.0473, "transfer_control_to_bench_sep": 0.675, "transfer_bench_to_control_sep": 0.8925, "stereotype_excess_over_lexical": 0.0 }, "BBQ/Disability_status": { "benchmark_acc": 0.998, "benchmark_ci95": 0.0014, "transfer_control_to_bench_sep": 0.588, "transfer_bench_to_control_sep": 0.7875, "stereotype_excess_over_lexical": 0.4099 } } }, "44": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/disability": { "benchmark_acc": 0.7167, "benchmark_ci95": 0.0473, "transfer_control_to_bench_sep": 0.6917, "transfer_bench_to_control_sep": 0.94, "stereotype_excess_over_lexical": 0.025 }, "BBQ/Disability_status": { "benchmark_acc": 0.9976, "benchmark_ci95": 0.0011, "transfer_control_to_bench_sep": 0.5848, "transfer_bench_to_control_sep": 0.62, "stereotype_excess_over_lexical": 0.4128 } } }, "48": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/disability": { "benchmark_acc": 0.6861, "benchmark_ci95": 0.0337, "transfer_control_to_bench_sep": 0.6583, "transfer_bench_to_control_sep": 0.965, "stereotype_excess_over_lexical": 0.0278 }, "BBQ/Disability_status": { "benchmark_acc": 0.9972, "benchmark_ci95": 0.0016, "transfer_control_to_bench_sep": 0.5813, "transfer_bench_to_control_sep": 0.8375, "stereotype_excess_over_lexical": 0.4159 } } }, "52": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/disability": { "benchmark_acc": 0.6417, "benchmark_ci95": 0.0405, "transfer_control_to_bench_sep": 0.6583, "transfer_bench_to_control_sep": 0.9875, "stereotype_excess_over_lexical": -0.0167 }, "BBQ/Disability_status": { "benchmark_acc": 0.9973, "benchmark_ci95": 0.0015, "transfer_control_to_bench_sep": 0.5479, "transfer_bench_to_control_sep": 0.9775, "stereotype_excess_over_lexical": 0.4494 } } }, "56": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/disability": { "benchmark_acc": 0.6472, "benchmark_ci95": 0.049, "transfer_control_to_bench_sep": 0.6333, "transfer_bench_to_control_sep": 0.9175, "stereotype_excess_over_lexical": 0.0139 }, "BBQ/Disability_status": { "benchmark_acc": 0.9953, "benchmark_ci95": 0.0016, "transfer_control_to_bench_sep": 0.5296, "transfer_bench_to_control_sep": 0.96, "stereotype_excess_over_lexical": 0.4657 } } }, "60": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/disability": { "benchmark_acc": 0.6306, "benchmark_ci95": 0.0436, "transfer_control_to_bench_sep": 0.6333, "transfer_bench_to_control_sep": 0.8475, "stereotype_excess_over_lexical": -0.0028 }, "BBQ/Disability_status": { "benchmark_acc": 0.9924, "benchmark_ci95": 0.0027, "transfer_control_to_bench_sep": 0.5527, "transfer_bench_to_control_sep": 0.8775, "stereotype_excess_over_lexical": 0.4397 } } }, "63": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/disability": { "benchmark_acc": 0.5722, "benchmark_ci95": 0.0461, "transfer_control_to_bench_sep": 0.625, "transfer_bench_to_control_sep": 0.85, "stereotype_excess_over_lexical": -0.0528 }, "BBQ/Disability_status": { "benchmark_acc": 0.9897, "benchmark_ci95": 0.0029, "transfer_control_to_bench_sep": 0.5392, "transfer_bench_to_control_sep": 0.6425, "stereotype_excess_over_lexical": 0.4505 } } } }, "final_layer_verdict": { "control_acc": 1.0, "max_stereotype_excess": 0.4505, "lexical_explains_benchmark": false } }, "sexual-orientation": { "n_control_pairs": 200, "layers": { "0": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/sexual-orientation": { "benchmark_acc": 0.5075, "benchmark_ci95": 0.0516, "transfer_control_to_bench_sep": 0.8036, "transfer_bench_to_control_sep": 0.745, "stereotype_excess_over_lexical": -0.296 }, "BBQ/Sexual_orientation": { "benchmark_acc": 0.6238, "benchmark_ci95": 0.0075, "transfer_control_to_bench_sep": 0.5741, "transfer_bench_to_control_sep": 0.6975, "stereotype_excess_over_lexical": 0.0498 } } }, "4": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/sexual-orientation": { "benchmark_acc": 0.6982, "benchmark_ci95": 0.0411, "transfer_control_to_bench_sep": 0.8512, "transfer_bench_to_control_sep": 0.98, "stereotype_excess_over_lexical": -0.153 }, "BBQ/Sexual_orientation": { "benchmark_acc": 0.8968, "benchmark_ci95": 0.0098, "transfer_control_to_bench_sep": 0.5747, "transfer_bench_to_control_sep": 0.535, "stereotype_excess_over_lexical": 0.3222 } } }, "8": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/sexual-orientation": { "benchmark_acc": 0.7123, "benchmark_ci95": 0.0397, "transfer_control_to_bench_sep": 0.875, "transfer_bench_to_control_sep": 0.945, "stereotype_excess_over_lexical": -0.1627 }, "BBQ/Sexual_orientation": { "benchmark_acc": 0.9313, "benchmark_ci95": 0.0081, "transfer_control_to_bench_sep": 0.5544, "transfer_bench_to_control_sep": 0.53, "stereotype_excess_over_lexical": 0.3769 } } }, "12": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/sexual-orientation": { "benchmark_acc": 0.7163, "benchmark_ci95": 0.0364, "transfer_control_to_bench_sep": 0.8869, "transfer_bench_to_control_sep": 0.9625, "stereotype_excess_over_lexical": -0.1706 }, "BBQ/Sexual_orientation": { "benchmark_acc": 0.9338, "benchmark_ci95": 0.0075, "transfer_control_to_bench_sep": 0.5556, "transfer_bench_to_control_sep": 0.605, "stereotype_excess_over_lexical": 0.3783 } } }, "16": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/sexual-orientation": { "benchmark_acc": 0.7105, "benchmark_ci95": 0.0304, "transfer_control_to_bench_sep": 0.8393, "transfer_bench_to_control_sep": 0.99, "stereotype_excess_over_lexical": -0.1288 }, "BBQ/Sexual_orientation": { "benchmark_acc": 0.9321, "benchmark_ci95": 0.0083, "transfer_control_to_bench_sep": 0.548, "transfer_bench_to_control_sep": 0.62, "stereotype_excess_over_lexical": 0.3841 } } }, "20": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/sexual-orientation": { "benchmark_acc": 0.7105, "benchmark_ci95": 0.0282, "transfer_control_to_bench_sep": 0.8155, "transfer_bench_to_control_sep": 0.9875, "stereotype_excess_over_lexical": -0.105 }, "BBQ/Sexual_orientation": { "benchmark_acc": 0.9427, "benchmark_ci95": 0.0098, "transfer_control_to_bench_sep": 0.5671, "transfer_bench_to_control_sep": 0.8125, "stereotype_excess_over_lexical": 0.3756 } } }, "24": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/sexual-orientation": { "benchmark_acc": 0.6928, "benchmark_ci95": 0.0327, "transfer_control_to_bench_sep": 0.8214, "transfer_bench_to_control_sep": 0.9875, "stereotype_excess_over_lexical": -0.1287 }, "BBQ/Sexual_orientation": { "benchmark_acc": 0.9358, "benchmark_ci95": 0.01, "transfer_control_to_bench_sep": 0.5503, "transfer_bench_to_control_sep": 0.68, "stereotype_excess_over_lexical": 0.3854 } } }, "28": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/sexual-orientation": { "benchmark_acc": 0.6849, "benchmark_ci95": 0.0382, "transfer_control_to_bench_sep": 0.8036, "transfer_bench_to_control_sep": 0.965, "stereotype_excess_over_lexical": -0.1187 }, "BBQ/Sexual_orientation": { "benchmark_acc": 0.9223, "benchmark_ci95": 0.0112, "transfer_control_to_bench_sep": 0.5446, "transfer_bench_to_control_sep": 0.7125, "stereotype_excess_over_lexical": 0.3777 } } }, "32": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/sexual-orientation": { "benchmark_acc": 0.6847, "benchmark_ci95": 0.0334, "transfer_control_to_bench_sep": 0.8036, "transfer_bench_to_control_sep": 0.9875, "stereotype_excess_over_lexical": -0.1188 }, "BBQ/Sexual_orientation": { "benchmark_acc": 0.9217, "benchmark_ci95": 0.0105, "transfer_control_to_bench_sep": 0.544, "transfer_bench_to_control_sep": 0.7075, "stereotype_excess_over_lexical": 0.3777 } } }, "36": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/sexual-orientation": { "benchmark_acc": 0.6967, "benchmark_ci95": 0.0359, "transfer_control_to_bench_sep": 0.8095, "transfer_bench_to_control_sep": 0.965, "stereotype_excess_over_lexical": -0.1129 }, "BBQ/Sexual_orientation": { "benchmark_acc": 0.9261, "benchmark_ci95": 0.0118, "transfer_control_to_bench_sep": 0.5295, "transfer_bench_to_control_sep": 0.73, "stereotype_excess_over_lexical": 0.3966 } } }, "40": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/sexual-orientation": { "benchmark_acc": 0.6769, "benchmark_ci95": 0.0382, "transfer_control_to_bench_sep": 0.7679, "transfer_bench_to_control_sep": 0.955, "stereotype_excess_over_lexical": -0.091 }, "BBQ/Sexual_orientation": { "benchmark_acc": 0.9242, "benchmark_ci95": 0.0096, "transfer_control_to_bench_sep": 0.5336, "transfer_bench_to_control_sep": 0.78, "stereotype_excess_over_lexical": 0.3906 } } }, "44": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/sexual-orientation": { "benchmark_acc": 0.7067, "benchmark_ci95": 0.0423, "transfer_control_to_bench_sep": 0.8095, "transfer_bench_to_control_sep": 0.9225, "stereotype_excess_over_lexical": -0.1029 }, "BBQ/Sexual_orientation": { "benchmark_acc": 0.9111, "benchmark_ci95": 0.0114, "transfer_control_to_bench_sep": 0.5376, "transfer_bench_to_control_sep": 0.6475, "stereotype_excess_over_lexical": 0.3734 } } }, "48": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/sexual-orientation": { "benchmark_acc": 0.7045, "benchmark_ci95": 0.0407, "transfer_control_to_bench_sep": 0.7976, "transfer_bench_to_control_sep": 0.9875, "stereotype_excess_over_lexical": -0.0932 }, "BBQ/Sexual_orientation": { "benchmark_acc": 0.8748, "benchmark_ci95": 0.0107, "transfer_control_to_bench_sep": 0.5208, "transfer_bench_to_control_sep": 0.8325, "stereotype_excess_over_lexical": 0.354 } } }, "52": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/sexual-orientation": { "benchmark_acc": 0.6629, "benchmark_ci95": 0.0263, "transfer_control_to_bench_sep": 0.7083, "transfer_bench_to_control_sep": 1.0, "stereotype_excess_over_lexical": -0.0455 }, "BBQ/Sexual_orientation": { "benchmark_acc": 0.8464, "benchmark_ci95": 0.0142, "transfer_control_to_bench_sep": 0.5284, "transfer_bench_to_control_sep": 0.71, "stereotype_excess_over_lexical": 0.3181 } } }, "56": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/sexual-orientation": { "benchmark_acc": 0.6453, "benchmark_ci95": 0.0383, "transfer_control_to_bench_sep": 0.6905, "transfer_bench_to_control_sep": 0.995, "stereotype_excess_over_lexical": -0.0451 }, "BBQ/Sexual_orientation": { "benchmark_acc": 0.825, "benchmark_ci95": 0.0125, "transfer_control_to_bench_sep": 0.5266, "transfer_bench_to_control_sep": 0.755, "stereotype_excess_over_lexical": 0.2984 } } }, "60": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/sexual-orientation": { "benchmark_acc": 0.6272, "benchmark_ci95": 0.0358, "transfer_control_to_bench_sep": 0.6845, "transfer_bench_to_control_sep": 0.9775, "stereotype_excess_over_lexical": -0.0573 }, "BBQ/Sexual_orientation": { "benchmark_acc": 0.8061, "benchmark_ci95": 0.0111, "transfer_control_to_bench_sep": 0.5104, "transfer_bench_to_control_sep": 0.7925, "stereotype_excess_over_lexical": 0.2957 } } }, "63": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/sexual-orientation": { "benchmark_acc": 0.6254, "benchmark_ci95": 0.046, "transfer_control_to_bench_sep": 0.6548, "transfer_bench_to_control_sep": 0.9525, "stereotype_excess_over_lexical": -0.0293 }, "BBQ/Sexual_orientation": { "benchmark_acc": 0.8026, "benchmark_ci95": 0.0106, "transfer_control_to_bench_sep": 0.5058, "transfer_bench_to_control_sep": 0.7225, "stereotype_excess_over_lexical": 0.2969 } } } }, "final_layer_verdict": { "control_acc": 1.0, "max_stereotype_excess": 0.2969, "lexical_explains_benchmark": false } }, "nationality": { "n_control_pairs": 200, "layers": { "0": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/nationality": { "benchmark_acc": 0.5638, "benchmark_ci95": 0.0342, "transfer_control_to_bench_sep": 0.6038, "transfer_bench_to_control_sep": 0.87, "stereotype_excess_over_lexical": -0.04 } } }, "4": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/nationality": { "benchmark_acc": 0.6593, "benchmark_ci95": 0.0324, "transfer_control_to_bench_sep": 0.6541, "transfer_bench_to_control_sep": 0.8925, "stereotype_excess_over_lexical": 0.0052 } } }, "8": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/nationality": { "benchmark_acc": 0.6866, "benchmark_ci95": 0.0311, "transfer_control_to_bench_sep": 0.673, "transfer_bench_to_control_sep": 0.9375, "stereotype_excess_over_lexical": 0.0137 } } }, "12": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/nationality": { "benchmark_acc": 0.6761, "benchmark_ci95": 0.0271, "transfer_control_to_bench_sep": 0.6509, "transfer_bench_to_control_sep": 0.9825, "stereotype_excess_over_lexical": 0.0251 } } }, "16": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/nationality": { "benchmark_acc": 0.6813, "benchmark_ci95": 0.0315, "transfer_control_to_bench_sep": 0.6792, "transfer_bench_to_control_sep": 0.9675, "stereotype_excess_over_lexical": 0.002 } } }, "20": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/nationality": { "benchmark_acc": 0.7012, "benchmark_ci95": 0.0192, "transfer_control_to_bench_sep": 0.7296, "transfer_bench_to_control_sep": 0.9075, "stereotype_excess_over_lexical": -0.0283 } } }, "24": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/nationality": { "benchmark_acc": 0.6855, "benchmark_ci95": 0.0216, "transfer_control_to_bench_sep": 0.6572, "transfer_bench_to_control_sep": 0.9475, "stereotype_excess_over_lexical": 0.0283 } } }, "28": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/nationality": { "benchmark_acc": 0.6877, "benchmark_ci95": 0.0293, "transfer_control_to_bench_sep": 0.7264, "transfer_bench_to_control_sep": 0.995, "stereotype_excess_over_lexical": -0.0387 } } }, "32": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/nationality": { "benchmark_acc": 0.6918, "benchmark_ci95": 0.0285, "transfer_control_to_bench_sep": 0.6415, "transfer_bench_to_control_sep": 0.99, "stereotype_excess_over_lexical": 0.0503 } } }, "36": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/nationality": { "benchmark_acc": 0.6897, "benchmark_ci95": 0.022, "transfer_control_to_bench_sep": 0.6509, "transfer_bench_to_control_sep": 0.9575, "stereotype_excess_over_lexical": 0.0388 } } }, "40": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/nationality": { "benchmark_acc": 0.6887, "benchmark_ci95": 0.0305, "transfer_control_to_bench_sep": 0.6824, "transfer_bench_to_control_sep": 0.925, "stereotype_excess_over_lexical": 0.0063 } } }, "44": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/nationality": { "benchmark_acc": 0.6792, "benchmark_ci95": 0.0269, "transfer_control_to_bench_sep": 0.6006, "transfer_bench_to_control_sep": 0.935, "stereotype_excess_over_lexical": 0.0786 } } }, "48": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/nationality": { "benchmark_acc": 0.6583, "benchmark_ci95": 0.0282, "transfer_control_to_bench_sep": 0.6509, "transfer_bench_to_control_sep": 0.93, "stereotype_excess_over_lexical": 0.0073 } } }, "52": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/nationality": { "benchmark_acc": 0.6498, "benchmark_ci95": 0.0314, "transfer_control_to_bench_sep": 0.695, "transfer_bench_to_control_sep": 0.9125, "stereotype_excess_over_lexical": -0.0452 } } }, "56": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/nationality": { "benchmark_acc": 0.6425, "benchmark_ci95": 0.0301, "transfer_control_to_bench_sep": 0.7138, "transfer_bench_to_control_sep": 0.985, "stereotype_excess_over_lexical": -0.0713 } } }, "60": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/nationality": { "benchmark_acc": 0.6572, "benchmark_ci95": 0.0295, "transfer_control_to_bench_sep": 0.7516, "transfer_bench_to_control_sep": 0.9425, "stereotype_excess_over_lexical": -0.0943 } } }, "63": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/nationality": { "benchmark_acc": 0.6184, "benchmark_ci95": 0.0323, "transfer_control_to_bench_sep": 0.6855, "transfer_bench_to_control_sep": 0.9425, "stereotype_excess_over_lexical": -0.0671 } } } }, "final_layer_verdict": { "control_acc": 1.0, "max_stereotype_excess": -0.0671, "lexical_explains_benchmark": true } }, "socioeconomic": { "n_control_pairs": 200, "layers": { "0": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/socioeconomic": { "benchmark_acc": 0.6026, "benchmark_ci95": 0.0244, "transfer_control_to_bench_sep": 0.7151, "transfer_bench_to_control_sep": 0.86, "stereotype_excess_over_lexical": -0.1125 } } }, "4": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/socioeconomic": { "benchmark_acc": 0.6395, "benchmark_ci95": 0.0184, "transfer_control_to_bench_sep": 0.7122, "transfer_bench_to_control_sep": 0.99, "stereotype_excess_over_lexical": -0.0727 } } }, "8": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/socioeconomic": { "benchmark_acc": 0.6414, "benchmark_ci95": 0.0282, "transfer_control_to_bench_sep": 0.6744, "transfer_bench_to_control_sep": 0.975, "stereotype_excess_over_lexical": -0.033 } } }, "12": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/socioeconomic": { "benchmark_acc": 0.6666, "benchmark_ci95": 0.0239, "transfer_control_to_bench_sep": 0.7587, "transfer_bench_to_control_sep": 0.9625, "stereotype_excess_over_lexical": -0.0921 } } }, "16": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/socioeconomic": { "benchmark_acc": 0.6879, "benchmark_ci95": 0.0219, "transfer_control_to_bench_sep": 0.7587, "transfer_bench_to_control_sep": 0.9575, "stereotype_excess_over_lexical": -0.0708 } } }, "20": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/socioeconomic": { "benchmark_acc": 0.7054, "benchmark_ci95": 0.0226, "transfer_control_to_bench_sep": 0.7994, "transfer_bench_to_control_sep": 0.95, "stereotype_excess_over_lexical": -0.094 } } }, "24": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/socioeconomic": { "benchmark_acc": 0.7102, "benchmark_ci95": 0.0231, "transfer_control_to_bench_sep": 0.7965, "transfer_bench_to_control_sep": 0.97, "stereotype_excess_over_lexical": -0.0863 } } }, "28": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/socioeconomic": { "benchmark_acc": 0.7005, "benchmark_ci95": 0.0236, "transfer_control_to_bench_sep": 0.7994, "transfer_bench_to_control_sep": 0.955, "stereotype_excess_over_lexical": -0.0989 } } }, "32": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/socioeconomic": { "benchmark_acc": 0.6773, "benchmark_ci95": 0.0234, "transfer_control_to_bench_sep": 0.8081, "transfer_bench_to_control_sep": 0.985, "stereotype_excess_over_lexical": -0.1308 } } }, "36": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/socioeconomic": { "benchmark_acc": 0.6937, "benchmark_ci95": 0.0193, "transfer_control_to_bench_sep": 0.8023, "transfer_bench_to_control_sep": 0.945, "stereotype_excess_over_lexical": -0.1086 } } }, "40": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/socioeconomic": { "benchmark_acc": 0.6704, "benchmark_ci95": 0.024, "transfer_control_to_bench_sep": 0.7703, "transfer_bench_to_control_sep": 0.945, "stereotype_excess_over_lexical": -0.0999 } } }, "44": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/socioeconomic": { "benchmark_acc": 0.6705, "benchmark_ci95": 0.0169, "transfer_control_to_bench_sep": 0.7849, "transfer_bench_to_control_sep": 0.945, "stereotype_excess_over_lexical": -0.1144 } } }, "48": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/socioeconomic": { "benchmark_acc": 0.6608, "benchmark_ci95": 0.0207, "transfer_control_to_bench_sep": 0.7791, "transfer_bench_to_control_sep": 0.955, "stereotype_excess_over_lexical": -0.1183 } } }, "52": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/socioeconomic": { "benchmark_acc": 0.6492, "benchmark_ci95": 0.0183, "transfer_control_to_bench_sep": 0.7907, "transfer_bench_to_control_sep": 0.9625, "stereotype_excess_over_lexical": -0.1415 } } }, "56": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/socioeconomic": { "benchmark_acc": 0.6636, "benchmark_ci95": 0.0237, "transfer_control_to_bench_sep": 0.75, "transfer_bench_to_control_sep": 0.9575, "stereotype_excess_over_lexical": -0.0864 } } }, "60": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/socioeconomic": { "benchmark_acc": 0.6501, "benchmark_ci95": 0.0236, "transfer_control_to_bench_sep": 0.7326, "transfer_bench_to_control_sep": 0.985, "stereotype_excess_over_lexical": -0.0824 } } }, "63": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/socioeconomic": { "benchmark_acc": 0.6201, "benchmark_ci95": 0.0215, "transfer_control_to_bench_sep": 0.7529, "transfer_bench_to_control_sep": 0.9625, "stereotype_excess_over_lexical": -0.1328 } } } }, "final_layer_verdict": { "control_acc": 1.0, "max_stereotype_excess": -0.1328, "lexical_explains_benchmark": true } }, "physical-appearance": { "n_control_pairs": 200, "layers": { "0": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/physical-appearance": { "benchmark_acc": 0.4022, "benchmark_ci95": 0.0349, "transfer_control_to_bench_sep": 0.6349, "transfer_bench_to_control_sep": 0.6125, "stereotype_excess_over_lexical": -0.2328 }, "BBQ/Physical_appearance": { "benchmark_acc": 0.9915, "benchmark_ci95": 0.0014, "transfer_control_to_bench_sep": 0.5603, "transfer_bench_to_control_sep": 0.7325, "stereotype_excess_over_lexical": 0.4313 } } }, "4": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/physical-appearance": { "benchmark_acc": 0.463, "benchmark_ci95": 0.0335, "transfer_control_to_bench_sep": 0.6508, "transfer_bench_to_control_sep": 0.6475, "stereotype_excess_over_lexical": -0.1878 }, "BBQ/Physical_appearance": { "benchmark_acc": 1.0, "benchmark_ci95": 0.0, "transfer_control_to_bench_sep": 0.5435, "transfer_bench_to_control_sep": 0.8825, "stereotype_excess_over_lexical": 0.4565 } } }, "8": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/physical-appearance": { "benchmark_acc": 0.4839, "benchmark_ci95": 0.0226, "transfer_control_to_bench_sep": 0.6429, "transfer_bench_to_control_sep": 0.6, "stereotype_excess_over_lexical": -0.159 }, "BBQ/Physical_appearance": { "benchmark_acc": 1.0, "benchmark_ci95": 0.0, "transfer_control_to_bench_sep": 0.5584, "transfer_bench_to_control_sep": 0.8675, "stereotype_excess_over_lexical": 0.4416 } } }, "12": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/physical-appearance": { "benchmark_acc": 0.4681, "benchmark_ci95": 0.0367, "transfer_control_to_bench_sep": 0.619, "transfer_bench_to_control_sep": 0.67, "stereotype_excess_over_lexical": -0.1509 }, "BBQ/Physical_appearance": { "benchmark_acc": 1.0, "benchmark_ci95": 0.0, "transfer_control_to_bench_sep": 0.5197, "transfer_bench_to_control_sep": 0.8575, "stereotype_excess_over_lexical": 0.4803 } } }, "16": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/physical-appearance": { "benchmark_acc": 0.4388, "benchmark_ci95": 0.045, "transfer_control_to_bench_sep": 0.6825, "transfer_bench_to_control_sep": 0.66, "stereotype_excess_over_lexical": -0.2438 }, "BBQ/Physical_appearance": { "benchmark_acc": 0.9998, "benchmark_ci95": 0.0003, "transfer_control_to_bench_sep": 0.5444, "transfer_bench_to_control_sep": 0.8425, "stereotype_excess_over_lexical": 0.4554 } } }, "20": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/physical-appearance": { "benchmark_acc": 0.4922, "benchmark_ci95": 0.0356, "transfer_control_to_bench_sep": 0.6587, "transfer_bench_to_control_sep": 0.6875, "stereotype_excess_over_lexical": -0.1665 }, "BBQ/Physical_appearance": { "benchmark_acc": 0.9998, "benchmark_ci95": 0.0003, "transfer_control_to_bench_sep": 0.506, "transfer_bench_to_control_sep": 0.8625, "stereotype_excess_over_lexical": 0.4938 } } }, "24": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/physical-appearance": { "benchmark_acc": 0.5025, "benchmark_ci95": 0.0426, "transfer_control_to_bench_sep": 0.6825, "transfer_bench_to_control_sep": 0.7125, "stereotype_excess_over_lexical": -0.1801 }, "BBQ/Physical_appearance": { "benchmark_acc": 0.9994, "benchmark_ci95": 0.0007, "transfer_control_to_bench_sep": 0.5165, "transfer_bench_to_control_sep": 0.8275, "stereotype_excess_over_lexical": 0.4829 } } }, "28": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/physical-appearance": { "benchmark_acc": 0.5107, "benchmark_ci95": 0.0388, "transfer_control_to_bench_sep": 0.6825, "transfer_bench_to_control_sep": 0.745, "stereotype_excess_over_lexical": -0.1719 }, "BBQ/Physical_appearance": { "benchmark_acc": 0.9996, "benchmark_ci95": 0.0006, "transfer_control_to_bench_sep": 0.5073, "transfer_bench_to_control_sep": 0.8575, "stereotype_excess_over_lexical": 0.4923 } } }, "32": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/physical-appearance": { "benchmark_acc": 0.5056, "benchmark_ci95": 0.0488, "transfer_control_to_bench_sep": 0.6905, "transfer_bench_to_control_sep": 0.7375, "stereotype_excess_over_lexical": -0.1848 }, "BBQ/Physical_appearance": { "benchmark_acc": 0.9994, "benchmark_ci95": 0.0007, "transfer_control_to_bench_sep": 0.5178, "transfer_bench_to_control_sep": 0.85, "stereotype_excess_over_lexical": 0.4816 } } }, "36": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/physical-appearance": { "benchmark_acc": 0.5236, "benchmark_ci95": 0.0347, "transfer_control_to_bench_sep": 0.6587, "transfer_bench_to_control_sep": 0.7275, "stereotype_excess_over_lexical": -0.1351 }, "BBQ/Physical_appearance": { "benchmark_acc": 0.9993, "benchmark_ci95": 0.0009, "transfer_control_to_bench_sep": 0.5222, "transfer_bench_to_control_sep": 0.825, "stereotype_excess_over_lexical": 0.4771 } } }, "40": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/physical-appearance": { "benchmark_acc": 0.5398, "benchmark_ci95": 0.0458, "transfer_control_to_bench_sep": 0.7143, "transfer_bench_to_control_sep": 0.78, "stereotype_excess_over_lexical": -0.1745 }, "BBQ/Physical_appearance": { "benchmark_acc": 0.9979, "benchmark_ci95": 0.0014, "transfer_control_to_bench_sep": 0.5339, "transfer_bench_to_control_sep": 0.87, "stereotype_excess_over_lexical": 0.4639 } } }, "44": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/physical-appearance": { "benchmark_acc": 0.5449, "benchmark_ci95": 0.0418, "transfer_control_to_bench_sep": 0.6508, "transfer_bench_to_control_sep": 0.835, "stereotype_excess_over_lexical": -0.1059 }, "BBQ/Physical_appearance": { "benchmark_acc": 0.9971, "benchmark_ci95": 0.0014, "transfer_control_to_bench_sep": 0.5057, "transfer_bench_to_control_sep": 0.735, "stereotype_excess_over_lexical": 0.4914 } } }, "48": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/physical-appearance": { "benchmark_acc": 0.5024, "benchmark_ci95": 0.0404, "transfer_control_to_bench_sep": 0.6587, "transfer_bench_to_control_sep": 0.8525, "stereotype_excess_over_lexical": -0.1564 }, "BBQ/Physical_appearance": { "benchmark_acc": 0.9962, "benchmark_ci95": 0.0026, "transfer_control_to_bench_sep": 0.5263, "transfer_bench_to_control_sep": 0.7875, "stereotype_excess_over_lexical": 0.4699 } } }, "52": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/physical-appearance": { "benchmark_acc": 0.4522, "benchmark_ci95": 0.0371, "transfer_control_to_bench_sep": 0.6429, "transfer_bench_to_control_sep": 0.6925, "stereotype_excess_over_lexical": -0.1907 }, "BBQ/Physical_appearance": { "benchmark_acc": 0.9964, "benchmark_ci95": 0.0016, "transfer_control_to_bench_sep": 0.5092, "transfer_bench_to_control_sep": 0.8775, "stereotype_excess_over_lexical": 0.4872 } } }, "56": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/physical-appearance": { "benchmark_acc": 0.3997, "benchmark_ci95": 0.0358, "transfer_control_to_bench_sep": 0.6508, "transfer_bench_to_control_sep": 0.64, "stereotype_excess_over_lexical": -0.2511 }, "BBQ/Physical_appearance": { "benchmark_acc": 0.9948, "benchmark_ci95": 0.0025, "transfer_control_to_bench_sep": 0.5063, "transfer_bench_to_control_sep": 0.81, "stereotype_excess_over_lexical": 0.4885 } } }, "60": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/physical-appearance": { "benchmark_acc": 0.3728, "benchmark_ci95": 0.038, "transfer_control_to_bench_sep": 0.6587, "transfer_bench_to_control_sep": 0.6375, "stereotype_excess_over_lexical": -0.2859 }, "BBQ/Physical_appearance": { "benchmark_acc": 0.9941, "benchmark_ci95": 0.0021, "transfer_control_to_bench_sep": 0.5035, "transfer_bench_to_control_sep": 0.7975, "stereotype_excess_over_lexical": 0.4906 } } }, "63": { "control_acc": 1.0, "control_ci95": 0.0, "benchmarks": { "CP/physical-appearance": { "benchmark_acc": 0.3517, "benchmark_ci95": 0.0412, "transfer_control_to_bench_sep": 0.5794, "transfer_bench_to_control_sep": 0.535, "stereotype_excess_over_lexical": -0.2277 }, "BBQ/Physical_appearance": { "benchmark_acc": 0.9901, "benchmark_ci95": 0.0025, "transfer_control_to_bench_sep": 0.5048, "transfer_bench_to_control_sep": 0.77, "stereotype_excess_over_lexical": 0.4853 } } } }, "final_layer_verdict": { "control_acc": 1.0, "max_stereotype_excess": 0.4853, "lexical_explains_benchmark": false } } } }