{ "schema_version": 1, "measured_at": "2026-08-03T09:23:45.176212+00:00", "harness": "scripts/sensing-select-calibrate-ood.py", "torch_version": "2.13.0+cu130", "device": "cuda", "args": { "run_json": "benchmarks/reports/tvws-retrain-seed7-2026-08-03.json", "checkpoint_dir": "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/checkpoints-seed7", "baseline_checkpoint": "external/tvws-sensing/models/checkpoints/best.pt", "ota_dir": "benchmarks/datasets/ota/nairobi-region-2026-07-29", "ota_records": 20, "samples_per_class": 2000, "seed": 7, "confidence_threshold": 0.7, "ood_quantile": 0.95, "min_noise_recall": 0.75, "min_negative_control_rate": 0.5, "ensemble_last": 5, "ensemble_stride": 0, "src_root": "external/tvws-sensing", "cache_dir": "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/cache", "json_out": "benchmarks/reports/tvws-retrain-metrics-seed7-2026-08-03.json" }, "selection": { "criterion": "argmax over epochs of real-RF records correct", "spearman_synthetic_val_acc_vs_real_records": 0.241781, "spearman_note": "Negative or near-zero means selecting by synthetic validation accuracy is uninformative about real RF for this run. It is one run; treat it as a consistency check on the 2026-07-30 result (-0.1087 and -0.316), not as an independent one.", "argmax_synthetic": { "epoch": 27, "synthetic_val_accuracy": 0.9316666666666666, "records_correct": 48, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 11, "negative_control_records": 20, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 17, "uhf32_562MHz_occupied.npy": 2, "uhf33_570MHz_occupied.npy": 9, "uhf35_586MHz_occupied.npy": 20 }, "per_channel_records": { "uhf25_506MHz_occupied.npy": 20, "uhf32_562MHz_occupied.npy": 20, "uhf33_570MHz_occupied.npy": 20, "uhf35_586MHz_occupied.npy": 20 }, "synthetic_val_noise_recall": 0.886667 }, "argmax_real_all_channels": { "epoch": 25, "criterion": "max occupied rate on verified muxes, subject to synthetic NOISE recall >= 0.75 and 618 MHz vacant rate >= 0.0", "epochs_passing_guard": 26, "epochs_total": 30, "selection_score": 0.8, "correct_on_selection_channels": 64, "records_on_selection_channels": 80, "negative_control_vacant_rate": 0.6, "synthetic_val_noise_recall": 0.86, "channels_vacant_on_selection_channels": 0, "synthetic_val_accuracy": 0.9238888888888889 }, "argmax_real_no_collapse_guard": { "epoch": 25, "criterion": "max occupied rate on verified muxes, NO collapse guard", "epochs_passing_guard": 30, "epochs_total": 30, "selection_score": 0.8, "correct_on_selection_channels": 64, "records_on_selection_channels": 80, "negative_control_vacant_rate": 0.6, "synthetic_val_noise_recall": 0.86, "channels_vacant_on_selection_channels": 0, "synthetic_val_accuracy": 0.9238888888888889 }, "guard_variants": { "primary_noise_recall_only": { "epoch": 25, "criterion": "max occupied rate on verified muxes, subject to synthetic NOISE recall >= 0.75 and 618 MHz vacant rate >= 0.0", "epochs_passing_guard": 26, "epochs_total": 30, "selection_score": 0.8, "correct_on_selection_channels": 64, "records_on_selection_channels": 80, "negative_control_vacant_rate": 0.6, "synthetic_val_noise_recall": 0.86, "channels_vacant_on_selection_channels": 0, "synthetic_val_accuracy": 0.9238888888888889 }, "strict_plus_618_arm": { "epoch": 25, "criterion": "max occupied rate on verified muxes, subject to synthetic NOISE recall >= 0.75 and 618 MHz vacant rate >= 0.5", "epochs_passing_guard": 19, "epochs_total": 30, "selection_score": 0.8, "correct_on_selection_channels": 64, "records_on_selection_channels": 80, "negative_control_vacant_rate": 0.6, "synthetic_val_noise_recall": 0.86, "channels_vacant_on_selection_channels": 0, "synthetic_val_accuracy": 0.9238888888888889 }, "no_guard": { "epoch": 25, "criterion": "max occupied rate on verified muxes, NO collapse guard", "epochs_passing_guard": 30, "epochs_total": 30, "selection_score": 0.8, "correct_on_selection_channels": 64, "records_on_selection_channels": 80, "negative_control_vacant_rate": 0.6, "synthetic_val_noise_recall": 0.86, "channels_vacant_on_selection_channels": 0, "synthetic_val_accuracy": 0.9238888888888889 } }, "collapse_guard": { "min_noise_recall": 0.75, "min_negative_control_rate": 0.0 }, "collapse_guard_choice": "PRIMARY guard is the synthetic NOISE-class recall floor ALONE. The 618 MHz arm is reported as a diagnostic and as the strict_plus_618_arm variant, but it is not the primary gate, because on the measured trajectory it rejects epoch 4 - which has NOISE recall 0.97 on 300 labelled validation records and is therefore demonstrably NOT collapsed. One channel of ambiguous provenance (the manifest calls it `unlabelled`) should not overrule 300 labelled records on the question 'can this model still say vacant'. DISCLOSURE: the 0.5 threshold on the 618 arm was set first and demoted after seeing that it binds against a non-collapsed model. Every variant is reported so the effect of that decision is visible rather than buried.", "collapse_guard_rationale": "The guard is a THRESHOLD, not a weighted term. An earlier version of this script weighted the 618 MHz vacant rate at 0.5 against the occupied rate; on this same trajectory it selected epoch 11 at 18/80 with 2 of 4 channels vacant over epoch 4 at 50/80, because one 20-record channel carried half the weight against four 80-record channels AND the behaviour it rewards is the failure mode being eliminated. Recorded because the number was produced, not because it was published.", "leave_one_channel_out": [ { "held_out_channel": "uhf25_506MHz_occupied.npy", "selected_on": [ "uhf32_562MHz_occupied.npy", "uhf33_570MHz_occupied.npy", "uhf35_586MHz_occupied.npy" ], "selected_epoch": 25, "selection_score": "46/60", "held_out_correct": 18, "held_out_records": 20, "held_out_called_vacant": false }, { "held_out_channel": "uhf32_562MHz_occupied.npy", "selected_on": [ "uhf25_506MHz_occupied.npy", "uhf33_570MHz_occupied.npy", "uhf35_586MHz_occupied.npy" ], "selected_epoch": 10, "selection_score": "57/60", "held_out_correct": 4, "held_out_records": 20, "held_out_called_vacant": false }, { "held_out_channel": "uhf33_570MHz_occupied.npy", "selected_on": [ "uhf25_506MHz_occupied.npy", "uhf32_562MHz_occupied.npy", "uhf35_586MHz_occupied.npy" ], "selected_epoch": 25, "selection_score": "45/60", "held_out_correct": 19, "held_out_records": 20, "held_out_called_vacant": false }, { "held_out_channel": "uhf35_586MHz_occupied.npy", "selected_on": [ "uhf25_506MHz_occupied.npy", "uhf32_562MHz_occupied.npy", "uhf33_570MHz_occupied.npy" ], "selected_epoch": 10, "selection_score": "44/60", "held_out_correct": 17, "held_out_records": 20, "held_out_called_vacant": false } ], "leave_one_channel_out_total": { "records_correct": 58, "records_total": 80, "channels_wrongly_vacant": 0, "channels": 4 }, "per_epoch": [ { "epoch": 1, "synthetic_val_accuracy": 0.49722222222222223, "records_correct": 1, "records_total": 80, "channels_called_vacant": 3, "channels_total": 4, "negative_control_correct": 19, "negative_control_records": 20, "synthetic_val_noise_recall": 0.003333, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 0, "uhf32_562MHz_occupied.npy": 1, "uhf33_570MHz_occupied.npy": 0, "uhf35_586MHz_occupied.npy": 0 } }, { "epoch": 2, "synthetic_val_accuracy": 0.6844444444444444, "records_correct": 38, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 7, "negative_control_records": 20, "synthetic_val_noise_recall": 0.4, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 5, "uhf32_562MHz_occupied.npy": 18, "uhf33_570MHz_occupied.npy": 10, "uhf35_586MHz_occupied.npy": 5 } }, { "epoch": 3, "synthetic_val_accuracy": 0.9005555555555556, "records_correct": 24, "records_total": 80, "channels_called_vacant": 1, "channels_total": 4, "negative_control_correct": 7, "negative_control_records": 20, "synthetic_val_noise_recall": 0.783333, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 2, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 20, "uhf35_586MHz_occupied.npy": 2 } }, { "epoch": 4, "synthetic_val_accuracy": 0.8405555555555555, "records_correct": 44, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 7, "negative_control_records": 20, "synthetic_val_noise_recall": 0.963333, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 18, "uhf32_562MHz_occupied.npy": 2, "uhf33_570MHz_occupied.npy": 4, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 5, "synthetic_val_accuracy": 0.8322222222222222, "records_correct": 48, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 2, "negative_control_records": 20, "synthetic_val_noise_recall": 0.75, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 13, "uhf32_562MHz_occupied.npy": 19, "uhf33_570MHz_occupied.npy": 10, "uhf35_586MHz_occupied.npy": 6 } }, { "epoch": 6, "synthetic_val_accuracy": 0.9061111111111111, "records_correct": 31, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 3, "negative_control_records": 20, "synthetic_val_noise_recall": 0.956667, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 4, "uhf32_562MHz_occupied.npy": 3, "uhf33_570MHz_occupied.npy": 6, "uhf35_586MHz_occupied.npy": 18 } }, { "epoch": 7, "synthetic_val_accuracy": 0.9055555555555556, "records_correct": 48, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 13, "negative_control_records": 20, "synthetic_val_noise_recall": 0.726667, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 18, "uhf32_562MHz_occupied.npy": 1, "uhf33_570MHz_occupied.npy": 13, "uhf35_586MHz_occupied.npy": 16 } }, { "epoch": 8, "synthetic_val_accuracy": 0.92, "records_correct": 40, "records_total": 80, "channels_called_vacant": 2, "channels_total": 4, "negative_control_correct": 19, "negative_control_records": 20, "synthetic_val_noise_recall": 0.943333, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 20, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 0, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 9, "synthetic_val_accuracy": 0.8677777777777778, "records_correct": 61, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 6, "negative_control_records": 20, "synthetic_val_noise_recall": 0.436667, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 19, "uhf32_562MHz_occupied.npy": 11, "uhf33_570MHz_occupied.npy": 11, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 10, "synthetic_val_accuracy": 0.9277777777777778, "records_correct": 61, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 11, "negative_control_records": 20, "synthetic_val_noise_recall": 0.843333, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 20, "uhf32_562MHz_occupied.npy": 4, "uhf33_570MHz_occupied.npy": 20, "uhf35_586MHz_occupied.npy": 17 } }, { "epoch": 11, "synthetic_val_accuracy": 0.9177777777777778, "records_correct": 47, "records_total": 80, "channels_called_vacant": 1, "channels_total": 4, "negative_control_correct": 11, "negative_control_records": 20, "synthetic_val_noise_recall": 0.853333, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 20, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 7, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 12, "synthetic_val_accuracy": 0.6372222222222222, "records_correct": 44, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 13, "negative_control_records": 20, "synthetic_val_noise_recall": 1.0, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 17, "uhf32_562MHz_occupied.npy": 2, "uhf33_570MHz_occupied.npy": 11, "uhf35_586MHz_occupied.npy": 14 } }, { "epoch": 13, "synthetic_val_accuracy": 0.9, "records_correct": 33, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 8, "negative_control_records": 20, "synthetic_val_noise_recall": 0.88, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 12, "uhf32_562MHz_occupied.npy": 6, "uhf33_570MHz_occupied.npy": 10, "uhf35_586MHz_occupied.npy": 5 } }, { "epoch": 14, "synthetic_val_accuracy": 0.9177777777777778, "records_correct": 47, "records_total": 80, "channels_called_vacant": 1, "channels_total": 4, "negative_control_correct": 11, "negative_control_records": 20, "synthetic_val_noise_recall": 0.863333, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 20, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 7, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 15, "synthetic_val_accuracy": 0.9233333333333333, "records_correct": 51, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 8, "negative_control_records": 20, "synthetic_val_noise_recall": 0.883333, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 18, "uhf32_562MHz_occupied.npy": 1, "uhf33_570MHz_occupied.npy": 12, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 16, "synthetic_val_accuracy": 0.9266666666666666, "records_correct": 41, "records_total": 80, "channels_called_vacant": 1, "channels_total": 4, "negative_control_correct": 16, "negative_control_records": 20, "synthetic_val_noise_recall": 0.92, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 18, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 3, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 17, "synthetic_val_accuracy": 0.9244444444444444, "records_correct": 54, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 14, "negative_control_records": 20, "synthetic_val_noise_recall": 0.87, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 19, "uhf32_562MHz_occupied.npy": 3, "uhf33_570MHz_occupied.npy": 12, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 18, "synthetic_val_accuracy": 0.9233333333333333, "records_correct": 45, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 11, "negative_control_records": 20, "synthetic_val_noise_recall": 0.876667, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 18, "uhf32_562MHz_occupied.npy": 1, "uhf33_570MHz_occupied.npy": 6, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 19, "synthetic_val_accuracy": 0.9233333333333333, "records_correct": 37, "records_total": 80, "channels_called_vacant": 1, "channels_total": 4, "negative_control_correct": 12, "negative_control_records": 20, "synthetic_val_noise_recall": 0.876667, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 16, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 1, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 20, "synthetic_val_accuracy": 0.9255555555555556, "records_correct": 49, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 12, "negative_control_records": 20, "synthetic_val_noise_recall": 0.866667, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 16, "uhf32_562MHz_occupied.npy": 2, "uhf33_570MHz_occupied.npy": 12, "uhf35_586MHz_occupied.npy": 19 } }, { "epoch": 21, "synthetic_val_accuracy": 0.9288888888888889, "records_correct": 40, "records_total": 80, "channels_called_vacant": 1, "channels_total": 4, "negative_control_correct": 10, "negative_control_records": 20, "synthetic_val_noise_recall": 0.873333, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 17, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 3, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 22, "synthetic_val_accuracy": 0.9277777777777778, "records_correct": 44, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 10, "negative_control_records": 20, "synthetic_val_noise_recall": 0.876667, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 18, "uhf32_562MHz_occupied.npy": 1, "uhf33_570MHz_occupied.npy": 5, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 23, "synthetic_val_accuracy": 0.9283333333333333, "records_correct": 44, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 10, "negative_control_records": 20, "synthetic_val_noise_recall": 0.876667, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 17, "uhf32_562MHz_occupied.npy": 2, "uhf33_570MHz_occupied.npy": 5, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 24, "synthetic_val_accuracy": 0.925, "records_correct": 44, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 8, "negative_control_records": 20, "synthetic_val_noise_recall": 0.85, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 17, "uhf32_562MHz_occupied.npy": 1, "uhf33_570MHz_occupied.npy": 6, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 25, "synthetic_val_accuracy": 0.9238888888888889, "records_correct": 64, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 12, "negative_control_records": 20, "synthetic_val_noise_recall": 0.86, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 18, "uhf32_562MHz_occupied.npy": 7, "uhf33_570MHz_occupied.npy": 19, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 26, "synthetic_val_accuracy": 0.9277777777777778, "records_correct": 47, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 11, "negative_control_records": 20, "synthetic_val_noise_recall": 0.876667, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 17, "uhf32_562MHz_occupied.npy": 2, "uhf33_570MHz_occupied.npy": 8, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 27, "synthetic_val_accuracy": 0.9316666666666666, "records_correct": 48, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 11, "negative_control_records": 20, "synthetic_val_noise_recall": 0.886667, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 17, "uhf32_562MHz_occupied.npy": 2, "uhf33_570MHz_occupied.npy": 9, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 28, "synthetic_val_accuracy": 0.9261111111111111, "records_correct": 47, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 11, "negative_control_records": 20, "synthetic_val_noise_recall": 0.946667, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 18, "uhf32_562MHz_occupied.npy": 1, "uhf33_570MHz_occupied.npy": 8, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 29, "synthetic_val_accuracy": 0.9261111111111111, "records_correct": 42, "records_total": 80, "channels_called_vacant": 1, "channels_total": 4, "negative_control_correct": 10, "negative_control_records": 20, "synthetic_val_noise_recall": 0.83, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 20, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 2, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 30, "synthetic_val_accuracy": 0.9316666666666666, "records_correct": 45, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 11, "negative_control_records": 20, "synthetic_val_noise_recall": 0.886667, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 19, "uhf32_562MHz_occupied.npy": 1, "uhf33_570MHz_occupied.npy": 5, "uhf35_586MHz_occupied.npy": 20 } } ] }, "checkpoints": [ { "label": "deployed-baseline", "members": [ "external/tvws-sensing/models/checkpoints/best.pt" ], "member_sha256": [ "8eca022f09b897a579ef74225881343949a1a4a53b880edabf82225aacbb34ee" ], "confidence_threshold": 0.7, "real_rf": { "records_correct": 40, "records_total": 80, "channels_wrongly_vacant": 2, "channels_total": 4, "mean_confidence_on_wrong_calls": 0.961457, "n_wrong_calls": 40, "per_channel": { "uhf25_506MHz_occupied.npy": { "uhf_channel": 25, "centre_mhz": 506, "excluded": false, "records": 20, "occupied": 20, "occupied_calibrated": 19, "class_counts": { "DVB-T": 20 }, "mean_confidence": 0.99928, "mean_confidence_on_vacant_calls": null, "mean_confidence_on_vacant_calls_calibrated": 0.698664 }, "uhf32_562MHz_occupied.npy": { "uhf_channel": 32, "centre_mhz": 562, "excluded": false, "records": 20, "occupied": 0, "occupied_calibrated": 0, "class_counts": { "Noise": 20 }, "mean_confidence": 0.990322, "mean_confidence_on_vacant_calls": 0.990322, "mean_confidence_on_vacant_calls_calibrated": 0.678017 }, "uhf33_570MHz_occupied.npy": { "uhf_channel": 33, "centre_mhz": 570, "excluded": false, "records": 20, "occupied": 0, "occupied_calibrated": 0, "class_counts": { "Noise": 20 }, "mean_confidence": 0.932593, "mean_confidence_on_vacant_calls": 0.932593, "mean_confidence_on_vacant_calls_calibrated": 0.575482 }, "uhf35_586MHz_occupied.npy": { "uhf_channel": 35, "centre_mhz": 586, "excluded": false, "records": 20, "occupied": 20, "occupied_calibrated": 20, "class_counts": { "DVB-T": 20 }, "mean_confidence": 0.999993, "mean_confidence_on_vacant_calls": null, "mean_confidence_on_vacant_calls_calibrated": null }, "uhf39_618MHz_occupied.npy": { "uhf_channel": 39, "centre_mhz": 618, "excluded": true, "records": 20, "occupied": 0, "occupied_calibrated": 0, "class_counts": { "Noise": 10, "Unknown": 10 }, "mean_confidence": 0.936383, "mean_confidence_on_vacant_calls": 0.936383, "mean_confidence_on_vacant_calls_calibrated": 0.499302 } } }, "real_rf_calibrated": { "records_correct": 39, "channels_wrongly_vacant": 2, "mean_confidence_on_wrong_calls": 0.628504, "note": "Temperature is fitted on SYNTHETIC data. Applying it to real captures is an extrapolation and is reported as such." }, "calibration": { "temperature": 4.111335, "fitted_on": "synthetic calibration split (disjoint from test split)", "synthetic_test_accuracy": 0.596667, "synthetic_per_class_recall": { "ATSC": 0.033333, "DVB-T": 0.493333, "ISDB-T": 0.486667, "Wireless Mic": 0.666667, "Noise": 0.953333, "Unknown": 0.946667 }, "synthetic_false_occupied_rate_on_vacant_classes": 0.033333, "collapse_guard_note": "Every real capture used for selection is occupied, so a model that has lost the ability to say vacant scores perfectly on the real arm. NOISE recall and the synthetic false-occupied rate are the check that this has not happened.", "ece_synthetic_before": 0.312255, "ece_synthetic_after": 0.081989, "ece_synthetic_before_bins": [ { "lo": 0.0, "hi": 0.06666666666666667, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.06666666666666667, "hi": 0.13333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.13333333333333333, "hi": 0.2, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.2, "hi": 0.26666666666666666, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.2667, "hi": 0.3333, "n": 1, "mean_conf": 0.2695, "accuracy": 0.0 }, { "lo": 0.3333, "hi": 0.4, "n": 5, "mean_conf": 0.3756, "accuracy": 0.2 }, { "lo": 0.4, "hi": 0.4667, "n": 6, "mean_conf": 0.4466, "accuracy": 0.3333 }, { "lo": 0.4667, "hi": 0.5333, "n": 22, "mean_conf": 0.5057, "accuracy": 0.3636 }, { "lo": 0.5333, "hi": 0.6, "n": 26, "mean_conf": 0.5616, "accuracy": 0.3077 }, { "lo": 0.6, "hi": 0.6667, "n": 29, "mean_conf": 0.6309, "accuracy": 0.3103 }, { "lo": 0.6667, "hi": 0.7333, "n": 36, "mean_conf": 0.6982, "accuracy": 0.2222 }, { "lo": 0.7333, "hi": 0.8, "n": 35, "mean_conf": 0.7741, "accuracy": 0.3143 }, { "lo": 0.8, "hi": 0.8667, "n": 49, "mean_conf": 0.8332, "accuracy": 0.3673 }, { "lo": 0.8667, "hi": 0.9333, "n": 82, "mean_conf": 0.902, "accuracy": 0.4146 }, { "lo": 0.9333, "hi": 1.0, "n": 609, "mean_conf": 0.9888, "accuracy": 0.7192 } ], "ece_synthetic_after_bins": [ { "lo": 0.0, "hi": 0.06666666666666667, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.06666666666666667, "hi": 0.13333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.1333, "hi": 0.2, "n": 1, "mean_conf": 0.1959, "accuracy": 0.0 }, { "lo": 0.2, "hi": 0.2667, "n": 6, "mean_conf": 0.2494, "accuracy": 0.0 }, { "lo": 0.2667, "hi": 0.3333, "n": 32, "mean_conf": 0.3057, "accuracy": 0.0625 }, { "lo": 0.3333, "hi": 0.4, "n": 64, "mean_conf": 0.3672, "accuracy": 0.3281 }, { "lo": 0.4, "hi": 0.4667, "n": 107, "mean_conf": 0.4322, "accuracy": 0.4393 }, { "lo": 0.4667, "hi": 0.5333, "n": 109, "mean_conf": 0.497, "accuracy": 0.3394 }, { "lo": 0.5333, "hi": 0.6, "n": 141, "mean_conf": 0.5663, "accuracy": 0.5319 }, { "lo": 0.6, "hi": 0.6667, "n": 139, "mean_conf": 0.6342, "accuracy": 0.6691 }, { "lo": 0.6667, "hi": 0.7333, "n": 154, "mean_conf": 0.6968, "accuracy": 0.7792 }, { "lo": 0.7333, "hi": 0.8, "n": 64, "mean_conf": 0.7636, "accuracy": 0.9375 }, { "lo": 0.8, "hi": 0.8667, "n": 51, "mean_conf": 0.8285, "accuracy": 0.9804 }, { "lo": 0.8667, "hi": 0.9333, "n": 23, "mean_conf": 0.9034, "accuracy": 1.0 }, { "lo": 0.9333, "hi": 1.0, "n": 9, "mean_conf": 0.9548, "accuracy": 1.0 } ], "ece_real_before": 0.480547, "ece_real_after": 0.331888, "ece_real_n": 80, "ece_real_note": "Top-label ECE on the real records, correctness = argmax is DVB-T, the nearest label the 6-class head has for a DVB-T2 multiplex. n=80, so the bin counts are small and this number is noisy." }, "ood": { "score": "mahalanobis_min_class_penultimate512", "threshold_quantile": 0.95, "threshold": 436.9858, "abstain_rate_synthetic_val": 0.05, "abstain_rate_synthetic_test": 0.05, "abstain_rate_real_scored": 0.2625, "abstain_rate_real_all": 0.21, "auroc_real_vs_synthetic_val": 0.776056, "median_mahalanobis_synthetic_val": 230.9166, "median_mahalanobis_real": 362.5779, "energy_comparison": { "threshold": -0.1109, "abstain_rate_synthetic_val": 0.05, "abstain_rate_synthetic_test": 0.041111, "abstain_rate_real_scored": 0.0, "auroc_real_vs_synthetic_val": 0.342494 }, "channels_wrongly_vacant_after_abstain_gate": 1, "gate_note": "A channel counts as wrongly vacant here only if EVERY record is a confident vacant DECISION. Abstained records are not decisions; the controller's standing rule is that PAWS grant state is the authority and sensing is advisory, so an abstention means do not transmit." } }, { "label": "retrain-selected-on-real-epoch25", "members": [ "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/checkpoints-seed7/epoch-025.pt" ], "member_sha256": [ "e0de40bbfa7d6ee52c7e8dcc0fb8e068cc49f3d3eade8386756138ad9205781b" ], "confidence_threshold": 0.7, "real_rf": { "records_correct": 64, "records_total": 80, "channels_wrongly_vacant": 0, "channels_total": 4, "mean_confidence_on_wrong_calls": 0.692625, "n_wrong_calls": 16, "per_channel": { "uhf25_506MHz_occupied.npy": { "uhf_channel": 25, "centre_mhz": 506, "excluded": false, "records": 20, "occupied": 18, "occupied_calibrated": 8, "class_counts": { "DVB-T": 17, "ISDB-T": 3 }, "mean_confidence": 0.846098, "mean_confidence_on_vacant_calls": 0.627611, "mean_confidence_on_vacant_calls_calibrated": 0.613781 }, "uhf32_562MHz_occupied.npy": { "uhf_channel": 32, "centre_mhz": 562, "excluded": false, "records": 20, "occupied": 7, "occupied_calibrated": 1, "class_counts": { "Noise": 7, "ISDB-T": 13 }, "mean_confidence": 0.751609, "mean_confidence_on_vacant_calls": 0.707146, "mean_confidence_on_vacant_calls_calibrated": 0.574036 }, "uhf33_570MHz_occupied.npy": { "uhf_channel": 33, "centre_mhz": 570, "excluded": false, "records": 20, "occupied": 19, "occupied_calibrated": 16, "class_counts": { "ISDB-T": 20 }, "mean_confidence": 0.930113, "mean_confidence_on_vacant_calls": 0.633878, "mean_confidence_on_vacant_calls_calibrated": 0.619693 }, "uhf35_586MHz_occupied.npy": { "uhf_channel": 35, "centre_mhz": 586, "excluded": false, "records": 20, "occupied": 20, "occupied_calibrated": 15, "class_counts": { "DVB-T": 20 }, "mean_confidence": 0.944545, "mean_confidence_on_vacant_calls": null, "mean_confidence_on_vacant_calls_calibrated": 0.660333 }, "uhf39_618MHz_occupied.npy": { "uhf_channel": 39, "centre_mhz": 618, "excluded": true, "records": 20, "occupied": 8, "occupied_calibrated": 8, "class_counts": { "Noise": 4, "ATSC": 12, "Unknown": 4 }, "mean_confidence": 0.828297, "mean_confidence_on_vacant_calls": 0.741129, "mean_confidence_on_vacant_calls_calibrated": 0.626312 } } }, "real_rf_calibrated": { "records_correct": 40, "channels_wrongly_vacant": 0, "mean_confidence_on_wrong_calls": 0.601312, "note": "Temperature is fitted on SYNTHETIC data. Applying it to real captures is an extrapolation and is reported as such." }, "calibration": { "temperature": 1.986975, "fitted_on": "synthetic calibration split (disjoint from test split)", "synthetic_test_accuracy": 0.918889, "synthetic_per_class_recall": { "ATSC": 0.953333, "DVB-T": 0.746667, "ISDB-T": 0.913333, "Wireless Mic": 1.0, "Noise": 0.9, "Unknown": 1.0 }, "synthetic_false_occupied_rate_on_vacant_classes": 0.046667, "collapse_guard_note": "Every real capture used for selection is occupied, so a model that has lost the ability to say vacant scores perfectly on the real arm. NOISE recall and the synthetic false-occupied rate are the check that this has not happened.", "ece_synthetic_before": 0.061622, "ece_synthetic_after": 0.028037, "ece_synthetic_before_bins": [ { "lo": 0.0, "hi": 0.06666666666666667, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.06666666666666667, "hi": 0.13333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.13333333333333333, "hi": 0.2, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.2, "hi": 0.26666666666666666, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.26666666666666666, "hi": 0.3333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.3333, "hi": 0.4, "n": 1, "mean_conf": 0.3849, "accuracy": 0.0 }, { "lo": 0.4, "hi": 0.4667, "n": 3, "mean_conf": 0.4541, "accuracy": 0.6667 }, { "lo": 0.4667, "hi": 0.5333, "n": 2, "mean_conf": 0.492, "accuracy": 0.0 }, { "lo": 0.5333, "hi": 0.6, "n": 5, "mean_conf": 0.5574, "accuracy": 0.6 }, { "lo": 0.6, "hi": 0.6667, "n": 11, "mean_conf": 0.6338, "accuracy": 0.5455 }, { "lo": 0.6667, "hi": 0.7333, "n": 3, "mean_conf": 0.6999, "accuracy": 0.3333 }, { "lo": 0.7333, "hi": 0.8, "n": 13, "mean_conf": 0.7724, "accuracy": 0.3077 }, { "lo": 0.8, "hi": 0.8667, "n": 9, "mean_conf": 0.8292, "accuracy": 0.6667 }, { "lo": 0.8667, "hi": 0.9333, "n": 23, "mean_conf": 0.8989, "accuracy": 0.6957 }, { "lo": 0.9333, "hi": 1.0, "n": 830, "mean_conf": 0.9976, "accuracy": 0.9506 } ], "ece_synthetic_after_bins": [ { "lo": 0.0, "hi": 0.06666666666666667, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.06666666666666667, "hi": 0.13333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.13333333333333333, "hi": 0.2, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.2, "hi": 0.26666666666666666, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.26666666666666666, "hi": 0.3333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.3333, "hi": 0.4, "n": 4, "mean_conf": 0.3757, "accuracy": 0.5 }, { "lo": 0.4, "hi": 0.4667, "n": 5, "mean_conf": 0.4294, "accuracy": 0.4 }, { "lo": 0.4667, "hi": 0.5333, "n": 9, "mean_conf": 0.4969, "accuracy": 0.6667 }, { "lo": 0.5333, "hi": 0.6, "n": 13, "mean_conf": 0.5673, "accuracy": 0.3077 }, { "lo": 0.6, "hi": 0.6667, "n": 17, "mean_conf": 0.6316, "accuracy": 0.4706 }, { "lo": 0.6667, "hi": 0.7333, "n": 18, "mean_conf": 0.6987, "accuracy": 0.6667 }, { "lo": 0.7333, "hi": 0.8, "n": 24, "mean_conf": 0.7674, "accuracy": 0.75 }, { "lo": 0.8, "hi": 0.8667, "n": 25, "mean_conf": 0.8418, "accuracy": 0.8 }, { "lo": 0.8667, "hi": 0.9333, "n": 53, "mean_conf": 0.9033, "accuracy": 0.6792 }, { "lo": 0.9333, "hi": 1.0, "n": 732, "mean_conf": 0.9864, "accuracy": 0.9822 } ], "ece_real_before": 0.405591, "ece_real_after": 0.248832, "ece_real_n": 80, "ece_real_note": "Top-label ECE on the real records, correctness = argmax is DVB-T, the nearest label the 6-class head has for a DVB-T2 multiplex. n=80, so the bin counts are small and this number is noisy." }, "ood": { "score": "mahalanobis_min_class_penultimate512", "threshold_quantile": 0.95, "threshold": 663.6078, "abstain_rate_synthetic_val": 0.05, "abstain_rate_synthetic_test": 0.07, "abstain_rate_real_scored": 0.625, "abstain_rate_real_all": 0.64, "auroc_real_vs_synthetic_val": 0.944081, "median_mahalanobis_synthetic_val": 360.2028, "median_mahalanobis_real": 747.8417, "energy_comparison": { "threshold": -1.4698, "abstain_rate_synthetic_val": 0.05, "abstain_rate_synthetic_test": 0.05, "abstain_rate_real_scored": 0.2125, "auroc_real_vs_synthetic_val": 0.906517 }, "channels_wrongly_vacant_after_abstain_gate": 0, "gate_note": "A channel counts as wrongly vacant here only if EVERY record is a confident vacant DECISION. Abstained records are not decisions; the controller's standing rule is that PAWS grant state is the authority and sensing is advisory, so an abstention means do not transmit." } }, { "label": "retrain-argmax-synthetic-epoch27", "members": [ "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/checkpoints-seed7/epoch-027.pt" ], "member_sha256": [ "8e444cb1590de5b3a57629a93d461f0323680a83ac2f6bb7e4cb3dc221b9d454" ], "confidence_threshold": 0.7, "real_rf": { "records_correct": 48, "records_total": 80, "channels_wrongly_vacant": 0, "channels_total": 4, "mean_confidence_on_wrong_calls": 0.770909, "n_wrong_calls": 32, "per_channel": { "uhf25_506MHz_occupied.npy": { "uhf_channel": 25, "centre_mhz": 506, "excluded": false, "records": 20, "occupied": 17, "occupied_calibrated": 15, "class_counts": { "DVB-T": 19, "ISDB-T": 1 }, "mean_confidence": 0.897105, "mean_confidence_on_vacant_calls": 0.564622, "mean_confidence_on_vacant_calls_calibrated": 0.563681 }, "uhf32_562MHz_occupied.npy": { "uhf_channel": 32, "centre_mhz": 562, "excluded": false, "records": 20, "occupied": 2, "occupied_calibrated": 0, "class_counts": { "Noise": 18, "ISDB-T": 2 }, "mean_confidence": 0.854919, "mean_confidence_on_vacant_calls": 0.860565, "mean_confidence_on_vacant_calls_calibrated": 0.685155 }, "uhf33_570MHz_occupied.npy": { "uhf_channel": 33, "centre_mhz": 570, "excluded": false, "records": 20, "occupied": 9, "occupied_calibrated": 2, "class_counts": { "Noise": 7, "ISDB-T": 13 }, "mean_confidence": 0.751582, "mean_confidence_on_vacant_calls": 0.680461, "mean_confidence_on_vacant_calls_calibrated": 0.582528 }, "uhf35_586MHz_occupied.npy": { "uhf_channel": 35, "centre_mhz": 586, "excluded": false, "records": 20, "occupied": 20, "occupied_calibrated": 20, "class_counts": { "DVB-T": 20 }, "mean_confidence": 0.981059, "mean_confidence_on_vacant_calls": null, "mean_confidence_on_vacant_calls_calibrated": null }, "uhf39_618MHz_occupied.npy": { "uhf_channel": 39, "centre_mhz": 618, "excluded": true, "records": 20, "occupied": 9, "occupied_calibrated": 8, "class_counts": { "Noise": 6, "ATSC": 10, "Unknown": 4 }, "mean_confidence": 0.861196, "mean_confidence_on_vacant_calls": 0.787105, "mean_confidence_on_vacant_calls_calibrated": 0.651308 } } }, "real_rf_calibrated": { "records_correct": 37, "channels_wrongly_vacant": 1, "mean_confidence_on_wrong_calls": 0.62807, "note": "Temperature is fitted on SYNTHETIC data. Applying it to real captures is an extrapolation and is reported as such." }, "calibration": { "temperature": 2.031633, "fitted_on": "synthetic calibration split (disjoint from test split)", "synthetic_test_accuracy": 0.921111, "synthetic_per_class_recall": { "ATSC": 0.966667, "DVB-T": 0.753333, "ISDB-T": 0.906667, "Wireless Mic": 1.0, "Noise": 0.9, "Unknown": 1.0 }, "synthetic_false_occupied_rate_on_vacant_classes": 0.046667, "collapse_guard_note": "Every real capture used for selection is occupied, so a model that has lost the ability to say vacant scores perfectly on the real arm. NOISE recall and the synthetic false-occupied rate are the check that this has not happened.", "ece_synthetic_before": 0.062082, "ece_synthetic_after": 0.031788, "ece_synthetic_before_bins": [ { "lo": 0.0, "hi": 0.06666666666666667, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.06666666666666667, "hi": 0.13333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.13333333333333333, "hi": 0.2, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.2, "hi": 0.26666666666666666, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.26666666666666666, "hi": 0.3333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.3333333333333333, "hi": 0.4, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.4, "hi": 0.4667, "n": 2, "mean_conf": 0.4256, "accuracy": 1.0 }, { "lo": 0.4667, "hi": 0.5333, "n": 4, "mean_conf": 0.508, "accuracy": 0.5 }, { "lo": 0.5333, "hi": 0.6, "n": 5, "mean_conf": 0.5732, "accuracy": 0.6 }, { "lo": 0.6, "hi": 0.6667, "n": 6, "mean_conf": 0.6394, "accuracy": 0.1667 }, { "lo": 0.6667, "hi": 0.7333, "n": 11, "mean_conf": 0.6975, "accuracy": 0.4545 }, { "lo": 0.7333, "hi": 0.8, "n": 7, "mean_conf": 0.7798, "accuracy": 0.5714 }, { "lo": 0.8, "hi": 0.8667, "n": 10, "mean_conf": 0.8314, "accuracy": 0.6 }, { "lo": 0.8667, "hi": 0.9333, "n": 15, "mean_conf": 0.8897, "accuracy": 0.4667 }, { "lo": 0.9333, "hi": 1.0, "n": 840, "mean_conf": 0.9975, "accuracy": 0.9512 } ], "ece_synthetic_after_bins": [ { "lo": 0.0, "hi": 0.06666666666666667, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.06666666666666667, "hi": 0.13333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.13333333333333333, "hi": 0.2, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.2, "hi": 0.26666666666666666, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.26666666666666666, "hi": 0.3333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.3333, "hi": 0.4, "n": 2, "mean_conf": 0.3662, "accuracy": 1.0 }, { "lo": 0.4, "hi": 0.4667, "n": 5, "mean_conf": 0.443, "accuracy": 0.6 }, { "lo": 0.4667, "hi": 0.5333, "n": 12, "mean_conf": 0.5001, "accuracy": 0.25 }, { "lo": 0.5333, "hi": 0.6, "n": 11, "mean_conf": 0.5568, "accuracy": 0.7273 }, { "lo": 0.6, "hi": 0.6667, "n": 14, "mean_conf": 0.6329, "accuracy": 0.4286 }, { "lo": 0.6667, "hi": 0.7333, "n": 17, "mean_conf": 0.6976, "accuracy": 0.5882 }, { "lo": 0.7333, "hi": 0.8, "n": 18, "mean_conf": 0.7646, "accuracy": 0.6667 }, { "lo": 0.8, "hi": 0.8667, "n": 40, "mean_conf": 0.835, "accuracy": 0.725 }, { "lo": 0.8667, "hi": 0.9333, "n": 41, "mean_conf": 0.9071, "accuracy": 0.7561 }, { "lo": 0.9333, "hi": 1.0, "n": 740, "mean_conf": 0.986, "accuracy": 0.9797 } ], "ece_real_before": 0.383666, "ece_real_after": 0.278444, "ece_real_n": 80, "ece_real_note": "Top-label ECE on the real records, correctness = argmax is DVB-T, the nearest label the 6-class head has for a DVB-T2 multiplex. n=80, so the bin counts are small and this number is noisy." }, "ood": { "score": "mahalanobis_min_class_penultimate512", "threshold_quantile": 0.95, "threshold": 679.0743, "abstain_rate_synthetic_val": 0.05, "abstain_rate_synthetic_test": 0.058889, "abstain_rate_real_scored": 0.575, "abstain_rate_real_all": 0.62, "auroc_real_vs_synthetic_val": 0.936356, "median_mahalanobis_synthetic_val": 366.3632, "median_mahalanobis_real": 733.1506, "energy_comparison": { "threshold": -1.5669, "abstain_rate_synthetic_val": 0.05, "abstain_rate_synthetic_test": 0.044444, "abstain_rate_real_scored": 0.1625, "auroc_real_vs_synthetic_val": 0.8952 }, "channels_wrongly_vacant_after_abstain_gate": 0, "gate_note": "A channel counts as wrongly vacant here only if EVERY record is a confident vacant DECISION. Abstained records are not decisions; the controller's standing rule is that PAWS grant state is the authority and sensing is advisory, so an abstention means do not transmit." } }, { "label": "retrain-ensemble-last5-SELECTION-FREE", "members": [ "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/checkpoints-seed7/epoch-026.pt", "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/checkpoints-seed7/epoch-027.pt", "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/checkpoints-seed7/epoch-028.pt", "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/checkpoints-seed7/epoch-029.pt", "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/checkpoints-seed7/epoch-030.pt" ], "member_sha256": [ "616d549607b85c6e05d2e9aa6f423d16314dbf363d8ed405be19bac1a360364a", "8e444cb1590de5b3a57629a93d461f0323680a83ac2f6bb7e4cb3dc221b9d454", "2b509a6a24b65de7d99a335db4e1132a8d2701e1d83b9f9d9ae86199b2dbcc7f", "acacca76db044a5e1555c189e4380711b2117b76ed56cdd0d140a077f0f7a7af", "fa68cd4b551dbb3dd8f4f5ef78fefcdf057a9aa8e88e500338eb1be446ce5114" ], "confidence_threshold": 0.7, "real_rf": { "records_correct": 43, "records_total": 80, "channels_wrongly_vacant": 0, "channels_total": 4, "mean_confidence_on_wrong_calls": 0.756326, "n_wrong_calls": 37, "per_channel": { "uhf25_506MHz_occupied.npy": { "uhf_channel": 25, "centre_mhz": 506, "excluded": false, "records": 20, "occupied": 17, "occupied_calibrated": 16, "class_counts": { "DVB-T": 20 }, "mean_confidence": 0.915003, "mean_confidence_on_vacant_calls": 0.643468, "mean_confidence_on_vacant_calls_calibrated": 0.549085 }, "uhf32_562MHz_occupied.npy": { "uhf_channel": 32, "centre_mhz": 562, "excluded": false, "records": 20, "occupied": 1, "occupied_calibrated": 0, "class_counts": { "Noise": 18, "ISDB-T": 2 }, "mean_confidence": 0.843209, "mean_confidence_on_vacant_calls": 0.843944, "mean_confidence_on_vacant_calls_calibrated": 0.691263 }, "uhf33_570MHz_occupied.npy": { "uhf_channel": 33, "centre_mhz": 570, "excluded": false, "records": 20, "occupied": 5, "occupied_calibrated": 1, "class_counts": { "Noise": 10, "ISDB-T": 10 }, "mean_confidence": 0.699191, "mean_confidence_on_vacant_calls": 0.667915, "mean_confidence_on_vacant_calls_calibrated": 0.569923 }, "uhf35_586MHz_occupied.npy": { "uhf_channel": 35, "centre_mhz": 586, "excluded": false, "records": 20, "occupied": 20, "occupied_calibrated": 20, "class_counts": { "DVB-T": 20 }, "mean_confidence": 0.979313, "mean_confidence_on_vacant_calls": null, "mean_confidence_on_vacant_calls_calibrated": null }, "uhf39_618MHz_occupied.npy": { "uhf_channel": 39, "centre_mhz": 618, "excluded": true, "records": 20, "occupied": 9, "occupied_calibrated": 9, "class_counts": { "Noise": 6, "ATSC": 9, "Unknown": 5 }, "mean_confidence": 0.843846, "mean_confidence_on_vacant_calls": 0.757147, "mean_confidence_on_vacant_calls_calibrated": 0.643395 } } }, "real_rf_calibrated": { "records_correct": 37, "channels_wrongly_vacant": 1, "mean_confidence_on_wrong_calls": 0.624422, "note": "Temperature is fitted on SYNTHETIC data. Applying it to real captures is an extrapolation and is reported as such." }, "calibration": { "temperature": 1.849418, "fitted_on": "synthetic calibration split (disjoint from test split)", "synthetic_test_accuracy": 0.92, "synthetic_per_class_recall": { "ATSC": 0.96, "DVB-T": 0.746667, "ISDB-T": 0.913333, "Wireless Mic": 1.0, "Noise": 0.9, "Unknown": 1.0 }, "synthetic_false_occupied_rate_on_vacant_classes": 0.043333, "collapse_guard_note": "Every real capture used for selection is occupied, so a model that has lost the ability to say vacant scores perfectly on the real arm. NOISE recall and the synthetic false-occupied rate are the check that this has not happened.", "ece_synthetic_before": 0.058682, "ece_synthetic_after": 0.026573, "ece_synthetic_before_bins": [ { "lo": 0.0, "hi": 0.06666666666666667, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.06666666666666667, "hi": 0.13333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.13333333333333333, "hi": 0.2, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.2, "hi": 0.26666666666666666, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.26666666666666666, "hi": 0.3333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.3333333333333333, "hi": 0.4, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.4, "hi": 0.4667, "n": 2, "mean_conf": 0.4629, "accuracy": 0.5 }, { "lo": 0.4667, "hi": 0.5333, "n": 7, "mean_conf": 0.5066, "accuracy": 0.5714 }, { "lo": 0.5333, "hi": 0.6, "n": 3, "mean_conf": 0.5582, "accuracy": 1.0 }, { "lo": 0.6, "hi": 0.6667, "n": 9, "mean_conf": 0.6347, "accuracy": 0.3333 }, { "lo": 0.6667, "hi": 0.7333, "n": 11, "mean_conf": 0.6922, "accuracy": 0.4545 }, { "lo": 0.7333, "hi": 0.8, "n": 18, "mean_conf": 0.7653, "accuracy": 0.6111 }, { "lo": 0.8, "hi": 0.8667, "n": 13, "mean_conf": 0.8333, "accuracy": 0.4615 }, { "lo": 0.8667, "hi": 0.9333, "n": 18, "mean_conf": 0.9057, "accuracy": 0.5556 }, { "lo": 0.9333, "hi": 1.0, "n": 819, "mean_conf": 0.9972, "accuracy": 0.9585 } ], "ece_synthetic_after_bins": [ { "lo": 0.0, "hi": 0.06666666666666667, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.06666666666666667, "hi": 0.13333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.13333333333333333, "hi": 0.2, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.2, "hi": 0.26666666666666666, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.26666666666666666, "hi": 0.3333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.3333, "hi": 0.4, "n": 2, "mean_conf": 0.3948, "accuracy": 0.5 }, { "lo": 0.4, "hi": 0.4667, "n": 5, "mean_conf": 0.4333, "accuracy": 0.6 }, { "lo": 0.4667, "hi": 0.5333, "n": 13, "mean_conf": 0.4966, "accuracy": 0.5385 }, { "lo": 0.5333, "hi": 0.6, "n": 17, "mean_conf": 0.562, "accuracy": 0.5882 }, { "lo": 0.6, "hi": 0.6667, "n": 20, "mean_conf": 0.6301, "accuracy": 0.5 }, { "lo": 0.6667, "hi": 0.7333, "n": 16, "mean_conf": 0.7038, "accuracy": 0.3125 }, { "lo": 0.7333, "hi": 0.8, "n": 24, "mean_conf": 0.7739, "accuracy": 0.7917 }, { "lo": 0.8, "hi": 0.8667, "n": 30, "mean_conf": 0.8356, "accuracy": 0.8 }, { "lo": 0.8667, "hi": 0.9333, "n": 37, "mean_conf": 0.9035, "accuracy": 0.7297 }, { "lo": 0.9333, "hi": 1.0, "n": 736, "mean_conf": 0.9879, "accuracy": 0.981 } ], "ece_real_before": 0.359179, "ece_real_after": 0.280267, "ece_real_n": 80, "ece_real_note": "Top-label ECE on the real records, correctness = argmax is DVB-T, the nearest label the 6-class head has for a DVB-T2 multiplex. n=80, so the bin counts are small and this number is noisy." }, "ood": { "score": "mahalanobis_min_class_penultimate512", "threshold_quantile": 0.95, "threshold": 681.419, "abstain_rate_synthetic_val": 0.05, "abstain_rate_synthetic_test": 0.057778, "abstain_rate_real_scored": 0.525, "abstain_rate_real_all": 0.58, "auroc_real_vs_synthetic_val": 0.937217, "median_mahalanobis_synthetic_val": 366.5517, "median_mahalanobis_real": 763.3402, "energy_comparison": { "threshold": -1.505, "abstain_rate_synthetic_val": 0.05, "abstain_rate_synthetic_test": 0.045556, "abstain_rate_real_scored": 0.1875, "auroc_real_vs_synthetic_val": 0.892911 }, "channels_wrongly_vacant_after_abstain_gate": 0, "gate_note": "A channel counts as wrongly vacant here only if EVERY record is a confident vacant DECISION. Abstained records are not decisions; the controller's standing rule is that PAWS grant state is the authority and sensing is advisory, so an abstention means do not transmit." } } ], "honesty": { "selection_set_is_not_a_test_set": "The four verified multiplexes were used to SELECT the shipped epoch. Their score for that checkpoint is a selection score and is optimistically biased. The leave-one-channel-out total is the closest to unbiased that four channels allow: each held-out channel is scored by a checkpoint selected without it.", "what_no_real_number_here_can_prove": "One site, one afternoon, one mismatched 860-930 MHz antenna at 470-700 MHz, four channels, every capture occupied. There are ZERO real vacant captures, so no false-positive rate on real RF can be measured at all, and a model answering 'occupied' unconditionally scores 80/80 here. Nothing here generalises to another site or another receiver.", "calibration_temperature_is_synthetic": "Temperature is fitted on held-out SYNTHETIC data because that is the only labelled data with enough records to fit anything. Applying it to real captures is an extrapolation. The real-RF ECE is reported before and after so the extrapolation is visible." } }