{ "schema_version": 1, "measured_at": "2026-08-03T09:17:01.537987+00:00", "harness": "scripts/sensing-select-calibrate-ood.py", "torch_version": "2.13.0+cu130", "device": "cuda", "args": { "run_json": "benchmarks/reports/tvws-retrain-seed42-2026-08-03.json", "checkpoint_dir": "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/checkpoints-seed42", "baseline_checkpoint": "external/tvws-sensing/models/checkpoints/best.pt", "ota_dir": "benchmarks/datasets/ota/nairobi-region-2026-07-29", "ota_records": 20, "samples_per_class": 2000, "seed": 42, "confidence_threshold": 0.7, "ood_quantile": 0.95, "min_noise_recall": 0.75, "min_negative_control_rate": 0.5, "ensemble_last": 5, "ensemble_stride": 5, "src_root": "external/tvws-sensing", "cache_dir": "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/cache", "json_out": "benchmarks/reports/tvws-retrain-metrics-seed42-2026-08-03.json" }, "selection": { "criterion": "argmax over epochs of real-RF records correct", "spearman_synthetic_val_acc_vs_real_records": 0.322124, "spearman_note": "Negative or near-zero means selecting by synthetic validation accuracy is uninformative about real RF for this run. It is one run; treat it as a consistency check on the 2026-07-30 result (-0.1087 and -0.316), not as an independent one.", "argmax_synthetic": { "epoch": 26, "synthetic_val_accuracy": 0.9372222222222222, "records_correct": 43, "records_total": 80, "channels_called_vacant": 1, "channels_total": 4, "negative_control_correct": 9, "negative_control_records": 20, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 20, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 3, "uhf35_586MHz_occupied.npy": 20 }, "per_channel_records": { "uhf25_506MHz_occupied.npy": 20, "uhf32_562MHz_occupied.npy": 20, "uhf33_570MHz_occupied.npy": 20, "uhf35_586MHz_occupied.npy": 20 }, "synthetic_val_noise_recall": 0.866667 }, "argmax_real_all_channels": { "epoch": 4, "criterion": "max occupied rate on verified muxes, subject to synthetic NOISE recall >= 0.75 and 618 MHz vacant rate >= 0.0", "epochs_passing_guard": 25, "epochs_total": 30, "selection_score": 0.625, "correct_on_selection_channels": 50, "records_on_selection_channels": 80, "negative_control_vacant_rate": 0.45, "synthetic_val_noise_recall": 0.97, "channels_vacant_on_selection_channels": 1, "synthetic_val_accuracy": 0.9177777777777778 }, "argmax_real_no_collapse_guard": { "epoch": 4, "criterion": "max occupied rate on verified muxes, NO collapse guard", "epochs_passing_guard": 30, "epochs_total": 30, "selection_score": 0.625, "correct_on_selection_channels": 50, "records_on_selection_channels": 80, "negative_control_vacant_rate": 0.45, "synthetic_val_noise_recall": 0.97, "channels_vacant_on_selection_channels": 1, "synthetic_val_accuracy": 0.9177777777777778 }, "guard_variants": { "primary_noise_recall_only": { "epoch": 4, "criterion": "max occupied rate on verified muxes, subject to synthetic NOISE recall >= 0.75 and 618 MHz vacant rate >= 0.0", "epochs_passing_guard": 25, "epochs_total": 30, "selection_score": 0.625, "correct_on_selection_channels": 50, "records_on_selection_channels": 80, "negative_control_vacant_rate": 0.45, "synthetic_val_noise_recall": 0.97, "channels_vacant_on_selection_channels": 1, "synthetic_val_accuracy": 0.9177777777777778 }, "strict_plus_618_arm": { "epoch": 13, "criterion": "max occupied rate on verified muxes, subject to synthetic NOISE recall >= 0.75 and 618 MHz vacant rate >= 0.5", "epochs_passing_guard": 6, "epochs_total": 30, "selection_score": 0.475, "correct_on_selection_channels": 38, "records_on_selection_channels": 80, "negative_control_vacant_rate": 0.5, "synthetic_val_noise_recall": 0.853333, "channels_vacant_on_selection_channels": 1, "synthetic_val_accuracy": 0.93 }, "no_guard": { "epoch": 4, "criterion": "max occupied rate on verified muxes, NO collapse guard", "epochs_passing_guard": 30, "epochs_total": 30, "selection_score": 0.625, "correct_on_selection_channels": 50, "records_on_selection_channels": 80, "negative_control_vacant_rate": 0.45, "synthetic_val_noise_recall": 0.97, "channels_vacant_on_selection_channels": 1, "synthetic_val_accuracy": 0.9177777777777778 } }, "collapse_guard": { "min_noise_recall": 0.75, "min_negative_control_rate": 0.0 }, "collapse_guard_choice": "PRIMARY guard is the synthetic NOISE-class recall floor ALONE. The 618 MHz arm is reported as a diagnostic and as the strict_plus_618_arm variant, but it is not the primary gate, because on the measured trajectory it rejects epoch 4 - which has NOISE recall 0.97 on 300 labelled validation records and is therefore demonstrably NOT collapsed. One channel of ambiguous provenance (the manifest calls it `unlabelled`) should not overrule 300 labelled records on the question 'can this model still say vacant'. DISCLOSURE: the 0.5 threshold on the 618 arm was set first and demoted after seeing that it binds against a non-collapsed model. Every variant is reported so the effect of that decision is visible rather than buried.", "collapse_guard_rationale": "The guard is a THRESHOLD, not a weighted term. An earlier version of this script weighted the 618 MHz vacant rate at 0.5 against the occupied rate; on this same trajectory it selected epoch 11 at 18/80 with 2 of 4 channels vacant over epoch 4 at 50/80, because one 20-record channel carried half the weight against four 80-record channels AND the behaviour it rewards is the failure mode being eliminated. Recorded because the number was produced, not because it was published.", "leave_one_channel_out": [ { "held_out_channel": "uhf25_506MHz_occupied.npy", "selected_on": [ "uhf32_562MHz_occupied.npy", "uhf33_570MHz_occupied.npy", "uhf35_586MHz_occupied.npy" ], "selected_epoch": 4, "selection_score": "37/60", "held_out_correct": 13, "held_out_records": 20, "held_out_called_vacant": false }, { "held_out_channel": "uhf32_562MHz_occupied.npy", "selected_on": [ "uhf25_506MHz_occupied.npy", "uhf33_570MHz_occupied.npy", "uhf35_586MHz_occupied.npy" ], "selected_epoch": 4, "selection_score": "50/60", "held_out_correct": 0, "held_out_records": 20, "held_out_called_vacant": true }, { "held_out_channel": "uhf33_570MHz_occupied.npy", "selected_on": [ "uhf25_506MHz_occupied.npy", "uhf32_562MHz_occupied.npy", "uhf35_586MHz_occupied.npy" ], "selected_epoch": 15, "selection_score": "41/60", "held_out_correct": 1, "held_out_records": 20, "held_out_called_vacant": false }, { "held_out_channel": "uhf35_586MHz_occupied.npy", "selected_on": [ "uhf25_506MHz_occupied.npy", "uhf32_562MHz_occupied.npy", "uhf33_570MHz_occupied.npy" ], "selected_epoch": 4, "selection_score": "30/60", "held_out_correct": 20, "held_out_records": 20, "held_out_called_vacant": false } ], "leave_one_channel_out_total": { "records_correct": 34, "records_total": 80, "channels_wrongly_vacant": 1, "channels": 4 }, "per_epoch": [ { "epoch": 1, "synthetic_val_accuracy": 0.45, "records_correct": 9, "records_total": 80, "channels_called_vacant": 1, "channels_total": 4, "negative_control_correct": 12, "negative_control_records": 20, "synthetic_val_noise_recall": 0.29, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 3, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 5, "uhf35_586MHz_occupied.npy": 1 } }, { "epoch": 2, "synthetic_val_accuracy": 0.8311111111111111, "records_correct": 7, "records_total": 80, "channels_called_vacant": 3, "channels_total": 4, "negative_control_correct": 17, "negative_control_records": 20, "synthetic_val_noise_recall": 0.856667, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 0, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 7, "uhf35_586MHz_occupied.npy": 0 } }, { "epoch": 3, "synthetic_val_accuracy": 0.8794444444444445, "records_correct": 36, "records_total": 80, "channels_called_vacant": 1, "channels_total": 4, "negative_control_correct": 10, "negative_control_records": 20, "synthetic_val_noise_recall": 0.97, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 4, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 16, "uhf35_586MHz_occupied.npy": 16 } }, { "epoch": 4, "synthetic_val_accuracy": 0.9177777777777778, "records_correct": 50, "records_total": 80, "channels_called_vacant": 1, "channels_total": 4, "negative_control_correct": 9, "negative_control_records": 20, "synthetic_val_noise_recall": 0.97, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 13, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 17, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 5, "synthetic_val_accuracy": 0.8994444444444445, "records_correct": 42, "records_total": 80, "channels_called_vacant": 1, "channels_total": 4, "negative_control_correct": 6, "negative_control_records": 20, "synthetic_val_noise_recall": 0.696667, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 19, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 3, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 6, "synthetic_val_accuracy": 0.9094444444444445, "records_correct": 20, "records_total": 80, "channels_called_vacant": 1, "channels_total": 4, "negative_control_correct": 11, "negative_control_records": 20, "synthetic_val_noise_recall": 0.903333, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 8, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 4, "uhf35_586MHz_occupied.npy": 8 } }, { "epoch": 7, "synthetic_val_accuracy": 0.905, "records_correct": 45, "records_total": 80, "channels_called_vacant": 1, "channels_total": 4, "negative_control_correct": 9, "negative_control_records": 20, "synthetic_val_noise_recall": 0.676667, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 19, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 6, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 8, "synthetic_val_accuracy": 0.9233333333333333, "records_correct": 38, "records_total": 80, "channels_called_vacant": 1, "channels_total": 4, "negative_control_correct": 1, "negative_control_records": 20, "synthetic_val_noise_recall": 0.883333, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 17, "uhf32_562MHz_occupied.npy": 1, "uhf33_570MHz_occupied.npy": 0, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 9, "synthetic_val_accuracy": 0.9038888888888889, "records_correct": 27, "records_total": 80, "channels_called_vacant": 1, "channels_total": 4, "negative_control_correct": 15, "negative_control_records": 20, "synthetic_val_noise_recall": 0.906667, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 11, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 1, "uhf35_586MHz_occupied.npy": 15 } }, { "epoch": 10, "synthetic_val_accuracy": 0.8522222222222222, "records_correct": 44, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 2, "negative_control_records": 20, "synthetic_val_noise_recall": 0.576667, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 19, "uhf32_562MHz_occupied.npy": 1, "uhf33_570MHz_occupied.npy": 5, "uhf35_586MHz_occupied.npy": 19 } }, { "epoch": 11, "synthetic_val_accuracy": 0.6811111111111111, "records_correct": 18, "records_total": 80, "channels_called_vacant": 2, "channels_total": 4, "negative_control_correct": 19, "negative_control_records": 20, "synthetic_val_noise_recall": 0.96, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 7, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 0, "uhf35_586MHz_occupied.npy": 11 } }, { "epoch": 12, "synthetic_val_accuracy": 0.9238888888888889, "records_correct": 36, "records_total": 80, "channels_called_vacant": 1, "channels_total": 4, "negative_control_correct": 8, "negative_control_records": 20, "synthetic_val_noise_recall": 0.883333, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 15, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 3, "uhf35_586MHz_occupied.npy": 18 } }, { "epoch": 13, "synthetic_val_accuracy": 0.93, "records_correct": 38, "records_total": 80, "channels_called_vacant": 1, "channels_total": 4, "negative_control_correct": 10, "negative_control_records": 20, "synthetic_val_noise_recall": 0.853333, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 14, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 4, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 14, "synthetic_val_accuracy": 0.9316666666666666, "records_correct": 37, "records_total": 80, "channels_called_vacant": 2, "channels_total": 4, "negative_control_correct": 6, "negative_control_records": 20, "synthetic_val_noise_recall": 0.916667, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 17, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 0, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 15, "synthetic_val_accuracy": 0.9294444444444444, "records_correct": 42, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 5, "negative_control_records": 20, "synthetic_val_noise_recall": 0.843333, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 20, "uhf32_562MHz_occupied.npy": 1, "uhf33_570MHz_occupied.npy": 1, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 16, "synthetic_val_accuracy": 0.925, "records_correct": 30, "records_total": 80, "channels_called_vacant": 2, "channels_total": 4, "negative_control_correct": 8, "negative_control_records": 20, "synthetic_val_noise_recall": 0.883333, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 16, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 0, "uhf35_586MHz_occupied.npy": 14 } }, { "epoch": 17, "synthetic_val_accuracy": 0.8788888888888889, "records_correct": 38, "records_total": 80, "channels_called_vacant": 0, "channels_total": 4, "negative_control_correct": 1, "negative_control_records": 20, "synthetic_val_noise_recall": 0.636667, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 12, "uhf32_562MHz_occupied.npy": 2, "uhf33_570MHz_occupied.npy": 4, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 18, "synthetic_val_accuracy": 0.9322222222222222, "records_correct": 40, "records_total": 80, "channels_called_vacant": 2, "channels_total": 4, "negative_control_correct": 8, "negative_control_records": 20, "synthetic_val_noise_recall": 0.893333, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 20, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 0, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 19, "synthetic_val_accuracy": 0.9316666666666666, "records_correct": 39, "records_total": 80, "channels_called_vacant": 1, "channels_total": 4, "negative_control_correct": 9, "negative_control_records": 20, "synthetic_val_noise_recall": 0.87, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 16, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 3, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 20, "synthetic_val_accuracy": 0.9322222222222222, "records_correct": 41, "records_total": 80, "channels_called_vacant": 1, "channels_total": 4, "negative_control_correct": 9, "negative_control_records": 20, "synthetic_val_noise_recall": 0.883333, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 19, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 2, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 21, "synthetic_val_accuracy": 0.93, "records_correct": 38, "records_total": 80, "channels_called_vacant": 2, "channels_total": 4, "negative_control_correct": 9, "negative_control_records": 20, "synthetic_val_noise_recall": 0.873333, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 18, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 0, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 22, "synthetic_val_accuracy": 0.9305555555555556, "records_correct": 36, "records_total": 80, "channels_called_vacant": 2, "channels_total": 4, "negative_control_correct": 7, "negative_control_records": 20, "synthetic_val_noise_recall": 0.88, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 16, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 0, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 23, "synthetic_val_accuracy": 0.9361111111111111, "records_correct": 40, "records_total": 80, "channels_called_vacant": 2, "channels_total": 4, "negative_control_correct": 3, "negative_control_records": 20, "synthetic_val_noise_recall": 0.903333, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 20, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 0, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 24, "synthetic_val_accuracy": 0.9344444444444444, "records_correct": 37, "records_total": 80, "channels_called_vacant": 2, "channels_total": 4, "negative_control_correct": 6, "negative_control_records": 20, "synthetic_val_noise_recall": 0.873333, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 17, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 0, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 25, "synthetic_val_accuracy": 0.935, "records_correct": 42, "records_total": 80, "channels_called_vacant": 1, "channels_total": 4, "negative_control_correct": 6, "negative_control_records": 20, "synthetic_val_noise_recall": 0.88, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 20, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 2, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 26, "synthetic_val_accuracy": 0.9372222222222222, "records_correct": 43, "records_total": 80, "channels_called_vacant": 1, "channels_total": 4, "negative_control_correct": 9, "negative_control_records": 20, "synthetic_val_noise_recall": 0.866667, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 20, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 3, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 27, "synthetic_val_accuracy": 0.9294444444444444, "records_correct": 38, "records_total": 80, "channels_called_vacant": 1, "channels_total": 4, "negative_control_correct": 8, "negative_control_records": 20, "synthetic_val_noise_recall": 0.906667, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 17, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 1, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 28, "synthetic_val_accuracy": 0.9327777777777778, "records_correct": 39, "records_total": 80, "channels_called_vacant": 2, "channels_total": 4, "negative_control_correct": 7, "negative_control_records": 20, "synthetic_val_noise_recall": 0.9, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 19, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 0, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 29, "synthetic_val_accuracy": 0.9327777777777778, "records_correct": 34, "records_total": 80, "channels_called_vacant": 2, "channels_total": 4, "negative_control_correct": 8, "negative_control_records": 20, "synthetic_val_noise_recall": 0.85, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 14, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 0, "uhf35_586MHz_occupied.npy": 20 } }, { "epoch": 30, "synthetic_val_accuracy": 0.9344444444444444, "records_correct": 37, "records_total": 80, "channels_called_vacant": 2, "channels_total": 4, "negative_control_correct": 8, "negative_control_records": 20, "synthetic_val_noise_recall": 0.886667, "per_channel_correct": { "uhf25_506MHz_occupied.npy": 17, "uhf32_562MHz_occupied.npy": 0, "uhf33_570MHz_occupied.npy": 0, "uhf35_586MHz_occupied.npy": 20 } } ] }, "checkpoints": [ { "label": "deployed-baseline", "members": [ "external/tvws-sensing/models/checkpoints/best.pt" ], "member_sha256": [ "8eca022f09b897a579ef74225881343949a1a4a53b880edabf82225aacbb34ee" ], "confidence_threshold": 0.7, "real_rf": { "records_correct": 40, "records_total": 80, "channels_wrongly_vacant": 2, "channels_total": 4, "mean_confidence_on_wrong_calls": 0.961457, "n_wrong_calls": 40, "per_channel": { "uhf25_506MHz_occupied.npy": { "uhf_channel": 25, "centre_mhz": 506, "excluded": false, "records": 20, "occupied": 20, "occupied_calibrated": 20, "class_counts": { "DVB-T": 20 }, "mean_confidence": 0.99928, "mean_confidence_on_vacant_calls": null, "mean_confidence_on_vacant_calls_calibrated": null }, "uhf32_562MHz_occupied.npy": { "uhf_channel": 32, "centre_mhz": 562, "excluded": false, "records": 20, "occupied": 0, "occupied_calibrated": 0, "class_counts": { "Noise": 20 }, "mean_confidence": 0.990322, "mean_confidence_on_vacant_calls": 0.990322, "mean_confidence_on_vacant_calls_calibrated": 0.686633 }, "uhf33_570MHz_occupied.npy": { "uhf_channel": 33, "centre_mhz": 570, "excluded": false, "records": 20, "occupied": 0, "occupied_calibrated": 0, "class_counts": { "Noise": 20 }, "mean_confidence": 0.932593, "mean_confidence_on_vacant_calls": 0.932593, "mean_confidence_on_vacant_calls_calibrated": 0.581446 }, "uhf35_586MHz_occupied.npy": { "uhf_channel": 35, "centre_mhz": 586, "excluded": false, "records": 20, "occupied": 20, "occupied_calibrated": 20, "class_counts": { "DVB-T": 20 }, "mean_confidence": 0.999993, "mean_confidence_on_vacant_calls": null, "mean_confidence_on_vacant_calls_calibrated": null }, "uhf39_618MHz_occupied.npy": { "uhf_channel": 39, "centre_mhz": 618, "excluded": true, "records": 20, "occupied": 0, "occupied_calibrated": 0, "class_counts": { "Noise": 10, "Unknown": 10 }, "mean_confidence": 0.936383, "mean_confidence_on_vacant_calls": 0.936383, "mean_confidence_on_vacant_calls_calibrated": 0.506889 } } }, "real_rf_calibrated": { "records_correct": 40, "channels_wrongly_vacant": 2, "mean_confidence_on_wrong_calls": 0.63404, "note": "Temperature is fitted on SYNTHETIC data. Applying it to real captures is an extrapolation and is reported as such." }, "calibration": { "temperature": 4.017584, "fitted_on": "synthetic calibration split (disjoint from test split)", "synthetic_test_accuracy": 0.586667, "synthetic_per_class_recall": { "ATSC": 0.013333, "DVB-T": 0.56, "ISDB-T": 0.426667, "Wireless Mic": 0.606667, "Noise": 0.94, "Unknown": 0.973333 }, "synthetic_false_occupied_rate_on_vacant_classes": 0.023333, "collapse_guard_note": "Every real capture used for selection is occupied, so a model that has lost the ability to say vacant scores perfectly on the real arm. NOISE recall and the synthetic false-occupied rate are the check that this has not happened.", "ece_synthetic_before": 0.31869, "ece_synthetic_after": 0.075864, "ece_synthetic_before_bins": [ { "lo": 0.0, "hi": 0.06666666666666667, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.06666666666666667, "hi": 0.13333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.13333333333333333, "hi": 0.2, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.2, "hi": 0.26666666666666666, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.26666666666666666, "hi": 0.3333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.3333, "hi": 0.4, "n": 3, "mean_conf": 0.3876, "accuracy": 0.0 }, { "lo": 0.4, "hi": 0.4667, "n": 6, "mean_conf": 0.4397, "accuracy": 0.3333 }, { "lo": 0.4667, "hi": 0.5333, "n": 18, "mean_conf": 0.5115, "accuracy": 0.1667 }, { "lo": 0.5333, "hi": 0.6, "n": 43, "mean_conf": 0.5722, "accuracy": 0.4419 }, { "lo": 0.6, "hi": 0.6667, "n": 34, "mean_conf": 0.6366, "accuracy": 0.2941 }, { "lo": 0.6667, "hi": 0.7333, "n": 42, "mean_conf": 0.7027, "accuracy": 0.2381 }, { "lo": 0.7333, "hi": 0.8, "n": 27, "mean_conf": 0.7597, "accuracy": 0.2963 }, { "lo": 0.8, "hi": 0.8667, "n": 38, "mean_conf": 0.8343, "accuracy": 0.4737 }, { "lo": 0.8667, "hi": 0.9333, "n": 79, "mean_conf": 0.9039, "accuracy": 0.3291 }, { "lo": 0.9333, "hi": 1.0, "n": 610, "mean_conf": 0.9876, "accuracy": 0.7082 } ], "ece_synthetic_after_bins": [ { "lo": 0.0, "hi": 0.06666666666666667, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.06666666666666667, "hi": 0.13333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.13333333333333333, "hi": 0.2, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.2, "hi": 0.2667, "n": 8, "mean_conf": 0.2414, "accuracy": 0.0 }, { "lo": 0.2667, "hi": 0.3333, "n": 32, "mean_conf": 0.3094, "accuracy": 0.0938 }, { "lo": 0.3333, "hi": 0.4, "n": 67, "mean_conf": 0.3678, "accuracy": 0.194 }, { "lo": 0.4, "hi": 0.4667, "n": 109, "mean_conf": 0.4301, "accuracy": 0.4312 }, { "lo": 0.4667, "hi": 0.5333, "n": 104, "mean_conf": 0.4988, "accuracy": 0.3558 }, { "lo": 0.5333, "hi": 0.6, "n": 140, "mean_conf": 0.5696, "accuracy": 0.5643 }, { "lo": 0.6, "hi": 0.6667, "n": 135, "mean_conf": 0.6308, "accuracy": 0.6667 }, { "lo": 0.6667, "hi": 0.7333, "n": 149, "mean_conf": 0.6996, "accuracy": 0.7987 }, { "lo": 0.7333, "hi": 0.8, "n": 74, "mean_conf": 0.7623, "accuracy": 0.7973 }, { "lo": 0.8, "hi": 0.8667, "n": 48, "mean_conf": 0.8324, "accuracy": 0.9792 }, { "lo": 0.8667, "hi": 0.9333, "n": 22, "mean_conf": 0.8984, "accuracy": 1.0 }, { "lo": 0.9333, "hi": 1.0, "n": 12, "mean_conf": 0.9505, "accuracy": 1.0 } ], "ece_real_before": 0.480547, "ece_real_after": 0.333514, "ece_real_n": 80, "ece_real_note": "Top-label ECE on the real records, correctness = argmax is DVB-T, the nearest label the 6-class head has for a DVB-T2 multiplex. n=80, so the bin counts are small and this number is noisy." }, "ood": { "score": "mahalanobis_min_class_penultimate512", "threshold_quantile": 0.95, "threshold": 436.732, "abstain_rate_synthetic_val": 0.05, "abstain_rate_synthetic_test": 0.048889, "abstain_rate_real_scored": 0.325, "abstain_rate_real_all": 0.26, "auroc_real_vs_synthetic_val": 0.783544, "median_mahalanobis_synthetic_val": 223.6611, "median_mahalanobis_real": 362.5661, "energy_comparison": { "threshold": -0.14, "abstain_rate_synthetic_val": 0.05, "abstain_rate_synthetic_test": 0.051111, "abstain_rate_real_scored": 0.0, "auroc_real_vs_synthetic_val": 0.3489 }, "channels_wrongly_vacant_after_abstain_gate": 1, "gate_note": "A channel counts as wrongly vacant here only if EVERY record is a confident vacant DECISION. Abstained records are not decisions; the controller's standing rule is that PAWS grant state is the authority and sensing is advisory, so an abstention means do not transmit." } }, { "label": "retrain-selected-on-real-epoch4", "members": [ "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/checkpoints-seed42/epoch-004.pt" ], "member_sha256": [ "35ce5bc9018d1e74896e7e0f07bcd6005c40d3bf0d4c071d7abd451d40efc882" ], "confidence_threshold": 0.7, "real_rf": { "records_correct": 50, "records_total": 80, "channels_wrongly_vacant": 1, "channels_total": 4, "mean_confidence_on_wrong_calls": 0.809479, "n_wrong_calls": 30, "per_channel": { "uhf25_506MHz_occupied.npy": { "uhf_channel": 25, "centre_mhz": 506, "excluded": false, "records": 20, "occupied": 13, "occupied_calibrated": 14, "class_counts": { "DVB-T": 19, "ISDB-T": 1 }, "mean_confidence": 0.754548, "mean_confidence_on_vacant_calls": 0.614499, "mean_confidence_on_vacant_calls_calibrated": 0.619212 }, "uhf32_562MHz_occupied.npy": { "uhf_channel": 32, "centre_mhz": 562, "excluded": false, "records": 20, "occupied": 0, "occupied_calibrated": 0, "class_counts": { "Noise": 20 }, "mean_confidence": 0.901316, "mean_confidence_on_vacant_calls": 0.901316, "mean_confidence_on_vacant_calls_calibrated": 0.930353 }, "uhf33_570MHz_occupied.npy": { "uhf_channel": 33, "centre_mhz": 570, "excluded": false, "records": 20, "occupied": 17, "occupied_calibrated": 17, "class_counts": { "ISDB-T": 20 }, "mean_confidence": 0.849443, "mean_confidence_on_vacant_calls": 0.652188, "mean_confidence_on_vacant_calls_calibrated": 0.680672 }, "uhf35_586MHz_occupied.npy": { "uhf_channel": 35, "centre_mhz": 586, "excluded": false, "records": 20, "occupied": 20, "occupied_calibrated": 20, "class_counts": { "DVB-T": 20 }, "mean_confidence": 0.912192, "mean_confidence_on_vacant_calls": null, "mean_confidence_on_vacant_calls_calibrated": null }, "uhf39_618MHz_occupied.npy": { "uhf_channel": 39, "centre_mhz": 618, "excluded": true, "records": 20, "occupied": 11, "occupied_calibrated": 13, "class_counts": { "ATSC": 13, "Noise": 3, "Unknown": 4 }, "mean_confidence": 0.793763, "mean_confidence_on_vacant_calls": 0.688144, "mean_confidence_on_vacant_calls_calibrated": 0.719286 } } }, "real_rf_calibrated": { "records_correct": 51, "channels_wrongly_vacant": 1, "mean_confidence_on_wrong_calls": 0.84015, "note": "Temperature is fitted on SYNTHETIC data. Applying it to real captures is an extrapolation and is reported as such." }, "calibration": { "temperature": 0.887687, "fitted_on": "synthetic calibration split (disjoint from test split)", "synthetic_test_accuracy": 0.934444, "synthetic_per_class_recall": { "ATSC": 0.966667, "DVB-T": 0.826667, "ISDB-T": 0.866667, "Wireless Mic": 1.0, "Noise": 0.946667, "Unknown": 1.0 }, "synthetic_false_occupied_rate_on_vacant_classes": 0.01, "collapse_guard_note": "Every real capture used for selection is occupied, so a model that has lost the ability to say vacant scores perfectly on the real arm. NOISE recall and the synthetic false-occupied rate are the check that this has not happened.", "ece_synthetic_before": 0.023767, "ece_synthetic_after": 0.020619, "ece_synthetic_before_bins": [ { "lo": 0.0, "hi": 0.06666666666666667, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.06666666666666667, "hi": 0.13333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.13333333333333333, "hi": 0.2, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.2, "hi": 0.2667, "n": 1, "mean_conf": 0.2555, "accuracy": 0.0 }, { "lo": 0.2667, "hi": 0.3333, "n": 1, "mean_conf": 0.3009, "accuracy": 0.0 }, { "lo": 0.3333333333333333, "hi": 0.4, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.4, "hi": 0.4667, "n": 3, "mean_conf": 0.4448, "accuracy": 0.3333 }, { "lo": 0.4667, "hi": 0.5333, "n": 26, "mean_conf": 0.5048, "accuracy": 0.6538 }, { "lo": 0.5333, "hi": 0.6, "n": 22, "mean_conf": 0.5604, "accuracy": 0.7273 }, { "lo": 0.6, "hi": 0.6667, "n": 28, "mean_conf": 0.6329, "accuracy": 0.6786 }, { "lo": 0.6667, "hi": 0.7333, "n": 31, "mean_conf": 0.7043, "accuracy": 0.7097 }, { "lo": 0.7333, "hi": 0.8, "n": 39, "mean_conf": 0.7703, "accuracy": 0.7179 }, { "lo": 0.8, "hi": 0.8667, "n": 53, "mean_conf": 0.8316, "accuracy": 0.8491 }, { "lo": 0.8667, "hi": 0.9333, "n": 67, "mean_conf": 0.8993, "accuracy": 0.9552 }, { "lo": 0.9333, "hi": 1.0, "n": 629, "mean_conf": 0.9924, "accuracy": 1.0 } ], "ece_synthetic_after_bins": [ { "lo": 0.0, "hi": 0.06666666666666667, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.06666666666666667, "hi": 0.13333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.13333333333333333, "hi": 0.2, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.2, "hi": 0.2667, "n": 1, "mean_conf": 0.2615, "accuracy": 0.0 }, { "lo": 0.2667, "hi": 0.3333, "n": 1, "mean_conf": 0.3072, "accuracy": 0.0 }, { "lo": 0.3333333333333333, "hi": 0.4, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.4, "hi": 0.4667, "n": 1, "mean_conf": 0.42, "accuracy": 0.0 }, { "lo": 0.4667, "hi": 0.5333, "n": 16, "mean_conf": 0.4984, "accuracy": 0.5 }, { "lo": 0.5333, "hi": 0.6, "n": 29, "mean_conf": 0.5603, "accuracy": 0.7586 }, { "lo": 0.6, "hi": 0.6667, "n": 19, "mean_conf": 0.6359, "accuracy": 0.6316 }, { "lo": 0.6667, "hi": 0.7333, "n": 26, "mean_conf": 0.6991, "accuracy": 0.7308 }, { "lo": 0.7333, "hi": 0.8, "n": 30, "mean_conf": 0.7638, "accuracy": 0.7 }, { "lo": 0.8, "hi": 0.8667, "n": 57, "mean_conf": 0.8334, "accuracy": 0.7719 }, { "lo": 0.8667, "hi": 0.9333, "n": 65, "mean_conf": 0.9051, "accuracy": 0.9385 }, { "lo": 0.9333, "hi": 1.0, "n": 655, "mean_conf": 0.9934, "accuracy": 0.9985 } ], "ece_real_before": 0.388213, "ece_real_after": 0.429857, "ece_real_n": 80, "ece_real_note": "Top-label ECE on the real records, correctness = argmax is DVB-T, the nearest label the 6-class head has for a DVB-T2 multiplex. n=80, so the bin counts are small and this number is noisy." }, "ood": { "score": "mahalanobis_min_class_penultimate512", "threshold_quantile": 0.95, "threshold": 650.012, "abstain_rate_synthetic_val": 0.05, "abstain_rate_synthetic_test": 0.051111, "abstain_rate_real_scored": 1.0, "abstain_rate_real_all": 0.93, "auroc_real_vs_synthetic_val": 0.983539, "median_mahalanobis_synthetic_val": 297.0427, "median_mahalanobis_real": 874.5811, "energy_comparison": { "threshold": -0.8251, "abstain_rate_synthetic_val": 0.05, "abstain_rate_synthetic_test": 0.034444, "abstain_rate_real_scored": 0.0, "auroc_real_vs_synthetic_val": 0.7287 }, "channels_wrongly_vacant_after_abstain_gate": 0, "gate_note": "A channel counts as wrongly vacant here only if EVERY record is a confident vacant DECISION. Abstained records are not decisions; the controller's standing rule is that PAWS grant state is the authority and sensing is advisory, so an abstention means do not transmit." } }, { "label": "retrain-argmax-synthetic-epoch26", "members": [ "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/checkpoints-seed42/epoch-026.pt" ], "member_sha256": [ "5fce1bc008ff4c028653b39e0e1e77f3cca19874fa36090655d6fb7ad7cea847" ], "confidence_threshold": 0.7, "real_rf": { "records_correct": 43, "records_total": 80, "channels_wrongly_vacant": 1, "channels_total": 4, "mean_confidence_on_wrong_calls": 0.831782, "n_wrong_calls": 37, "per_channel": { "uhf25_506MHz_occupied.npy": { "uhf_channel": 25, "centre_mhz": 506, "excluded": false, "records": 20, "occupied": 20, "occupied_calibrated": 14, "class_counts": { "DVB-T": 20 }, "mean_confidence": 0.911084, "mean_confidence_on_vacant_calls": null, "mean_confidence_on_vacant_calls_calibrated": 0.639997 }, "uhf32_562MHz_occupied.npy": { "uhf_channel": 32, "centre_mhz": 562, "excluded": false, "records": 20, "occupied": 0, "occupied_calibrated": 0, "class_counts": { "Noise": 19, "ATSC": 1 }, "mean_confidence": 0.928953, "mean_confidence_on_vacant_calls": 0.928953, "mean_confidence_on_vacant_calls_calibrated": 0.791083 }, "uhf33_570MHz_occupied.npy": { "uhf_channel": 33, "centre_mhz": 570, "excluded": false, "records": 20, "occupied": 3, "occupied_calibrated": 1, "class_counts": { "Noise": 15, "ISDB-T": 5 }, "mean_confidence": 0.732936, "mean_confidence_on_vacant_calls": 0.717464, "mean_confidence_on_vacant_calls_calibrated": 0.584246 }, "uhf35_586MHz_occupied.npy": { "uhf_channel": 35, "centre_mhz": 586, "excluded": false, "records": 20, "occupied": 20, "occupied_calibrated": 20, "class_counts": { "DVB-T": 20 }, "mean_confidence": 0.987647, "mean_confidence_on_vacant_calls": null, "mean_confidence_on_vacant_calls_calibrated": null }, "uhf39_618MHz_occupied.npy": { "uhf_channel": 39, "centre_mhz": 618, "excluded": true, "records": 20, "occupied": 11, "occupied_calibrated": 10, "class_counts": { "Noise": 6, "ATSC": 12, "Unknown": 2 }, "mean_confidence": 0.867307, "mean_confidence_on_vacant_calls": 0.7556, "mean_confidence_on_vacant_calls_calibrated": 0.641728 } } }, "real_rf_calibrated": { "records_correct": 35, "channels_wrongly_vacant": 1, "mean_confidence_on_wrong_calls": 0.683607, "note": "Temperature is fitted on SYNTHETIC data. Applying it to real captures is an extrapolation and is reported as such." }, "calibration": { "temperature": 1.923704, "fitted_on": "synthetic calibration split (disjoint from test split)", "synthetic_test_accuracy": 0.937778, "synthetic_per_class_recall": { "ATSC": 0.973333, "DVB-T": 0.886667, "ISDB-T": 0.926667, "Wireless Mic": 1.0, "Noise": 0.84, "Unknown": 1.0 }, "synthetic_false_occupied_rate_on_vacant_classes": 0.066667, "collapse_guard_note": "Every real capture used for selection is occupied, so a model that has lost the ability to say vacant scores perfectly on the real arm. NOISE recall and the synthetic false-occupied rate are the check that this has not happened.", "ece_synthetic_before": 0.04193, "ece_synthetic_after": 0.019158, "ece_synthetic_before_bins": [ { "lo": 0.0, "hi": 0.06666666666666667, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.06666666666666667, "hi": 0.13333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.13333333333333333, "hi": 0.2, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.2, "hi": 0.26666666666666666, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.26666666666666666, "hi": 0.3333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.3333333333333333, "hi": 0.4, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.4, "hi": 0.4667, "n": 2, "mean_conf": 0.4488, "accuracy": 0.5 }, { "lo": 0.4667, "hi": 0.5333, "n": 3, "mean_conf": 0.5012, "accuracy": 0.3333 }, { "lo": 0.5333, "hi": 0.6, "n": 5, "mean_conf": 0.5626, "accuracy": 0.6 }, { "lo": 0.6, "hi": 0.6667, "n": 12, "mean_conf": 0.6239, "accuracy": 0.6667 }, { "lo": 0.6667, "hi": 0.7333, "n": 11, "mean_conf": 0.701, "accuracy": 0.6364 }, { "lo": 0.7333, "hi": 0.8, "n": 7, "mean_conf": 0.771, "accuracy": 0.4286 }, { "lo": 0.8, "hi": 0.8667, "n": 16, "mean_conf": 0.8402, "accuracy": 0.5625 }, { "lo": 0.8667, "hi": 0.9333, "n": 10, "mean_conf": 0.9041, "accuracy": 0.9 }, { "lo": 0.9333, "hi": 1.0, "n": 834, "mean_conf": 0.9974, "accuracy": 0.9628 } ], "ece_synthetic_after_bins": [ { "lo": 0.0, "hi": 0.06666666666666667, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.06666666666666667, "hi": 0.13333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.13333333333333333, "hi": 0.2, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.2, "hi": 0.26666666666666666, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.26666666666666666, "hi": 0.3333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.3333, "hi": 0.4, "n": 3, "mean_conf": 0.3771, "accuracy": 0.6667 }, { "lo": 0.4, "hi": 0.4667, "n": 5, "mean_conf": 0.44, "accuracy": 0.6 }, { "lo": 0.4667, "hi": 0.5333, "n": 11, "mean_conf": 0.4977, "accuracy": 0.5455 }, { "lo": 0.5333, "hi": 0.6, "n": 20, "mean_conf": 0.568, "accuracy": 0.55 }, { "lo": 0.6, "hi": 0.6667, "n": 15, "mean_conf": 0.6313, "accuracy": 0.6 }, { "lo": 0.6667, "hi": 0.7333, "n": 10, "mean_conf": 0.7046, "accuracy": 0.9 }, { "lo": 0.7333, "hi": 0.8, "n": 17, "mean_conf": 0.7727, "accuracy": 0.6471 }, { "lo": 0.8, "hi": 0.8667, "n": 38, "mean_conf": 0.8344, "accuracy": 0.7368 }, { "lo": 0.8667, "hi": 0.9333, "n": 45, "mean_conf": 0.9054, "accuracy": 0.8 }, { "lo": 0.9333, "hi": 1.0, "n": 736, "mean_conf": 0.9882, "accuracy": 0.9905 } ], "ece_real_before": 0.390155, "ece_real_after": 0.266834, "ece_real_n": 80, "ece_real_note": "Top-label ECE on the real records, correctness = argmax is DVB-T, the nearest label the 6-class head has for a DVB-T2 multiplex. n=80, so the bin counts are small and this number is noisy." }, "ood": { "score": "mahalanobis_min_class_penultimate512", "threshold_quantile": 0.95, "threshold": 690.0525, "abstain_rate_synthetic_val": 0.05, "abstain_rate_synthetic_test": 0.042222, "abstain_rate_real_scored": 0.4, "abstain_rate_real_all": 0.44, "auroc_real_vs_synthetic_val": 0.922483, "median_mahalanobis_synthetic_val": 359.0688, "median_mahalanobis_real": 677.0945, "energy_comparison": { "threshold": -1.4565, "abstain_rate_synthetic_val": 0.05, "abstain_rate_synthetic_test": 0.044444, "abstain_rate_real_scored": 0.0625, "auroc_real_vs_synthetic_val": 0.854944 }, "channels_wrongly_vacant_after_abstain_gate": 0, "gate_note": "A channel counts as wrongly vacant here only if EVERY record is a confident vacant DECISION. Abstained records are not decisions; the controller's standing rule is that PAWS grant state is the authority and sensing is advisory, so an abstention means do not transmit." } }, { "label": "retrain-ensemble-last5-SELECTION-FREE", "members": [ "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/checkpoints-seed42/epoch-026.pt", "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/checkpoints-seed42/epoch-027.pt", "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/checkpoints-seed42/epoch-028.pt", "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/checkpoints-seed42/epoch-029.pt", "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/checkpoints-seed42/epoch-030.pt" ], "member_sha256": [ "5fce1bc008ff4c028653b39e0e1e77f3cca19874fa36090655d6fb7ad7cea847", "b3474d85e0ff4104791bcffb767588f930af7a9cde1542790b99280713af8e7f", "8c38825275c49f593a8b2c8c679257fe7f88f5fe8481fd215a69f6985e1c550c", "a6e29a1e7511bbc710d5e34e7acd30156828861c92298445918da13a3203e205", "575c3d91b6d36d835b26656dc63a7db84578a12f550a53bd729c40ecb0d0a624" ], "confidence_threshold": 0.7, "real_rf": { "records_correct": 37, "records_total": 80, "channels_wrongly_vacant": 2, "channels_total": 4, "mean_confidence_on_wrong_calls": 0.87228, "n_wrong_calls": 43, "per_channel": { "uhf25_506MHz_occupied.npy": { "uhf_channel": 25, "centre_mhz": 506, "excluded": false, "records": 20, "occupied": 17, "occupied_calibrated": 12, "class_counts": { "DVB-T": 20 }, "mean_confidence": 0.871445, "mean_confidence_on_vacant_calls": 0.634798, "mean_confidence_on_vacant_calls_calibrated": 0.599088 }, "uhf32_562MHz_occupied.npy": { "uhf_channel": 32, "centre_mhz": 562, "excluded": false, "records": 20, "occupied": 0, "occupied_calibrated": 0, "class_counts": { "Noise": 20 }, "mean_confidence": 0.95982, "mean_confidence_on_vacant_calls": 0.95982, "mean_confidence_on_vacant_calls_calibrated": 0.850212 }, "uhf33_570MHz_occupied.npy": { "uhf_channel": 33, "centre_mhz": 570, "excluded": false, "records": 20, "occupied": 0, "occupied_calibrated": 0, "class_counts": { "Noise": 18, "ISDB-T": 2 }, "mean_confidence": 0.820362, "mean_confidence_on_vacant_calls": 0.820362, "mean_confidence_on_vacant_calls_calibrated": 0.674168 }, "uhf35_586MHz_occupied.npy": { "uhf_channel": 35, "centre_mhz": 586, "excluded": false, "records": 20, "occupied": 20, "occupied_calibrated": 20, "class_counts": { "DVB-T": 20 }, "mean_confidence": 0.975946, "mean_confidence_on_vacant_calls": null, "mean_confidence_on_vacant_calls_calibrated": null }, "uhf39_618MHz_occupied.npy": { "uhf_channel": 39, "centre_mhz": 618, "excluded": true, "records": 20, "occupied": 12, "occupied_calibrated": 11, "class_counts": { "ATSC": 13, "Noise": 5, "Unknown": 2 }, "mean_confidence": 0.86847, "mean_confidence_on_vacant_calls": 0.744865, "mean_confidence_on_vacant_calls_calibrated": 0.627128 } } }, "real_rf_calibrated": { "records_correct": 32, "channels_wrongly_vacant": 2, "mean_confidence_on_wrong_calls": 0.735006, "note": "Temperature is fitted on SYNTHETIC data. Applying it to real captures is an extrapolation and is reported as such." }, "calibration": { "temperature": 1.888762, "fitted_on": "synthetic calibration split (disjoint from test split)", "synthetic_test_accuracy": 0.938889, "synthetic_per_class_recall": { "ATSC": 0.98, "DVB-T": 0.86, "ISDB-T": 0.92, "Wireless Mic": 1.0, "Noise": 0.873333, "Unknown": 1.0 }, "synthetic_false_occupied_rate_on_vacant_classes": 0.05, "collapse_guard_note": "Every real capture used for selection is occupied, so a model that has lost the ability to say vacant scores perfectly on the real arm. NOISE recall and the synthetic false-occupied rate are the check that this has not happened.", "ece_synthetic_before": 0.042079, "ece_synthetic_after": 0.019353, "ece_synthetic_before_bins": [ { "lo": 0.0, "hi": 0.06666666666666667, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.06666666666666667, "hi": 0.13333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.13333333333333333, "hi": 0.2, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.2, "hi": 0.26666666666666666, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.26666666666666666, "hi": 0.3333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.3333333333333333, "hi": 0.4, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.4, "hi": 0.4666666666666667, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.4667, "hi": 0.5333, "n": 5, "mean_conf": 0.5163, "accuracy": 0.2 }, { "lo": 0.5333, "hi": 0.6, "n": 7, "mean_conf": 0.5703, "accuracy": 0.7143 }, { "lo": 0.6, "hi": 0.6667, "n": 13, "mean_conf": 0.6359, "accuracy": 0.6923 }, { "lo": 0.6667, "hi": 0.7333, "n": 9, "mean_conf": 0.7034, "accuracy": 0.5556 }, { "lo": 0.7333, "hi": 0.8, "n": 8, "mean_conf": 0.77, "accuracy": 0.5 }, { "lo": 0.8, "hi": 0.8667, "n": 14, "mean_conf": 0.8351, "accuracy": 0.5714 }, { "lo": 0.8667, "hi": 0.9333, "n": 20, "mean_conf": 0.9095, "accuracy": 0.8 }, { "lo": 0.9333, "hi": 1.0, "n": 824, "mean_conf": 0.9978, "accuracy": 0.9672 } ], "ece_synthetic_after_bins": [ { "lo": 0.0, "hi": 0.06666666666666667, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.06666666666666667, "hi": 0.13333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.13333333333333333, "hi": 0.2, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.2, "hi": 0.26666666666666666, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.26666666666666666, "hi": 0.3333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.3333333333333333, "hi": 0.4, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.4, "hi": 0.4667, "n": 5, "mean_conf": 0.4483, "accuracy": 0.8 }, { "lo": 0.4667, "hi": 0.5333, "n": 18, "mean_conf": 0.4951, "accuracy": 0.5556 }, { "lo": 0.5333, "hi": 0.6, "n": 17, "mean_conf": 0.5653, "accuracy": 0.4706 }, { "lo": 0.6, "hi": 0.6667, "n": 6, "mean_conf": 0.6155, "accuracy": 0.3333 }, { "lo": 0.6667, "hi": 0.7333, "n": 21, "mean_conf": 0.7009, "accuracy": 0.8095 }, { "lo": 0.7333, "hi": 0.8, "n": 24, "mean_conf": 0.768, "accuracy": 0.7083 }, { "lo": 0.8, "hi": 0.8667, "n": 19, "mean_conf": 0.8375, "accuracy": 0.6842 }, { "lo": 0.8667, "hi": 0.9333, "n": 49, "mean_conf": 0.9017, "accuracy": 0.8367 }, { "lo": 0.9333, "hi": 1.0, "n": 741, "mean_conf": 0.9872, "accuracy": 0.9892 } ], "ece_real_before": 0.406893, "ece_real_after": 0.284479, "ece_real_n": 80, "ece_real_note": "Top-label ECE on the real records, correctness = argmax is DVB-T, the nearest label the 6-class head has for a DVB-T2 multiplex. n=80, so the bin counts are small and this number is noisy." }, "ood": { "score": "mahalanobis_min_class_penultimate512", "threshold_quantile": 0.95, "threshold": 666.271, "abstain_rate_synthetic_val": 0.05, "abstain_rate_synthetic_test": 0.047778, "abstain_rate_real_scored": 0.45, "abstain_rate_real_all": 0.49, "auroc_real_vs_synthetic_val": 0.932583, "median_mahalanobis_synthetic_val": 358.9312, "median_mahalanobis_real": 663.1259, "energy_comparison": { "threshold": -1.6342, "abstain_rate_synthetic_val": 0.05, "abstain_rate_synthetic_test": 0.037778, "abstain_rate_real_scored": 0.0375, "auroc_real_vs_synthetic_val": 0.857667 }, "channels_wrongly_vacant_after_abstain_gate": 0, "gate_note": "A channel counts as wrongly vacant here only if EVERY record is a confident vacant DECISION. Abstained records are not decisions; the controller's standing rule is that PAWS grant state is the authority and sensing is advisory, so an abstention means do not transmit." } }, { "label": "retrain-ensemble-stride5-SELECTION-FREE", "members": [ "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/checkpoints-seed42/epoch-005.pt", "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/checkpoints-seed42/epoch-010.pt", "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/checkpoints-seed42/epoch-015.pt", "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/checkpoints-seed42/epoch-020.pt", "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/checkpoints-seed42/epoch-025.pt", "~/Ulap/model-artifacts/tvws-retrain-2026-08-03/checkpoints-seed42/epoch-030.pt" ], "member_sha256": [ "d8d49a76d30a0b58ab8b9dc025ccf8e28b14fb8d32a4967744c63c06b287b589", "3a42ab3eacf431d3d7058e276518963fbb6a280055a59ccc0bf48650a04d133f", "02097a2a0e2def28f33e848fc66f3ac0e500080ec291719204d73a9d67c52b07", "220deabd98036876e74c7711bec63e767e91506ae5f1255c95925a5fc87b9fa7", "38a576e5664338bc07c08e8848acc1184f4f366c6840e508fa6d2e6aefa02cda", "575c3d91b6d36d835b26656dc63a7db84578a12f550a53bd729c40ecb0d0a624" ], "confidence_threshold": 0.7, "real_rf": { "records_correct": 41, "records_total": 80, "channels_wrongly_vacant": 1, "channels_total": 4, "mean_confidence_on_wrong_calls": 0.808273, "n_wrong_calls": 39, "per_channel": { "uhf25_506MHz_occupied.npy": { "uhf_channel": 25, "centre_mhz": 506, "excluded": false, "records": 20, "occupied": 20, "occupied_calibrated": 20, "class_counts": { "DVB-T": 20 }, "mean_confidence": 0.928272, "mean_confidence_on_vacant_calls": null, "mean_confidence_on_vacant_calls_calibrated": null }, "uhf32_562MHz_occupied.npy": { "uhf_channel": 32, "centre_mhz": 562, "excluded": false, "records": 20, "occupied": 0, "occupied_calibrated": 0, "class_counts": { "Noise": 20 }, "mean_confidence": 0.938398, "mean_confidence_on_vacant_calls": 0.938398, "mean_confidence_on_vacant_calls_calibrated": 0.946786 }, "uhf33_570MHz_occupied.npy": { "uhf_channel": 33, "centre_mhz": 570, "excluded": false, "records": 20, "occupied": 1, "occupied_calibrated": 1, "class_counts": { "Noise": 17, "ISDB-T": 3 }, "mean_confidence": 0.673067, "mean_confidence_on_vacant_calls": 0.671299, "mean_confidence_on_vacant_calls_calibrated": 0.688793 }, "uhf35_586MHz_occupied.npy": { "uhf_channel": 35, "centre_mhz": 586, "excluded": false, "records": 20, "occupied": 20, "occupied_calibrated": 20, "class_counts": { "DVB-T": 20 }, "mean_confidence": 0.976437, "mean_confidence_on_vacant_calls": null, "mean_confidence_on_vacant_calls_calibrated": null }, "uhf39_618MHz_occupied.npy": { "uhf_channel": 39, "centre_mhz": 618, "excluded": true, "records": 20, "occupied": 13, "occupied_calibrated": 13, "class_counts": { "ATSC": 17, "Noise": 1, "Unknown": 2 }, "mean_confidence": 0.826524, "mean_confidence_on_vacant_calls": 0.623606, "mean_confidence_on_vacant_calls_calibrated": 0.63359 } } }, "real_rf_calibrated": { "records_correct": 41, "channels_wrongly_vacant": 1, "mean_confidence_on_wrong_calls": 0.821097, "note": "Temperature is fitted on SYNTHETIC data. Applying it to real captures is an extrapolation and is reported as such." }, "calibration": { "temperature": 0.934852, "fitted_on": "synthetic calibration split (disjoint from test split)", "synthetic_test_accuracy": 0.942222, "synthetic_per_class_recall": { "ATSC": 0.986667, "DVB-T": 0.873333, "ISDB-T": 0.913333, "Wireless Mic": 1.0, "Noise": 0.88, "Unknown": 1.0 }, "synthetic_false_occupied_rate_on_vacant_classes": 0.026667, "collapse_guard_note": "Every real capture used for selection is occupied, so a model that has lost the ability to say vacant scores perfectly on the real arm. NOISE recall and the synthetic false-occupied rate are the check that this has not happened.", "ece_synthetic_before": 0.015492, "ece_synthetic_after": 0.017074, "ece_synthetic_before_bins": [ { "lo": 0.0, "hi": 0.06666666666666667, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.06666666666666667, "hi": 0.13333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.13333333333333333, "hi": 0.2, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.2, "hi": 0.26666666666666666, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.26666666666666666, "hi": 0.3333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.3333, "hi": 0.4, "n": 2, "mean_conf": 0.36, "accuracy": 0.5 }, { "lo": 0.4, "hi": 0.4667, "n": 5, "mean_conf": 0.4258, "accuracy": 0.4 }, { "lo": 0.4667, "hi": 0.5333, "n": 17, "mean_conf": 0.5087, "accuracy": 0.5294 }, { "lo": 0.5333, "hi": 0.6, "n": 16, "mean_conf": 0.5607, "accuracy": 0.5 }, { "lo": 0.6, "hi": 0.6667, "n": 20, "mean_conf": 0.6348, "accuracy": 0.75 }, { "lo": 0.6667, "hi": 0.7333, "n": 25, "mean_conf": 0.7061, "accuracy": 0.84 }, { "lo": 0.7333, "hi": 0.8, "n": 38, "mean_conf": 0.7684, "accuracy": 0.7895 }, { "lo": 0.8, "hi": 0.8667, "n": 52, "mean_conf": 0.8346, "accuracy": 0.8077 }, { "lo": 0.8667, "hi": 0.9333, "n": 51, "mean_conf": 0.9021, "accuracy": 0.9412 }, { "lo": 0.9333, "hi": 1.0, "n": 674, "mean_conf": 0.9935, "accuracy": 0.997 } ], "ece_synthetic_after_bins": [ { "lo": 0.0, "hi": 0.06666666666666667, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.06666666666666667, "hi": 0.13333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.13333333333333333, "hi": 0.2, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.2, "hi": 0.26666666666666666, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.26666666666666666, "hi": 0.3333333333333333, "n": 0, "mean_conf": null, "accuracy": null }, { "lo": 0.3333, "hi": 0.4, "n": 2, "mean_conf": 0.3655, "accuracy": 0.5 }, { "lo": 0.4, "hi": 0.4667, "n": 5, "mean_conf": 0.4358, "accuracy": 0.4 }, { "lo": 0.4667, "hi": 0.5333, "n": 12, "mean_conf": 0.5108, "accuracy": 0.4167 }, { "lo": 0.5333, "hi": 0.6, "n": 19, "mean_conf": 0.5635, "accuracy": 0.5263 }, { "lo": 0.6, "hi": 0.6667, "n": 16, "mean_conf": 0.6437, "accuracy": 0.6875 }, { "lo": 0.6667, "hi": 0.7333, "n": 21, "mean_conf": 0.7059, "accuracy": 0.8571 }, { "lo": 0.7333, "hi": 0.8, "n": 34, "mean_conf": 0.7713, "accuracy": 0.8235 }, { "lo": 0.8, "hi": 0.8667, "n": 52, "mean_conf": 0.8375, "accuracy": 0.8654 }, { "lo": 0.8667, "hi": 0.9333, "n": 52, "mean_conf": 0.9029, "accuracy": 0.8269 }, { "lo": 0.9333, "hi": 1.0, "n": 687, "mean_conf": 0.9941, "accuracy": 0.9971 } ], "ece_real_before": 0.379043, "ece_real_after": 0.389753, "ece_real_n": 80, "ece_real_note": "Top-label ECE on the real records, correctness = argmax is DVB-T, the nearest label the 6-class head has for a DVB-T2 multiplex. n=80, so the bin counts are small and this number is noisy." }, "ood": { "score": "mahalanobis_min_class_penultimate512", "threshold_quantile": 0.95, "threshold": 647.7759, "abstain_rate_synthetic_val": 0.05, "abstain_rate_synthetic_test": 0.034444, "abstain_rate_real_scored": 0.7125, "abstain_rate_real_all": 0.71, "auroc_real_vs_synthetic_val": 0.963678, "median_mahalanobis_synthetic_val": 362.4488, "median_mahalanobis_real": 725.5859, "energy_comparison": { "threshold": -1.0395, "abstain_rate_synthetic_val": 0.05, "abstain_rate_synthetic_test": 0.036667, "abstain_rate_real_scored": 0.0, "auroc_real_vs_synthetic_val": 0.792628 }, "channels_wrongly_vacant_after_abstain_gate": 0, "gate_note": "A channel counts as wrongly vacant here only if EVERY record is a confident vacant DECISION. Abstained records are not decisions; the controller's standing rule is that PAWS grant state is the authority and sensing is advisory, so an abstention means do not transmit." } } ], "honesty": { "selection_set_is_not_a_test_set": "The four verified multiplexes were used to SELECT the shipped epoch. Their score for that checkpoint is a selection score and is optimistically biased. The leave-one-channel-out total is the closest to unbiased that four channels allow: each held-out channel is scored by a checkpoint selected without it.", "what_no_real_number_here_can_prove": "One site, one afternoon, one mismatched 860-930 MHz antenna at 470-700 MHz, four channels, every capture occupied. There are ZERO real vacant captures, so no false-positive rate on real RF can be measured at all, and a model answering 'occupied' unconditionally scores 80/80 here. Nothing here generalises to another site or another receiver.", "calibration_temperature_is_synthetic": "Temperature is fitted on held-out SYNTHETIC data because that is the only labelled data with enough records to fit anything. Applying it to real captures is an extrapolation. The real-RF ECE is reported before and after so the extrapolation is visible." } }