Fix: CsI(Tl) non-linear response correction + detector calibration overhaul
Root cause of Am-241 misidentification: the Radiacode 103's CsI(Tl) crystal shifts low-energy peaks upward (59.5 keV → 71.6 keV for Am-241) due to non-proportional scintillation response. The model was trained on theoretical peak positions and couldn't match the shifted real peaks. Changes: - Add inverse CsI(Tl) non-linear correction to inference pipeline (radiacode_monitor.py, web/config.py, test_detection.py) E_apparent = E_true * (1 + 0.37 * exp(-E_true/100)) Corrects channel mapping so peaks appear at theoretical energies - Fix energy calibration: DetectorConfig now uses E = 0.33 + 2.97*ch with 1023 channels, matching the real detector (was energy_min=20, skip_first_channel=True, different channel width) - Add K-escape peaks for CsI(Tl) iodine X-ray escape (E - 28.5 keV) - Add asymmetric peak shapes for low-energy tails (< 200 keV) - Add log1p normalization in dataset and inference (replaces max-norm) - Add background-subtracted training mode (subtract_background flag) - Add low-signal augmentation (0.01-5 Bq activities, 30-300s durations) - Update docker-compose.yml: batch_size=32, duration=30-300s, CSI_NONLINEAR_ALPHA/BETA env vars for detect and web - Web dashboard: apply CsI correction to displayed spectra - Various UI fixes (Chart.js width, zoom/pan, isotope lines) Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Jacquin Antoine
@ -4,19 +4,15 @@ CsI(Tl) detector response continuum calibration for Radiacode 103.
|
||||
Models ONLY the detector's noise continuum. Photopeaks from environmental
|
||||
isotopes depend on measurement location and are NOT part of this model.
|
||||
|
||||
Uses two approaches:
|
||||
1. Spline-based: non-parametric, automatically fits any shape
|
||||
2. Parametric: for the /fit endpoint (comparison with measured data)
|
||||
|
||||
The spline approach is preferred — it uses scipy's smoothing spline with
|
||||
Generalized Cross-Validation to automatically find the right smoothness,
|
||||
after iterative peak subtraction.
|
||||
Uses iterative peak subtraction followed by Gaussian smoothing to produce
|
||||
a clean continuum shape. This approach tracks the measured background closely
|
||||
at all energies, unlike log-space splines which collapse in low-signal regions.
|
||||
"""
|
||||
|
||||
import json
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
from scipy.interpolate import make_smoothing_spline
|
||||
from scipy.ndimage import gaussian_filter1d
|
||||
from scipy.signal import savgol_filter
|
||||
from app.config import ENERGY_OFFSET, ENERGY_SLOPE, NUM_CHANNELS
|
||||
|
||||
@ -31,77 +27,76 @@ def _sigma_keV(E):
|
||||
return max(12.0, 23.6 * np.sqrt(max(E, 1.0) / 662.0))
|
||||
|
||||
|
||||
def _smooth(y):
|
||||
window = min(51, len(y) // 10 * 2 + 1)
|
||||
if window < 5:
|
||||
window = 5
|
||||
return savgol_filter(y, window_length=window, polyorder=3)
|
||||
def _sigma_ch(E_keV):
|
||||
fwhm_keV = 0.08 * E_keV * (E_keV / 662.0) ** 0.5
|
||||
sigma_keV = fwhm_keV / 2.355
|
||||
return max(sigma_keV / ENERGY_SLOPE, 2.0)
|
||||
|
||||
|
||||
def _subtract_peaks(energy_axis, smoothed_cps):
|
||||
"""Iteratively estimate and subtract photopeak contributions."""
|
||||
continuum = smoothed_cps.copy()
|
||||
peak_amplitudes = []
|
||||
def _subtract_peaks(energy_axis, spectrum):
|
||||
"""Remove known isotope photopeaks from spectrum."""
|
||||
continuum = spectrum.copy()
|
||||
channels = np.arange(len(spectrum), dtype=np.float64)
|
||||
|
||||
for line_energy, _ in PHOTOPEAK_LINES:
|
||||
sig = _sigma_keV(line_energy)
|
||||
idx = np.argmin(np.abs(energy_axis - line_energy))
|
||||
n_sigma = max(int(2 * sig / 2.97), 3)
|
||||
off_lo = continuum[max(0, idx - 3 * n_sigma):max(1, idx - n_sigma)]
|
||||
off_hi = continuum[min(len(continuum), idx + n_sigma):min(len(continuum), idx + 3 * n_sigma)]
|
||||
off_peak = np.concatenate([off_lo, off_hi])
|
||||
local_bg = np.median(off_peak) if len(off_peak) > 0 else 0
|
||||
idx = int(np.argmin(np.abs(energy_axis - line_energy)))
|
||||
sig = _sigma_ch(line_energy)
|
||||
far = int(5 * sig) + 3
|
||||
|
||||
lo_start = max(0, idx - far - int(3 * sig))
|
||||
lo_end = max(0, idx - far)
|
||||
hi_start = min(len(spectrum), idx + far)
|
||||
hi_end = min(len(spectrum), idx + far + int(3 * sig))
|
||||
|
||||
baseline_regions = []
|
||||
if lo_end > lo_start:
|
||||
baseline_regions.append(continuum[lo_start:lo_end])
|
||||
if hi_end > hi_start:
|
||||
baseline_regions.append(continuum[hi_start:hi_end])
|
||||
|
||||
if not baseline_regions:
|
||||
continue
|
||||
|
||||
local_bg = float(np.median(np.concatenate(baseline_regions)))
|
||||
peak_height = continuum[idx] - local_bg
|
||||
|
||||
if peak_height > 0:
|
||||
amplitude = peak_height * sig * np.sqrt(2 * np.pi)
|
||||
gaussian = amplitude * np.exp(-0.5 * ((energy_axis - line_energy) / sig) ** 2) / (sig * np.sqrt(2 * np.pi))
|
||||
gaussian = peak_height * np.exp(-0.5 * ((channels - idx) / sig) ** 2)
|
||||
continuum -= gaussian
|
||||
continuum = np.maximum(continuum, 0)
|
||||
peak_amplitudes.append({"energy_keV": line_energy, "amplitude": float(max(0, peak_height) * sig * np.sqrt(2 * np.pi)) if peak_height > 0 else 0.0})
|
||||
|
||||
return continuum, peak_amplitudes
|
||||
return np.maximum(continuum, 0), [{"energy_keV": e, "amplitude": 0.0} for e, _ in PHOTOPEAK_LINES]
|
||||
|
||||
|
||||
def calibrate_spline(measured_cps, energy_axis):
|
||||
"""
|
||||
Fit a smoothing spline to the peak-subtracted continuum.
|
||||
Fit continuum using peak subtraction + Gaussian smoothing.
|
||||
|
||||
Uses scipy's make_smoothing_spline with GCV (Generalized Cross-Validation)
|
||||
to automatically find the optimal smoothing parameter.
|
||||
|
||||
Returns a dict with the fitted spline evaluated at all channels.
|
||||
Uses scipy's gaussian_filter1d after iterative peak subtraction,
|
||||
producing a smooth continuum that tracks the measured background
|
||||
closely at all energies including the high-energy tail.
|
||||
"""
|
||||
E = energy_axis
|
||||
y_smooth = _smooth(measured_cps)
|
||||
continuum, peak_amplitudes = _subtract_peaks(E, y_smooth)
|
||||
# Step 1: Smooth to reduce statistical noise
|
||||
window = min(51, len(measured_cps) // 10 * 2 + 1)
|
||||
if window < 5:
|
||||
window = 5
|
||||
y_smooth = savgol_filter(measured_cps, window_length=window, polyorder=3)
|
||||
|
||||
# Ensure positive values for spline fitting
|
||||
continuum = np.maximum(continuum, 0)
|
||||
# Step 2: Subtract known photopeaks
|
||||
continuum, peak_amplitudes = _subtract_peaks(energy_axis, y_smooth)
|
||||
|
||||
# Use log-space for better fit at low-signal high-energy region
|
||||
# Add small offset to avoid log(0)
|
||||
offset = continuum[continuum > 0].min() * 0.1 if (continuum > 0).any() else 1e-6
|
||||
log_continuum = np.log(continuum + offset)
|
||||
|
||||
# Fit smoothing spline in log-space (GCV auto-selects lambda)
|
||||
try:
|
||||
spline = make_smoothing_spline(E, log_continuum)
|
||||
log_fit = spline(E)
|
||||
# Convert back from log-space
|
||||
fit_cps = np.exp(log_fit) - offset
|
||||
fit_cps = np.maximum(fit_cps, 0)
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
# Step 3: Gaussian smooth for final continuum shape
|
||||
sigma = max(15, len(continuum) // 60)
|
||||
continuum_smooth = gaussian_filter1d(continuum, sigma=sigma)
|
||||
continuum_smooth = np.maximum(continuum_smooth, 0)
|
||||
|
||||
# Quality metrics
|
||||
residuals = continuum - fit_cps
|
||||
residuals = continuum - continuum_smooth
|
||||
ss_res = np.sum(residuals ** 2)
|
||||
ss_tot = np.sum((continuum - continuum.mean()) ** 2)
|
||||
r_squared = 1.0 - ss_res / ss_tot if ss_tot > 0 else 0
|
||||
|
||||
return {
|
||||
"continuum_cps": fit_cps,
|
||||
"continuum_cps": continuum_smooth,
|
||||
"peak_amplitudes": peak_amplitudes,
|
||||
"r_squared": float(r_squared),
|
||||
"residuals_rms": float(np.sqrt(np.mean(residuals ** 2))),
|
||||
@ -109,29 +104,24 @@ def calibrate_spline(measured_cps, energy_axis):
|
||||
|
||||
|
||||
def calibrate_background(measured_cps, energy_axis):
|
||||
"""
|
||||
Fit the continuum model using smoothing spline.
|
||||
Returns both spline-based fit and parameters for the /fit endpoint.
|
||||
"""
|
||||
"""Fit the continuum model using peak subtraction + Gaussian smoothing."""
|
||||
result = calibrate_spline(measured_cps, energy_axis)
|
||||
if "error" in result:
|
||||
return result
|
||||
|
||||
# The spline result is the continuum CPS array
|
||||
return {
|
||||
"params": {}, # Non-parametric model, no params
|
||||
"params": {},
|
||||
"continuum_cps": result["continuum_cps"],
|
||||
"peak_amplitudes": result["peak_amplitudes"],
|
||||
"r_squared": result["r_squared"],
|
||||
"residuals_rms": result["residuals_rms"],
|
||||
"method": "smoothing_spline_gcv",
|
||||
"method": "peak_subtract_gaussian",
|
||||
}
|
||||
|
||||
|
||||
def build_calibrated_continuum(energy_axis, total_counts, params):
|
||||
"""Build the continuum from calibrated parameters."""
|
||||
if "continuum_cps" in params:
|
||||
# Spline-based: already have the CPS array
|
||||
cps = np.array(params["continuum_cps"])
|
||||
if cps.sum() > 0:
|
||||
return cps * total_counts / cps.sum()
|
||||
@ -151,13 +141,20 @@ def load_or_calibrate():
|
||||
if _cached_result is not None:
|
||||
return _cached_result
|
||||
|
||||
# Try loading from cache file first (read-only volume is fine for reads)
|
||||
if _CALIBRATION_PATH.exists():
|
||||
try:
|
||||
with open(_CALIBRATION_PATH) as f:
|
||||
_cached_result = json.load(f)
|
||||
return _cached_result
|
||||
cached = json.load(f)
|
||||
# Invalidate if method changed
|
||||
if cached.get("method") != "peak_subtract_gaussian":
|
||||
cached = None
|
||||
except Exception:
|
||||
pass
|
||||
cached = None
|
||||
|
||||
if cached and "continuum_cps" in cached:
|
||||
_cached_result = cached
|
||||
return _cached_result
|
||||
|
||||
from app.config import BACKGROUND_PATH, BACKGROUND_SNAPSHOT_PATH
|
||||
|
||||
@ -199,10 +196,14 @@ def load_or_calibrate():
|
||||
"r_squared": result["r_squared"],
|
||||
}
|
||||
|
||||
_CALIBRATION_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = _CALIBRATION_PATH.with_suffix(".tmp")
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(_cached_result, f, indent=2)
|
||||
tmp.replace(_CALIBRATION_PATH)
|
||||
# Write cache if volume is writable (may fail on read-only mounts)
|
||||
try:
|
||||
_CALIBRATION_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = _CALIBRATION_PATH.with_suffix(".tmp")
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(_cached_result, f, indent=2)
|
||||
tmp.replace(_CALIBRATION_PATH)
|
||||
except OSError:
|
||||
pass # Read-only volume — in-memory cache is sufficient
|
||||
|
||||
return _cached_result
|
||||
Reference in New Issue
Block a user