code_public/radiacode
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix: CsI(Tl) non-linear response correction + detector calibration overhaul

Browse Source 
Root cause of Am-241 misidentification: the Radiacode 103's CsI(Tl) crystal
shifts low-energy peaks upward (59.5 keV → 71.6 keV for Am-241) due to
non-proportional scintillation response. The model was trained on theoretical
peak positions and couldn't match the shifted real peaks.

Changes:
- Add inverse CsI(Tl) non-linear correction to inference pipeline
  (radiacode_monitor.py, web/config.py, test_detection.py)
  E_apparent = E_true * (1 + 0.37 * exp(-E_true/100))
  Corrects channel mapping so peaks appear at theoretical energies
- Fix energy calibration: DetectorConfig now uses E = 0.33 + 2.97*ch
  with 1023 channels, matching the real detector (was energy_min=20,
  skip_first_channel=True, different channel width)
- Add K-escape peaks for CsI(Tl) iodine X-ray escape (E - 28.5 keV)
- Add asymmetric peak shapes for low-energy tails (< 200 keV)
- Add log1p normalization in dataset and inference (replaces max-norm)
- Add background-subtracted training mode (subtract_background flag)
- Add low-signal augmentation (0.01-5 Bq activities, 30-300s durations)
- Update docker-compose.yml: batch_size=32, duration=30-300s,
  CSI_NONLINEAR_ALPHA/BETA env vars for detect and web
- Web dashboard: apply CsI correction to displayed spectra
- Various UI fixes (Chart.js width, zoom/pan, isotope lines)

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Jacquin Antoine
2026-05-21 17:35:22 +02:00
parent 3b4446b181
commit 0847a3fc80
21 changed files with 913 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
web/app/bg_calibration.py
View File

@ -4,19 +4,15 @@ CsI(Tl) detector response continuum calibration for Radiacode 103.
Models ONLY the detector's noise continuum. Photopeaks from environmental
isotopes depend on measurement location and are NOT part of this model.
Uses two approaches:
1. Spline-based: non-parametric, automatically fits any shape
2. Parametric: for the /fit endpoint (comparison with measured data)
The spline approach is preferred — it uses scipy's smoothing spline with
Generalized Cross-Validation to automatically find the right smoothness,
after iterative peak subtraction.
Uses iterative peak subtraction followed by Gaussian smoothing to produce
a clean continuum shape. This approach tracks the measured background closely
at all energies, unlike log-space splines which collapse in low-signal regions.
"""
import json
import numpy as np
from pathlib import Path
from scipy.interpolate import make_smoothing_spline
from scipy.ndimage import gaussian_filter1d
from scipy.signal import savgol_filter
from app.config import ENERGY_OFFSET, ENERGY_SLOPE, NUM_CHANNELS
@ -31,77 +27,76 @@ def _sigma_keV(E):
return max(12.0, 23.6 * np.sqrt(max(E, 1.0) / 662.0))
def _smooth(y):
window = min(51, len(y) // 10 * 2 + 1)
if window < 5:
window = 5
return savgol_filter(y, window_length=window, polyorder=3)
def _sigma_ch(E_keV):
fwhm_keV = 0.08 * E_keV * (E_keV / 662.0) ** 0.5
sigma_keV = fwhm_keV / 2.355
return max(sigma_keV / ENERGY_SLOPE, 2.0)
def _subtract_peaks(energy_axis, smoothed_cps):
"""Iteratively estimate and subtract photopeak contributions."""
continuum = smoothed_cps.copy()
peak_amplitudes = []
def _subtract_peaks(energy_axis, spectrum):
"""Remove known isotope photopeaks from spectrum."""
continuum = spectrum.copy()
channels = np.arange(len(spectrum), dtype=np.float64)
for line_energy, _ in PHOTOPEAK_LINES:
sig = _sigma_keV(line_energy)
idx = np.argmin(np.abs(energy_axis - line_energy))
n_sigma = max(int(2 * sig / 2.97), 3)
off_lo = continuum[max(0, idx - 3 * n_sigma):max(1, idx - n_sigma)]
off_hi = continuum[min(len(continuum), idx + n_sigma):min(len(continuum), idx + 3 * n_sigma)]
off_peak = np.concatenate([off_lo, off_hi])
local_bg = np.median(off_peak) if len(off_peak) > 0 else 0
idx = int(np.argmin(np.abs(energy_axis - line_energy)))
sig = _sigma_ch(line_energy)
far = int(5 * sig) + 3
lo_start = max(0, idx - far - int(3 * sig))
lo_end = max(0, idx - far)
hi_start = min(len(spectrum), idx + far)
hi_end = min(len(spectrum), idx + far + int(3 * sig))
baseline_regions = []
if lo_end > lo_start:
baseline_regions.append(continuum[lo_start:lo_end])
if hi_end > hi_start:
baseline_regions.append(continuum[hi_start:hi_end])
if not baseline_regions:
continue
local_bg = float(np.median(np.concatenate(baseline_regions)))
peak_height = continuum[idx] - local_bg
if peak_height > 0:
amplitude = peak_height * sig * np.sqrt(2 * np.pi)
gaussian = amplitude * np.exp(-0.5 * ((energy_axis - line_energy) / sig) ** 2) / (sig * np.sqrt(2 * np.pi))
gaussian = peak_height * np.exp(-0.5 * ((channels - idx) / sig) ** 2)
continuum -= gaussian
continuum = np.maximum(continuum, 0)
peak_amplitudes.append({"energy_keV": line_energy, "amplitude": float(max(0, peak_height) * sig * np.sqrt(2 * np.pi)) if peak_height > 0 else 0.0})
return continuum, peak_amplitudes
return np.maximum(continuum, 0), [{"energy_keV": e, "amplitude": 0.0} for e, _ in PHOTOPEAK_LINES]
def calibrate_spline(measured_cps, energy_axis):
"""
Fit a smoothing spline to the peak-subtracted continuum.
Fit continuum using peak subtraction + Gaussian smoothing.
Uses scipy's make_smoothing_spline with GCV (Generalized Cross-Validation)
to automatically find the optimal smoothing parameter.
Returns a dict with the fitted spline evaluated at all channels.
Uses scipy's gaussian_filter1d after iterative peak subtraction,
producing a smooth continuum that tracks the measured background
closely at all energies including the high-energy tail.
"""
E = energy_axis
y_smooth = _smooth(measured_cps)
continuum, peak_amplitudes = _subtract_peaks(E, y_smooth)
# Step 1: Smooth to reduce statistical noise
window = min(51, len(measured_cps) // 10 * 2 + 1)
if window < 5:
window = 5
y_smooth = savgol_filter(measured_cps, window_length=window, polyorder=3)
# Ensure positive values for spline fitting
continuum = np.maximum(continuum, 0)
# Step 2: Subtract known photopeaks
continuum, peak_amplitudes = _subtract_peaks(energy_axis, y_smooth)
# Use log-space for better fit at low-signal high-energy region
# Add small offset to avoid log(0)
offset = continuum[continuum > 0].min() * 0.1 if (continuum > 0).any() else 1e-6
log_continuum = np.log(continuum + offset)
# Fit smoothing spline in log-space (GCV auto-selects lambda)
try:
spline = make_smoothing_spline(E, log_continuum)
log_fit = spline(E)
# Convert back from log-space
fit_cps = np.exp(log_fit) - offset
fit_cps = np.maximum(fit_cps, 0)
except Exception as e:
return {"error": str(e)}
# Step 3: Gaussian smooth for final continuum shape
sigma = max(15, len(continuum) // 60)
continuum_smooth = gaussian_filter1d(continuum, sigma=sigma)
continuum_smooth = np.maximum(continuum_smooth, 0)
# Quality metrics
residuals = continuum - fit_cps
residuals = continuum - continuum_smooth
ss_res = np.sum(residuals ** 2)
ss_tot = np.sum((continuum - continuum.mean()) ** 2)
r_squared = 1.0 - ss_res / ss_tot if ss_tot > 0 else 0
return {
"continuum_cps": fit_cps,
"continuum_cps": continuum_smooth,
"peak_amplitudes": peak_amplitudes,
"r_squared": float(r_squared),
"residuals_rms": float(np.sqrt(np.mean(residuals ** 2))),
@ -109,29 +104,24 @@ def calibrate_spline(measured_cps, energy_axis):
def calibrate_background(measured_cps, energy_axis):
"""
Fit the continuum model using smoothing spline.
Returns both spline-based fit and parameters for the /fit endpoint.
"""
"""Fit the continuum model using peak subtraction + Gaussian smoothing."""
result = calibrate_spline(measured_cps, energy_axis)
if "error" in result:
return result
# The spline result is the continuum CPS array
return {
"params": {}, # Non-parametric model, no params
"params": {},
"continuum_cps": result["continuum_cps"],
"peak_amplitudes": result["peak_amplitudes"],
"r_squared": result["r_squared"],
"residuals_rms": result["residuals_rms"],
"method": "smoothing_spline_gcv",
"method": "peak_subtract_gaussian",
}
def build_calibrated_continuum(energy_axis, total_counts, params):
"""Build the continuum from calibrated parameters."""
if "continuum_cps" in params:
# Spline-based: already have the CPS array
cps = np.array(params["continuum_cps"])
if cps.sum() > 0:
return cps * total_counts / cps.sum()
@ -151,13 +141,20 @@ def load_or_calibrate():
if _cached_result is not None:
return _cached_result
# Try loading from cache file first (read-only volume is fine for reads)
if _CALIBRATION_PATH.exists():
try:
with open(_CALIBRATION_PATH) as f:
_cached_result = json.load(f)
return _cached_result
cached = json.load(f)
# Invalidate if method changed
if cached.get("method") != "peak_subtract_gaussian":
cached = None
except Exception:
pass
cached = None
if cached and "continuum_cps" in cached:
_cached_result = cached
return _cached_result
from app.config import BACKGROUND_PATH, BACKGROUND_SNAPSHOT_PATH
@ -199,10 +196,14 @@ def load_or_calibrate():
"r_squared": result["r_squared"],
}
_CALIBRATION_PATH.parent.mkdir(parents=True, exist_ok=True)
tmp = _CALIBRATION_PATH.with_suffix(".tmp")
with open(tmp, "w") as f:
json.dump(_cached_result, f, indent=2)
tmp.replace(_CALIBRATION_PATH)
# Write cache if volume is writable (may fail on read-only mounts)
try:
_CALIBRATION_PATH.parent.mkdir(parents=True, exist_ok=True)
tmp = _CALIBRATION_PATH.with_suffix(".tmp")
with open(tmp, "w") as f:
json.dump(_cached_result, f, indent=2)
tmp.replace(_CALIBRATION_PATH)
except OSError:
pass # Read-only volume — in-memory cache is sufficient
return _cached_result

44
web/app/config.py
View File

@ -1,4 +1,5 @@
import os
import numpy as np
from pathlib import Path
STATE_PATH = Path(os.environ.get("STATE_PATH", "/data/monitor_state.json"))
@ -11,8 +12,47 @@ ISOTOPE_INDEX_PATH = Path(os.environ.get("ISOTOPE_INDEX_PATH", "/models/vega_iso
ENERGY_OFFSET = float(os.environ.get("ENERGY_CALIBRATION_OFFSET", "0.33"))
ENERGY_SLOPE = float(os.environ.get("ENERGY_CALIBRATION_SLOPE", "2.97"))
NUM_CHANNELS = 1023 # Last channel (1023) is overflow bin, excluded from display
ENERGY_MIN = 30.0 # keV - detector lower limit
ENERGY_MAX = 3000.0 # keV - detector upper limit (3 MeV)
# CsI(Tl) non-linear response correction parameters
# Matches the detector's non-proportional scintillation response
CSI_NONLINEAR_ALPHA = float(os.environ.get("CSI_NONLINEAR_ALPHA", "0.37"))
CSI_NONLINEAR_BETA = float(os.environ.get("CSI_NONLINEAR_BETA", "100.0"))
def correct_csi_nonlinear(spectrum, num_channels=1023):
"""Apply inverse CsI(Tl) non-linear response correction.
Remaps spectrum channels so peaks appear at their theoretical energy
positions, correcting for the detector's non-proportional scintillation
response that shifts low-energy peaks upward.
"""
alpha = CSI_NONLINEAR_ALPHA
beta = CSI_NONLINEAR_BETA
output_channels = np.arange(num_channels, dtype=np.float64)
e_true = ENERGY_OFFSET + ENERGY_SLOPE * output_channels
e_apparent = e_true * (1 + alpha * np.exp(-e_true / beta))
source_channels = (e_apparent - ENERGY_OFFSET) / ENERGY_SLOPE
source_channels = np.clip(source_channels, 0, num_channels - 1.001)
lower = np.floor(source_channels).astype(int)
upper = np.minimum(lower + 1, num_channels - 1)
frac = source_channels - lower
return spectrum[lower] * (1 - frac) + spectrum[upper] * frac
def energy_axis():
"""Generate energy axis in keV from channel numbers."""
return [round(ENERGY_OFFSET + ENERGY_SLOPE * i, 2) for i in range(NUM_CHANNELS)]
"""Generate energy axis in keV from channel numbers, clipped to detector range."""
axis = [round(ENERGY_OFFSET + ENERGY_SLOPE * i, 2) for i in range(NUM_CHANNELS)]
return [e for e in axis if ENERGY_MIN <= e <= ENERGY_MAX]
def energy_mask():
"""Return boolean mask of channels within detector energy range."""
return [ENERGY_MIN <= ENERGY_OFFSET + ENERGY_SLOPE * i <= ENERGY_MAX for i in range(NUM_CHANNELS)]
def clip_to_range(arr):
"""Clip array to detector energy range using energy mask."""
mask = energy_mask()
return [arr[i] for i in range(len(arr)) if mask[i]]

33
web/app/routers/background.py
View File

@ -1,6 +1,6 @@
import json
from fastapi import APIRouter, HTTPException
from app.config import BACKGROUND_SNAPSHOT_PATH, BACKGROUND_PATH, energy_axis, NUM_CHANNELS, ENERGY_OFFSET, ENERGY_SLOPE
from app.config import BACKGROUND_SNAPSHOT_PATH, BACKGROUND_PATH, energy_axis, NUM_CHANNELS, ENERGY_OFFSET, ENERGY_SLOPE, clip_to_range
from app.theoretical_bg import generate_continuum_only
from app.noise import extract_continuum
import numpy as np
@ -25,8 +25,9 @@ def _load_reference():
return None
try:
bg_data = np.load(str(BACKGROUND_PATH), allow_pickle=True).item()
raw_counts = [round(float(c), 1) for c in bg_data["counts"][:NUM_CHANNELS]]
return {
"counts": [round(float(c), 1) for c in bg_data["counts"][:NUM_CHANNELS]],
"counts": clip_to_range(raw_counts),
"live_time_s": round(float(bg_data["duration"]), 1),
}
except Exception:
@ -54,11 +55,12 @@ async def get_background_spectrum():
"""Live background spectrum (from snapshot) with energy axis."""
snapshot = _load_snapshot()
live_time = snapshot.get("live_time_s", 0)
raw_spectrum = snapshot.get("spectrum", [0] * 1024)[:NUM_CHANNELS]
return {
"channels": list(range(NUM_CHANNELS)),
"channels": clip_to_range(list(range(NUM_CHANNELS))),
"energy_kev": energy_axis(),
"counts": snapshot.get("spectrum", [0] * 1024)[:NUM_CHANNELS],
"counts": clip_to_range(raw_spectrum),
"live_time_s": live_time,
"cps": snapshot.get("cps", 0),
"top_peaks": snapshot.get("top_peaks", []),
@ -74,7 +76,7 @@ async def get_background_reference():
raise HTTPException(status_code=404, detail="No 24h reference background available")
return {
"channels": list(range(NUM_CHANNELS)),
"channels": clip_to_range(list(range(NUM_CHANNELS))),
"energy_kev": energy_axis(),
"counts": ref["counts"],
"live_time_s": ref["live_time_s"],
@ -84,7 +86,10 @@ async def get_background_reference():
@router.get("/continuum")
async def get_continuum(cps: float = 6.0, live_time_s: float = 3600.0):
"""CsI(Tl) detector response continuum only (no photopeaks, no noise)."""
return generate_continuum_only(cps=cps, live_time_s=live_time_s)
raw = generate_continuum_only(cps=cps, live_time_s=live_time_s)
raw["energy_kev"] = clip_to_range(raw["energy_kev"])
raw["counts"] = clip_to_range(raw["counts"])
return raw
@router.get("/fit")
@ -132,10 +137,14 @@ async def fit_background():
# Build fitted curve at same scale as measured
fitted_counts = build_calibrated_continuum(e_axis, measured_counts.sum(), result)
e_list = [round(float(E), 2) for E in e_axis]
m_list = [round(float(c), 1) for c in measured_counts]
f_list = [round(float(c), 1) for c in fitted_counts]
return {
"energy_kev": [round(float(E), 2) for E in e_axis],
"measured_counts": [round(float(c), 1) for c in measured_counts],
"fitted_counts": [round(float(c), 1) for c in fitted_counts],
"energy_kev": clip_to_range(e_list),
"measured_counts": clip_to_range(m_list),
"fitted_counts": clip_to_range(f_list),
"method": result.get("method", "spline"),
"r_squared": result["r_squared"],
"residuals_rms": result["residuals_rms"],
@ -174,7 +183,9 @@ async def get_background_noise():
e_axis = ENERGY_OFFSET + ENERGY_SLOPE * channels
continuum = extract_continuum(counts, energy_axis=e_axis)
e_list = [round(float(E), 2) for E in e_axis]
c_list = [round(float(c), 1) for c in continuum]
return {
"energy_kev": [round(float(E), 2) for E in e_axis],
"counts": [round(float(c), 1) for c in continuum],
"energy_kev": clip_to_range(e_list),
"counts": clip_to_range(c_list),
}

25
web/app/routers/spectrum.py
View File

@ -1,6 +1,7 @@
import json
from fastapi import APIRouter, HTTPException
from app.config import STATE_PATH, BACKGROUND_PATH, energy_axis, NUM_CHANNELS
from app.config import (STATE_PATH, BACKGROUND_PATH, energy_axis, NUM_CHANNELS,
clip_to_range, correct_csi_nonlinear)
import numpy as np
router = APIRouter()
@ -8,7 +9,7 @@ router = APIRouter()
@router.get("/current")
async def get_current_spectrum():
"""Current accumulated spectrum with energy axis."""
"""Current accumulated spectrum with energy axis (CsI-corrected)."""
if not STATE_PATH.exists():
raise HTTPException(status_code=503, detail="Monitor not started yet")
@ -18,6 +19,9 @@ async def get_current_spectrum():
except (json.JSONDecodeError, OSError):
raise HTTPException(status_code=503, detail="Monitor state file corrupt")
raw_counts = state.get("counts", [0] * 1024)[:NUM_CHANNELS]
# Apply CsI correction so peaks appear at theoretical energy positions
corrected_counts = correct_csi_nonlinear(np.array(raw_counts, dtype=np.float64))
return {
"timestamp": state.get("timestamp", ""),
"connected": state.get("connected", False),
@ -27,15 +31,15 @@ async def get_current_spectrum():
"total_counts": state.get("total_counts", 0),
"background_subtracted": state.get("background_subtracted", False),
"isotopes_detected": state.get("isotopes_detected", []),
"channels": list(range(NUM_CHANNELS)),
"channels": clip_to_range(list(range(NUM_CHANNELS))),
"energy_kev": energy_axis(),
"counts": state.get("counts", [0] * 1024)[:NUM_CHANNELS],
"counts": clip_to_range([round(float(c), 1) for c in corrected_counts]),
}
@router.get("/difference")
async def get_difference_spectrum():
"""Background-subtracted spectrum (net signal)."""
"""Background-subtracted spectrum (net signal, CsI-corrected)."""
if not STATE_PATH.exists():
raise HTTPException(status_code=503, detail="Monitor not started yet")
@ -59,18 +63,21 @@ async def get_difference_spectrum():
bg_live_time = float(bg_data["duration"])
bg_rate = bg_counts / bg_live_time
net_rate = np.clip(rate - bg_rate, 0, None)
net_counts = net_rate * live_time
# Apply CsI correction to net spectrum
corrected_net = correct_csi_nonlinear(net_rate)
net_counts = corrected_net * live_time
bg_available = True
else:
net_counts = counts
bg_available = False
net_counts_list = [round(float(c), 1) for c in net_counts]
return {
"timestamp": state.get("timestamp", ""),
"cumulated_live_time_s": live_time,
"background_subtracted": bg_available,
"channels": list(range(NUM_CHANNELS)),
"channels": clip_to_range(list(range(NUM_CHANNELS))),
"energy_kev": energy_axis(),
"counts": [round(float(c), 1) for c in net_counts],
"raw_counts": state.get("counts", [])[:NUM_CHANNELS],
"counts": clip_to_range(net_counts_list),
"raw_counts": clip_to_range(state.get("counts", [])[:NUM_CHANNELS]),
}

4
web/app/theoretical_bg.py
View File

@ -30,7 +30,7 @@ def generate_continuum_only(cps: float = 6.0, live_time_s: float = 3600.0):
energy_axis = ENERGY_OFFSET + ENERGY_SLOPE * channels
total_counts = cps * live_time_s
# Try calibrated spline first
# Try calibrated spline first (fits measured background)
continuum_cps = _get_continuum_cps()
if continuum_cps is not None and len(continuum_cps) == NUM_CHANNELS:
@ -39,7 +39,7 @@ def generate_continuum_only(cps: float = 6.0, live_time_s: float = 3600.0):
if continuum.sum() > 0:
continuum *= total_counts / continuum.sum()
else:
# Fallback: simple parametric model
# Fallback: parametric model
continuum = _fallback_continuum(energy_axis, total_counts)
return {