from __future__ import annotations
import warnings
from collections.abc import Sequence
from dataclasses import dataclass
import numpy as np
from astropy.table import Table, vstack
from scipy import fft, ndimage, optimize, stats
from .data import sky_direction
from .matchedfilter import TEMPLATE_KEYS, SearchResult, continuum
from .model import C_KMS, LINE_FLUX
from .utils import BLOCK_BYTES
def _jackknife(result: SearchResult) -> SearchResult:
if result.jackknife is None:
raise ValueError(
"the search has no jackknife to measure the noise on; run it with MatchedFilter.run(jackknife=True)"
)
return result.jackknife
[docs]
def jackknife_spread(result: SearchResult) -> np.ndarray:
"""Robust spread of the jackknife S/N per template, ``(n_template,)``: 1 under H0."""
snr = _jackknife(result).snr
return np.array([stats.median_abs_deviation(x[np.isfinite(x)], scale="normal") for x in np.moveaxis(snr, 2, 0)])
[docs]
@dataclass(frozen=True, eq=False)
class ResponseCorrelation:
"""Correlation function of the matched-filter response under the null (see :func:`response_correlation`)."""
spatial: np.ndarray
lags_ra: np.ndarray
lags_dec: np.ndarray
spectral: np.ndarray
templates: np.ndarray
hwhm_major: np.ndarray
hwhm_minor: np.ndarray
pa: np.ndarray
spectral_hwhm: np.ndarray
spectral_hwhm_kms: np.ndarray
sidelobe: np.ndarray
def _channel_width(freqs: np.ndarray) -> float:
"""Median channel width in Hz; infinite for a single channel."""
return float(np.median(np.abs(np.diff(freqs)))) if len(freqs) > 1 else np.inf
def _lattice(axis, max_lag: float) -> tuple[float, int]:
"""Signed step of a uniform position axis in arcsec and how many steps fit within ``max_lag``."""
a = np.asarray(axis, dtype=float)
if a.size < 2:
return 1.0, 0
step = (a[-1] - a[0]) / (a.size - 1)
if not np.allclose(np.diff(a), step, rtol=1e-6, atol=0):
raise ValueError("the search positions are not on a uniform lattice")
return step, min(int(max_lag / abs(step) + 1e-9), a.size - 1)
def _windows(freqs: np.ndarray) -> list[slice]:
"""Runs of uniformly spaced channels: the spectral windows, as far as the frequencies tell."""
step = np.diff(freqs)
if step.size < 2:
return [slice(None)]
alike = np.isclose(step[1:], step[:-1], rtol=1e-6, atol=0)
regular = np.append(alike, False) | np.insert(alike, 0, False)
edges = [0, *(np.flatnonzero(~regular) + 1).tolist(), len(freqs)]
return [slice(a, b) for a, b in zip(edges[:-1], edges[1:], strict=True)]
def _lag_sums(x: np.ndarray, axes: tuple, lags: tuple) -> np.ndarray:
"""``sum_p x(p) x(p + l)`` for every lag ``|l_k| <= lags[k]`` along ``axes`` (sorted), summed over the other axes.
The result is centred on lag 0. The transform is zero-padded to ``n + lag``, so no pair wraps around.
"""
shape = [fft.next_fast_len(x.shape[a] + n) for a, n in zip(axes, lags, strict=True)]
f = fft.rfftn(x, s=shape, axes=axes)
power = np.sum(f.real**2 + f.imag**2, axis=tuple(a for a in range(x.ndim) if a not in axes))
return fft.irfftn(power, s=shape)[np.ix_(*(np.arange(-n, n + 1) for n in lags))]
def _autocorrelation(x: np.ndarray, m: np.ndarray, axes: tuple, lags: tuple, chunks: list[slice]) -> np.ndarray:
"""NaN-aware autocorrelation of ``x`` (zero where the mask ``m`` is 0), normalised to 1 at lag 0."""
num, den = (sum(_lag_sums(a[:, :, c], axes, lags) for c in chunks) for a in (x, m))
rho = np.where(den > 0.5, num / np.where(den > 0.5, den, 1.0), np.nan)
return rho / rho[tuple(lags)]
def _half_power(rho: np.ndarray, lags_ra: np.ndarray, lags_dec: np.ndarray) -> tuple[float, float, float]:
"""The ellipse of the main lobe's ``rho >= 0.5`` pixels, each a cell of the lattice (its own variance
``step^2 / 12``), so a lobe one pixel wide (short baselines cut) keeps a width."""
labels = ndimage.label(rho >= 0.5)[0]
i, j = np.nonzero(labels == labels[len(lags_ra) // 2, len(lags_dec) // 2])
cell = np.diag([np.ptp(lags) / max(lags.size - 1, 1) for lags in (lags_ra, lags_dec)]) ** 2 / 12
variance, vectors = np.linalg.eigh(np.cov(np.stack([lags_ra[i], lags_dec[j]]), bias=True) + cell)
minor, major = 2 * np.sqrt(np.maximum(variance, 0.0))
return major, minor, np.degrees(np.arctan2(*vectors[:, 1])) % 180
def _sidelobe(rho: np.ndarray) -> float:
"""Largest ``|rho|`` outside the main lobe, the connected region ``rho > 0`` around lag 0."""
labels = ndimage.label(rho > 0)[0]
outside = rho[(labels != labels[rho.shape[0] // 2, rho.shape[1] // 2]) & np.isfinite(rho)]
return float(np.abs(outside).max()) if outside.size else 0.0
def _half_width(rho: np.ndarray) -> float:
"""Lag of the first 0.5 crossing of ``rho`` (lags 0, 1, ...), linearly interpolated; NaN if it stays above."""
below = np.flatnonzero(rho < 0.5)
if not below.size:
return np.nan
k = below[0]
return k - 1 + (rho[k - 1] - 0.5) / (rho[k - 1] - rho[k])
[docs]
def response_correlation(
result: SearchResult, max_lag: float = 30.0, max_lag_chan: int | None = None
) -> ResponseCorrelation:
"""Measure the correlation function of the response under the null on the search's jackknife."""
snr = _jackknife(result).snr
n_dra, n_ddec, n_template, n_chan = snr.shape
(step_ra, lag_ra), (step_dec, lag_dec) = (_lattice(result.axes[k], max_lag) for k in ("dra", "ddec"))
nu = np.median(result.freqs)
dnu = _channel_width(result.freqs)
if max_lag_chan is None:
max_lag_chan = int(np.ceil(4 * nu * max(result.axes["width"]) / C_KMS / dnu))
max_lag_chan = min(max_lag_chan, n_chan - 1)
# standardised copy, (n_template, n_dra, n_ddec, n_chan), zero where masked
z = np.array(np.moveaxis(snr, 2, 0), dtype=float, order="C")
finite = np.isfinite(z)
for zt, ft in zip(z, finite, strict=True):
zt -= zt.mean(where=ft)
zt /= zt.std(where=ft)
zt[~ft] = 0.0
per_chunk = max(1, BLOCK_BYTES // (32 * n_dra * n_ddec))
chunks = [slice(c, c + per_chunk) for c in range(0, n_chan, per_chunk)]
windows = _windows(result.freqs)
spatial, spectral = [], []
for x, ft in zip(z, finite, strict=True):
m = ft.astype(float)
rho = _autocorrelation(x, m, (0, 1), (lag_ra, lag_dec), chunks)
# index lags follow the axes' order; flip a descending axis so the lags run east and north
spatial.append(rho[:: int(np.sign(step_ra)), :: int(np.sign(step_dec))])
spectral.append(_autocorrelation(x, m, (2,), (max_lag_chan,), windows)[max_lag_chan:])
flat, flat_mask = z.reshape(n_template, -1), finite.reshape(n_template, -1)
pairs = np.array([[np.count_nonzero(a & b) for b in flat_mask] for a in flat_mask])
templates = flat @ flat.T / np.maximum(pairs, 1)
lags_ra = abs(step_ra) * np.arange(-lag_ra, lag_ra + 1)
lags_dec = abs(step_dec) * np.arange(-lag_dec, lag_dec + 1)
major, minor, pa = np.array([_half_power(rho, lags_ra, lags_dec) for rho in spatial]).T
spectral_hwhm = np.array([_half_width(rho) for rho in spectral])
return ResponseCorrelation(
spatial=np.array(spatial),
lags_ra=lags_ra,
lags_dec=lags_dec,
spectral=np.array(spectral),
templates=templates,
hwhm_major=major,
hwhm_minor=minor,
pa=pa,
spectral_hwhm=spectral_hwhm,
spectral_hwhm_kms=spectral_hwhm * dnu / nu * C_KMS,
sidelobe=np.array([_sidelobe(rho) for rho in spatial]),
)
[docs]
@dataclass(frozen=True)
class Group:
"""Local maxima of the S/N merged into one candidate (see :func:`groups`)."""
i: int
j: int
t: int
k: int
snr: float
n_peaks: int
def _maxima(snr: np.ndarray, floor: float) -> tuple[np.ndarray, ...]:
"""Indices (i, j, t, k) and S/N of the 3x3x3 local maxima >= ``floor`` of the best-template S/N, brightest first."""
best = np.fmax.reduce(snr, axis=2)
best = np.where(np.isnan(best), -np.inf, best)
i, j, k = np.nonzero((best == ndimage.maximum_filter(best, size=3, mode="nearest")) & (best >= floor))
order = np.argsort(-best[i, j, k], kind="stable")
i, j, k = i[order], j[order], k[order]
t = np.nanargmax(snr[i, j, :, k], axis=1) if i.size else np.zeros(0, dtype=int)
return i, j, t, k, best[i, j, k]
[docs]
def groups(snr: np.ndarray, result: SearchResult, nc: ResponseCorrelation, floor: float = 4.0) -> list[Group]:
"""Group the local maxima of ``snr`` into candidates, one per source, brightest first."""
from scipy.spatial import cKDTree
i, j, t, k, value = _maxima(snr, floor)
x, y = np.asarray(result.axes["dra"])[i], np.asarray(result.axes["ddec"])[j]
line = result.freqs.size > 1 # else a continuum: one channel
chan = result.freqs[k] / _channel_width(result.freqs) if line else np.zeros(i.size)
pa = np.radians(nc.pa)
# every ellipsoid lies within sqrt(2) of its centre in units of the widest one
scale = np.array([np.max(nc.hwhm_major)] * 2 + [np.max(nc.spectral_hwhm) if line else 1.0])
tree = cKDTree(np.c_[x, y, chan] / scale)
owner = np.full(i.size, -1)
found = []
for r in range(i.size):
if owner[r] >= 0:
continue
near = np.asarray(tree.query_ball_point(np.r_[x[r], y[r], chan[r]] / scale, np.sqrt(2) * (1 + 1e-9)))
free, tr = np.sort(near[owner[near] < 0]), t[r]
dx, dy, dc = x[free] - x[r], y[free] - y[r], chan[free] - chan[r]
major = dx * np.sin(pa[tr]) + dy * np.cos(pa[tr])
minor = dx * np.cos(pa[tr]) - dy * np.sin(pa[tr])
spectral = (dc / nc.spectral_hwhm[tr]) ** 2 if line else 0.0
r2 = (major / nc.hwhm_major[tr]) ** 2 + (minor / nc.hwhm_minor[tr]) ** 2 + spectral
inside = free[r2 <= 1]
owner[inside] = r
found.append(Group(int(i[r]), int(j[r]), int(tr), int(k[r]), float(value[r]), inside.size))
return found
#: How the likelihood ratio and the fidelity combine into ``detected``: "and" (both) or "or" (either).
DETECTION_RULE = "and"
#: Default reliability tests: the S/N floor of the groups, the likelihood ratio ``k``, the fidelity and the rule.
RELIABILITY = {"floor": 4.0, "k": 3.0, "fidelity_min": 0.6, "rule": DETECTION_RULE}
#: Lowest S/N of the noise peaks the noise model is fitted to (the data's groups start at ``floor``).
NOISE_FLOOR = 2.5
#: Fewest noise peaks above :data:`NOISE_FLOOR` to fit the model to; fewer, and nothing is detected.
MIN_NOISE = 10
#: Width of the Gaussian prior on the exponent ``m``, about the field's dimension minus one.
EXPONENT_PRIOR = 1.0
[docs]
def peak_exponent(snr_noise, dim: int, floor: float = NOISE_FLOOR) -> float:
"""Exponent ``m`` of the noise peak counts ``N(>= u) ~ u^m exp(-u^2 / 2)`` above ``floor``, by unbinned MAP.
The expected number of peaks of a smooth Gaussian field of dimension ``dim`` above ``u`` (its Euler
characteristic, Vio & Andreani 2021) is a polynomial in ``u`` times ``exp(-u^2 / 2)``, of degree
``dim - 1``: the prior's centre, :data:`EXPONENT_PRIOR` wide, so a few noise peaks (a continuum)
cannot drive ``m`` far off and many (a line search) set it. NaN, with a warning, below
:data:`MIN_NOISE` peaks.
"""
u = np.asarray(snr_noise, dtype=float)
u = u[u >= floor]
if u.size < MIN_NOISE:
warnings.warn(f"only {u.size} noise peaks above S/N {floor:g}, too few to fit the noise", stacklevel=2)
return np.nan
x = np.log(u / floor)
def nll(m): # density -dS/du of the survival S(u) = (u / floor)^m exp(-(u^2 - floor^2) / 2)
return -np.sum(m * x + np.log(u - m / u)) + ((m - dim + 1) / EXPONENT_PRIOR) ** 2 / 2
return float(optimize.minimize_scalar(nll, bounds=(-5.0, floor**2 * (1 - 1e-6)), method="bounded").x)
[docs]
def noise_above(u, n: int, m: float, floor: float = NOISE_FLOOR) -> np.ndarray:
"""Expected noise peaks at or above ``u`` of a kernel with ``n`` noise peaks above ``floor``."""
u = np.asarray(u, dtype=float)
return n * np.exp(m * np.log(u / floor) - (u**2 - floor**2) / 2)
[docs]
def likelihood(snr_data, noise) -> np.ndarray:
"""``N_data(>= gamma) / N_noise(>= gamma)`` at every data S/N (Vio & Andreani 2021); ``noise(u)`` the
expected noise peaks at or above ``u`` (:func:`noise_above`), infinite where it is zero."""
snr_data = np.asarray(snr_data, dtype=float)
n_data = snr_data.size - np.searchsorted(np.sort(snr_data), snr_data)
with np.errstate(divide="ignore"):
return n_data / noise(snr_data)
[docs]
def fidelity(snr_data, snr_noise, noise, floor: float, step: float = 0.25) -> tuple[np.ndarray, dict]:
"""Fidelity ``1 - N_noise / N_data`` per S/N bin from ``floor``, ``N_noise`` from the noise model ``noise``.
Every data group gets its bin's fidelity. The bins hold the counts of the data and of the noise
groups ``snr_noise``, their binned fidelity with its error (for the eye) and the ``model`` fidelity.
"""
snr_data, snr_noise = np.asarray(snr_data, dtype=float), np.asarray(snr_noise, dtype=float)
top = max(snr_data.max(initial=floor), snr_noise.max(initial=floor))
edges = floor + step * np.arange(int((top - floor) / step) + 2)
n_data, n_noise = (np.histogram(x, edges)[0] for x in (snr_data, snr_noise))
expected = -np.diff(noise(edges))
model = np.clip(1 - expected / np.maximum(n_data, 1), 0, 1)
use = n_data > 0
d, n = n_data[use], n_noise[use]
bins = {
"snr": edges[:-1][use] + step / 2,
"n_data": d,
"n_noise": n,
"fidelity": np.clip(1 - n / d, 0, 1),
"error": np.sqrt(np.maximum(n, 1) + n**2 / d) / d,
"model": model[use],
}
return model[np.clip(np.searchsorted(edges, snr_data, side="right") - 1, 0, model.size - 1)], bins
#: Units of the :func:`catalogue` columns.
UNITS = {
"dra": "arcsec",
"ddec": "arcsec",
"ra": "deg",
"dec": "deg",
"freq_ghz": "GHz",
"bmin": "arcsec",
"bmaj": "arcsec",
"pa": "deg",
"width": "km/s",
"flux": "Jy",
"error": "Jy",
"line_flux": "Jy km/s",
"line_flux_error": "Jy km/s",
}
#: ``meta`` of :func:`catalogue` that belongs to one search, kept per part by :func:`merge_catalogues`.
SEARCH_META = ("jackknife_spread", "hwhm_major", "hwhm_minor", "pa", "spectral_hwhm", "sidelobe")
[docs]
def catalogue(
result: SearchResult,
ref=None,
floor: float = RELIABILITY["floor"],
k: float = RELIABILITY["k"],
fidelity_min: float = RELIABILITY["fidelity_min"],
rule: str = RELIABILITY["rule"],
band: tuple[float, float] | None = None,
) -> Table:
"""Catalogue the line candidates of a search: one row per group of the data, by S/N descending.
The noise correlation of ``result`` is measured, and the data, the jackknife and the negated
data are grouped (:func:`groups`) on the filter's S/N (see the module docstring), the data from
``floor``, the jackknife and the negated data from :data:`NOISE_FLOOR`. Each data group gets the
likelihood ratio and the fidelity against the noise peak counts modelled on the jackknife's groups
of its own kernel (:func:`peak_exponent`, :func:`likelihood`, :func:`fidelity`). It is ``detected``
if ``Lambda >= k`` and its fidelity is at least ``fidelity_min`` (``rule="and"``, the default), or
either (``rule="or"``); without a noise model (fewer than :data:`MIN_NOISE` noise peaks) nothing
is. Nothing is clipped: the candidates are ``cat[cat["detected"]]``.
Parameters
----------
result : SearchResult
PB-corrected (and, for a mosaic, combined) search with its jackknife.
ref : (RA, Dec) in rad, optional
The sky frame's reference (:attr:`~luv_finder.data.Metadata.ref`); adds ``ra`` and ``dec``.
floor : float
Lowest S/N of a local maximum, and the lower edge of the fidelity bins.
k : float
Likelihood-ratio threshold.
fidelity_min : float
Fidelity threshold.
rule : {"or", "and"}
How the two tests combine into ``detected``.
band : (low, high) Hz, optional
Keep only the groups (of the data, the jackknife and the negated data) that peak in
``low <= nu < high``: the chunk's own part of a spectrum searched in overlapping chunks.
Returns
-------
astropy.table.Table
Columns ``id``, ``dra``, ``ddec`` (arcsec), ``ra``, ``dec`` (deg, with ``ref``),
``freq_ghz``, the template's ``bmin``, ``bmaj``, ``pa`` and ``width``, the ``snr``,
``flux`` and ``error`` (Jy) and ``line_flux``, ``line_flux_error`` (Jy km/s) at the
brightest maximum, ``coverage``, ``n_peaks``, ``likelihood``, ``fidelity`` and
``detected``. ``meta`` holds the ``jackknife_spread``, the noise
correlation's ``hwhm_major``, ``hwhm_minor``, ``pa``, ``spectral_hwhm`` and ``sidelobe``,
the thresholds, the noise model (``noise_floor``, ``noise_exponent``, ``kernels``), the
lowest data S/N with ``Lambda >= k`` (``snr_likelihood``) and with a fidelity of at least
``fidelity_min`` (``snr_fidelity``), NaN if none, the S/N of every group of the data, the
jackknife and the negated data (``snr_data``, ``snr_jackknife``, ``snr_negative``) and the
binned fidelity (``fidelity_bins``), all as lists and floats.
Raises
------
ValueError
If ``result`` has no jackknife.
"""
nc = response_correlation(result)
found = {
name: groups(values, result, nc, low)
for name, values, low in (
("data", result.snr, floor),
("jackknife", result.jackknife.snr, NOISE_FLOOR),
("negative", -result.snr, NOISE_FLOOR),
)
}
if band is not None:
found = {name: [g for g in gs if band[0] <= result.freqs[g.k] < band[1]] for name, gs in found.items()}
snr = {name: np.array([g.snr for g in gs]) for name, gs in found.items()}
kernels = {name: result.templates[np.array([g.t for g in gs], dtype=int)] for name, gs in found.items()}
i, j, t, c = (np.array([getattr(g, a) for g in found["data"]], dtype=int) for a in "ijtk")
templates = result.templates[t]
columns = {"id": np.arange(1, i.size + 1), "dra": np.asarray(result.axes["dra"])[i]}
columns["ddec"] = np.asarray(result.axes["ddec"])[j]
if ref is not None:
columns["ra"], columns["dec"] = np.degrees(sky_direction((columns["dra"], columns["ddec"]), ref))
columns |= {"freq_ghz": result.freqs[c] / 1e9, **dict(zip(TEMPLATE_KEYS, templates.T, strict=True))}
peak, error = result.snr[i, j, t, c], result.error[i, j, t, c]
per_width = templates[:, TEMPLATE_KEYS.index("width")] * LINE_FLUX # line flux per peak flux density
columns |= {"snr": peak, "flux": peak * error, "error": error}
columns |= {"line_flux": peak * error * per_width, "line_flux_error": error * per_width}
columns["coverage"] = result.coverage[i, j, c]
columns["n_peaks"] = np.array([g.n_peaks for g in found["data"]], dtype=int)
dim = 3 if result.freqs.size > 1 else 2 # position and frequency, or a continuum's position
reliable, meta = _reliability(snr, kernels, dim, floor, k, fidelity_min, rule)
search = {"jackknife_spread": jackknife_spread(result)} | {key: getattr(nc, key) for key in SEARCH_META[1:]}
cat = Table(columns | reliable, meta={key: value.tolist() for key, value in search.items()} | meta)
for name, unit in UNITS.items():
if name in cat.colnames:
cat[name].unit = unit
return cat
[docs]
def continuum_catalogue(windows: SearchResult, ref=None, **reliability) -> Table:
"""Catalogue the continuum sources of a search's continuum per window (``SearchResult.continuum``).
Grouped and tested as :func:`catalogue` on the windows combined (:func:`~luv_finder.matchedfilter.continuum`,
``freq_ghz`` their mean frequency), each source has its in-band spectrum, the flux density and error of every
window at its position and template (``meta["spectra"]``), and the spectral index ``alpha`` (``S ~ nu^alpha``)
fitted to it, linearised about ``freq_ghz``.
"""
total = continuum(windows)
cat = catalogue(total, ref=ref, **reliability)
cat.remove_columns(["width", "line_flux", "line_flux_error"])
i, j = (
np.abs(np.asarray(windows.axes[k])[None] - np.asarray(cat[k])[:, None]).argmin(axis=1) for k in ("dra", "ddec")
)
shapes = np.c_[cat["bmin"], cat["bmaj"], cat["pa"]].astype(float)
t = np.abs(windows.templates[None, :, :3] - shapes[:, None]).sum(axis=2).argmin(axis=1)
flux, error = windows.flux[i, j, t], windows.error[i, j, t]
alpha = np.full((2, len(cat)), np.nan)
for n, (f, e) in enumerate(zip(flux, error, strict=True)):
ok = np.isfinite(f) & np.isfinite(e)
x, w = np.log(windows.freqs[ok] / total.freqs[0]), e[ok] ** -2.0
if x.size > 1 and np.ptp(x) > 0:
a = np.linalg.inv([[w.sum(), w @ x], [w @ x, w @ x**2]]) # covariance of (S0, S0 alpha)
s0, b = a @ [w @ f[ok], w @ (x * f[ok])]
alpha[:, n] = b / s0, np.sqrt(a[1, 1] - 2 * b / s0 * a[0, 1] + (b / s0) ** 2 * a[0, 0]) / abs(s0)
cat["alpha"], cat["alpha_error"] = alpha
cat.meta["spectra"] = {
str(k): {"freq_ghz": (windows.freqs / 1e9).tolist(), "flux": f.tolist(), "error": e.tolist()}
for k, f, e in zip(cat["id"], flux, error, strict=True)
}
return cat
[docs]
def bright(cat: Table, floor: float) -> np.ndarray:
"""The detections whose sidelobes reach ``floor``: S/N at least the brightest's times the response's highest
``sidelobe``, none if the brightest's own sidelobes stay below ``floor``. They are subtracted in rounds."""
snr, detected = np.asarray(cat["snr"], dtype=float), np.asarray(cat["detected"], dtype=bool)
level = max(cat.meta["sidelobe"]) * snr[detected].max(initial=0.0)
return detected & (snr >= level) if level >= floor else np.zeros(len(cat), dtype=bool)
[docs]
def passes(ratio, fid, k: float, fidelity_min: float) -> tuple[np.ndarray, np.ndarray]:
"""Which groups pass the likelihood ratio and which the fidelity (NaN passes neither)."""
return np.asarray(ratio) >= k, np.asarray(fid) >= fidelity_min
def _reliability(
snr: dict, kernels: dict, dim: int, floor: float, k: float, fidelity_min: float, rule: str = DETECTION_RULE
) -> tuple[dict, dict]:
"""Likelihood ratio, fidelity and detection of the data groups against the noise model, as columns and meta.
The noise is the jackknife's peaks above :data:`NOISE_FLOOR`: one exponent for the whole search of
dimension ``dim`` (:func:`peak_exponent`) and, per kernel (``kernels``: each group's template, a row of
:data:`TEMPLATE_KEYS`, as ``snr``), the number of its peaks, since kernels of different size make
different numbers of independent trials (González-López et al. 2019). Each data group is tested
against its own kernel's model. ``meta["kernels"]`` has each kernel's counts and thresholds.
"""
if rule not in ("or", "and"):
raise ValueError(f'rule is "or" or "and", not {rule!r}')
data, jackknife = snr["data"], snr["jackknife"]
m = peak_exponent(jackknife, dim)
ratio, fid = np.full(data.size, np.nan), np.full(data.size, np.nan)
per_kernel = []
for kernel in np.unique(np.r_[kernels["data"], kernels["jackknife"]], axis=0):
d = np.all(kernels["data"] == kernel, axis=1)
n = int(np.sum(np.all(kernels["jackknife"] == kernel, axis=1)))
def noise(u, n=n):
return noise_above(u, n, m)
ratio[d] = likelihood(data[d], noise)
fid[d] = fidelity(data[d], [], noise, floor)[0]
lr_k, fid_k = passes(ratio[d], fid[d], k, fidelity_min)
per_kernel.append(
{"template": kernel.tolist(), "n_data": int(d.sum()), "n_jackknife": n,
"snr_likelihood": float(data[d][lr_k].min()) if lr_k.any() else np.nan,
"snr_fidelity": float(data[d][fid_k].min()) if fid_k.any() else np.nan}
) # fmt: skip
n_all = [kn["n_jackknife"] for kn in per_kernel]
bins = fidelity(data, jackknife[jackknife >= floor], lambda u: noise_above(u, sum(n_all), m), floor)[1]
lr, fid_ok = passes(ratio, fid, k, fidelity_min)
detected = lr | fid_ok if rule == "or" else lr & fid_ok
columns = {"likelihood": ratio, "fidelity": fid, "detected": detected}
meta = {"floor": float(floor), "k": float(k), "fidelity_min": float(fidelity_min), "rule": rule}
meta |= {"noise_floor": NOISE_FLOOR, "noise_dim": int(dim), "noise_exponent": m}
for key, passed in (("snr_likelihood", lr), ("snr_fidelity", fid_ok)):
meta[key] = float(data[passed].min()) if passed.any() else np.nan
meta |= {f"snr_{name}": values.tolist() for name, values in snr.items()}
meta |= {f"template_{name}": np.asarray(values).tolist() for name, values in kernels.items() if name != "data"}
meta["fidelity_bins"] = {key: values.tolist() for key, values in bins.items()}
meta["kernels"] = per_kernel
return columns, meta
[docs]
def merge_catalogues(cats: Sequence[Table], reliability: dict | None = None, beam: dict | None = None) -> Table:
"""One catalogue of several searches (e.g. one per spectral window), its reliability over their whole volume.
``reliability`` (``floor``, ``k``, ``fidelity_min``, ``rule``) overrides the searches' own, so the
tests can change without a new search. With ``beam`` (arcsec per ``unit``), a line found by two
searches (:func:`duplicates`) counts once: the fainter group gets ``duplicate_of`` and is not detected.
"""
given = {key: cats[0].meta[key] for key in RELIABILITY} | (reliability or {})
cat = vstack(list(cats), metadata_conflicts="silent").copy() # vstack of one table returns it
order = np.argsort(-np.asarray(cat["snr"]), kind="stable")
snr = {"data": np.asarray(cat["snr"])[order]}
snr |= {name: np.concatenate([c.meta[f"snr_{name}"] for c in cats]) for name in ("jackknife", "negative")}
kernels = {
name: np.concatenate([np.reshape(c.meta[f"template_{name}"], (-1, len(TEMPLATE_KEYS))) for c in cats])
for name in ("jackknife", "negative")
}
cat = cat[order]
cat["id"] = np.arange(1, len(cat) + 1)
one = np.ones(len(cat), dtype=bool)
if beam is not None:
cat["duplicate_of"] = duplicates(cat, beam)
one = np.asarray(cat["duplicate_of"] == 0)
kernels["data"] = np.c_[tuple(np.asarray(cat[key], dtype=float) for key in TEMPLATE_KEYS)][one]
data = {**snr, "data": snr["data"][one]}
reliable, meta = _reliability(data, kernels, cats[0].meta["noise_dim"], *(given[key] for key in RELIABILITY))
for name, values in reliable.items():
cat[name] = np.full(len(cat), False if values.dtype == bool else np.nan, dtype=values.dtype)
cat[name][one] = values
cat.meta = meta | {"parts": [{key: c.meta[key] for key in SEARCH_META} for c in cats]}
return cat
[docs]
def duplicates(cat: Table, beam: dict, by: str = "unit") -> np.ndarray:
"""Id of a brighter group of the same line in another unit (another ``by``), else 0, for every row of ``cat``.
Two groups are one line when they are within ``beam[unit]`` (arcsec) of each other and their
templates overlap, ``|dv| <= (FWHM_a + FWHM_b) / 2``: a line wider than the gap it straddles between
two windows is found once in each, its halves far apart in frequency. ``cat`` is sorted by S/N.
"""
from scipy.spatial import cKDTree
dra, ddec, nu, width = (np.asarray(cat[k], dtype=float) for k in ("dra", "ddec", "freq_ghz", "width"))
unit = np.asarray(cat[by])
pairs = cKDTree(np.c_[dra, ddec]).query_pairs(max(beam.values()), output_type="ndarray")
pairs.sort(axis=1) # a brighter than b
a, b = pairs.T if pairs.size else (np.zeros(0, int), np.zeros(0, int))
same = (
(unit[a] != unit[b])
& (np.hypot(dra[a] - dra[b], ddec[a] - ddec[b]) <= np.array([beam[u] for u in unit[a]]))
& (np.abs(nu[b] / nu[a] - 1) * C_KMS <= (width[a] + width[b]) / 2)
)
of = np.zeros(len(cat), dtype=int)
for i, j in sorted(zip(a[same].tolist(), b[same].tolist(), strict=True)):
if not of[i] and not of[j]:
of[j] = cat["id"][i]
return of