Source code for luv_finder.data

"""Measurement-set I/O and visibility-domain operations.

Visibilities are held per (field, spectral window) in a :class:`Chunk`: the weighted
visibilities ``X = w * V`` and the weight of every visibility ``w`` (0 where flagged) as
``(n_chan, n_row)`` arrays, and what is constant along an axis (``u``, ``v``, time, baseline and
scan per row, frequency per channel) once. The weights are corrected on reading, per scan and
channel, by the noise measured in the data (:func:`reweighted`).

Positions are offsets in one sky frame per dataset: arcsec east and north of
:attr:`Metadata.ref`, the model's (dra, ddec) convention. Each chunk carries its field's phase
centre in that frame, so the same (dra, ddec) means the same sky position in every pointing.

CASA (``casatools``) is imported lazily inside the functions that read a measurement set, so
the NPZ path (:meth:`DataHandler.from_npz`) works in an environment without CASA.
"""

from __future__ import annotations

import dataclasses
import warnings
import zipfile
from collections.abc import Iterable, Iterator, Sequence
from dataclasses import dataclass
from functools import cache
from itertools import groupby
from typing import NamedTuple

import numpy as np

from ._casa import tools
from .utils import ARCSEC, C

#: Arrays stored per chunk in an NPZ, under ``{field}_{spw}_{name}``.
CHUNK_ARRAYS = ("offset", "freq", "u", "v", "time", "baseline", "scan", "X", "w")

#: Target size of one block of rows read from a measurement set, in visibility-channels.
READ_BLOCK = 25_000_000

#: Largest time between two integrations of a baseline that measure the weights' noise together, s.
MAX_GAP = 450.0

#: Channels handled at once when the weights are measured.
CHANNEL_BLOCK = 16

#: Longest channel lag :func:`channel_noise` measures, and the significance a lag's correlation needs to count.
MAX_LAG, LAG_SIGNIFICANCE = 8, 5.0


[docs] def pair_weight(a, b) -> np.ndarray: """Inverse variance of ``(x_a +- x_b) / 2`` from the weights of ``x_a`` and ``x_b``; 0 if either is.""" a, b = np.asarray(a, dtype=float), np.asarray(b, dtype=float) s = a + b return np.divide(4 * a * b, s, out=np.zeros(np.broadcast(a, b).shape), where=s > 0)
def _pairs(time: np.ndarray, baseline: np.ndarray, max_gap: float = np.inf) -> tuple[np.ndarray, np.ndarray]: """Disjoint pairs of consecutive integrations of each baseline, at most ``max_gap`` s apart, in time order.""" order = np.lexsort((time, baseline)) linked = (baseline[order][1:] == baseline[order][:-1]) & (np.diff(time[order]) <= max_gap) start = np.flatnonzero(np.r_[True, ~linked]) rank = np.arange(order.size) - start[np.cumsum(np.r_[True, ~linked]) - 1] pick = np.flatnonzero((rank % 2 == 0)[:-1] & linked) return order[pick], order[pick + 1] def _vis(X: np.ndarray, w: np.ndarray) -> np.ndarray: return np.divide(X, w, out=np.zeros_like(X), where=w > 0) def _difference(X: np.ndarray, w: np.ndarray, a: np.ndarray, b: np.ndarray) -> tuple[np.ndarray, np.ndarray]: """The pairs' half differences ``(V_a - V_b) / 2`` weighted, and their weights :func:`pair_weight`.""" wa, wb = w[:, a], w[:, b] weight = pair_weight(wa, wb) return weight * 0.5 * (_vis(X[:, a], wa) - _vis(X[:, b], wb)), weight
[docs] class WeightScale(NamedTuple): """Noise power over the weights per scan and channel, ``scale``, its robust ``check`` and that one's ``error``.""" scans: np.ndarray scale: np.ndarray check: np.ndarray error: np.ndarray
[docs] def at(self, scan: np.ndarray) -> np.ndarray: """``scale`` of rows in ``scan``, ``(n_chan, n_row)``.""" return self.scale[np.searchsorted(self.scans, scan)].T
[docs] def deviation(self) -> np.ndarray: """``check / scale - 1`` in units of its counting error: about N(0, 1) for Gaussian noise.""" return (self.check / self.scale - 1) / self.error
[docs] def weight_scale(chunks: Sequence[Chunk]) -> WeightScale: """``sum w z^2 / 2 sum w`` per (scan, channel) over a window's integrations of a baseline <= ``MAX_GAP`` apart. The integrations pair in their own field, so a pair sees one sky; a (scan, channel) with no such pair (a mosaic field seen once per visit) pairs across fields. A pair that spans two scans counts in both, so a scan of one integration is measured too. """ time, baseline, scan = (np.concatenate([getattr(c, k) for c in chunks]) for k in ("time", "baseline", "scan")) field = np.concatenate([np.full(c.time.size, c.field) for c in chunks]) scans = np.unique(scan) own = _scale_cells( chunks, *_pairs(time, baseline.astype(np.int64) * (field.max() + 1) + field, MAX_GAP), scan, scans ) if np.any(own["data"] & (own["den"] == 0)): across = _scale_cells(chunks, *_pairs(time, baseline, MAX_GAP), scan, scans) use = own["den"] > 0 own = {k: np.where(use, own[k], across[k]) for k in own} if np.any(own["data"] & (own["den"] == 0)): k = int(np.flatnonzero(np.any(own["data"] & (own["den"] == 0), axis=1))[0]) raise ValueError( f"spw {chunks[0].spw}, scan {scans[k] & 0xFFFFFFFF} of execution block {scans[k] >> 32}: no two " f"integrations of a baseline within {MAX_GAP:.0f} s to measure its weights; flag it or raise MAX_GAP" ) if np.any((own["den"] > 0) & (own["num"] == 0)): raise ValueError(f"spw {chunks[0].spw} holds no noise to measure; read noiseless data with reweight=False") scale = np.where(own["den"] > 0, own["num"] / np.where(own["den"] > 0, own["den"], 1.0), 1.0) return WeightScale(scans, scale, own["check"], own["error"])
def _scale_cells(chunks: Sequence[Chunk], a, b, scan, scans) -> dict: """``num``, ``den`` (of :func:`weight_scale`), the robust ``check``, its ``error`` and whether there are ``data``, per (scan, channel), from the pairs ``a``, ``b``.""" of_a, of_b, of_row = (np.searchsorted(scans, s) for s in (scan[a], scan[b], scan)) pairs = [np.flatnonzero((of_a == k) | (of_b == k)) for k in range(scans.size)] rows = [np.flatnonzero(of_row == k) for k in range(scans.size)] n_chan = chunks[0].freq.size out = {k: np.zeros((scans.size, n_chan)) for k in ("num", "den")} out |= {k: np.full((scans.size, n_chan), np.nan) for k in ("check", "error")} out["data"] = np.zeros((scans.size, n_chan), dtype=bool) for lo in range(0, n_chan, CHANNEL_BLOCK): sl = slice(lo, lo + CHANNEL_BLOCK) X, w = (np.concatenate([getattr(c, k)[sl] for c in chunks], axis=1) for k in ("X", "w")) diff, weight = _difference(X, w, a, b) wp = weight / 4 # the pair's inverse variance of V_a - V_b z2 = np.divide(np.abs(diff) ** 2, weight, out=np.zeros_like(weight), where=weight > 0) for k, (m, r) in enumerate(zip(pairs, rows, strict=True)): wpk, z2k, both = wp[:, m], z2[:, m], weight[:, m] > 0 half = np.sum(wpk, axis=1) out["num"][k, sl], out["den"][k, sl] = np.sum(wpk * z2k, axis=1), 2 * half out["data"][k, sl] = np.sum(w[:, r], axis=1) > 0 out["check"][k, sl] = [_robust(z[o]) for z, o in zip(z2k, both, strict=True)] n, ok = both.sum(axis=1), half > 0 # the median's variance 1 / ln(2)^2 / n, the weighted mean's, less twice their covariance 1 / n out["error"][k, sl][ok] = np.sqrt( (1 / np.log(2) ** 2 - 2) / n[ok] + np.sum(wpk**2, axis=1)[ok] / half[ok] ** 2 ) return out def _robust(z2: np.ndarray) -> float: """The noise power from the median of ``z^2``, ``median / 2 ln 2``, which outliers barely move.""" return float(np.median(z2) / (2 * np.log(2))) if z2.size else np.nan
[docs] def reweighted(chunks: Sequence[Chunk], ws: WeightScale | None = None) -> list[Chunk]: """The chunks of one window, weights divided by ``ws`` (default :func:`weight_scale`) per scan and channel.""" ws = weight_scale(chunks) if ws is None else ws _report(ws, chunks[0].spw) return [c.scaled(1 / ws.at(c.scan)) for c in chunks]
def _report(ws: WeightScale, spw: int) -> None: """Print a window's noise over its weights; warn if the weighted and robust measures disagree.""" deviation = np.nanmax(np.abs(ws.deviation())) print( f"spw {spw}: noise over the weights in {ws.scans.size} scans, median per scan " f"{np.median(ws.scale, axis=1).min():.3f}-{np.median(ws.scale, axis=1).max():.3f}, " f"range {ws.scale.min():.2f}-{ws.scale.max():.2f}; " f"robust check within {deviation:.1f} sigma", flush=True, ) if deviation > 5: warnings.warn(f"spw {spw}: the weighted and robust noise differ by {deviation:.1f} sigma", stacklevel=3)
[docs] class ChannelNoise(NamedTuple): """Noise of each channel over its weights, ``scale`` (1 where they hold), and between channels, ``rho``.""" scale: np.ndarray rho: np.ndarray
[docs] def channel_noise(jackknife: Iterable[Chunk]) -> ChannelNoise: """Of one window's jackknife (:meth:`Chunk.jackknife`): the noise power per channel over the weights, ``sum |X|^2 / 2 sum w``, and the channels' correlation ``(1, rho_1, ...)`` up to the first lag consistent with 0.""" num, norm_a, norm_b, n = (np.zeros(MAX_LAG + 1) for _ in range(4)) power = weight = 0.0 for jk in jackknife: power = power + np.sum(np.abs(jk.X) ** 2, axis=1) weight = weight + 2 * jk.w.sum(axis=1) z = np.divide(jk.X, np.sqrt(jk.w), out=np.zeros_like(jk.X), where=jk.w > 0) # 0 where flagged z2, live = np.abs(z) ** 2, jk.w > 0 for lag in range(1, min(MAX_LAG, z.shape[0] - 1) + 1): ok = live[:-lag] & live[lag:] num[lag] += np.sum((z[:-lag] * z[lag:].conj()).real) norm_a[lag] += np.sum(z2[:-lag], where=ok) norm_b[lag] += np.sum(z2[lag:], where=ok) n[lag] += 2 * ok.sum() if not np.any(power): raise ValueError("the jackknife holds no noise to measure; pass channel_noise=None or a noisy dataset's") rho = [1.0] for lag in range(1, MAX_LAG + 1): if norm_a[lag] * norm_b[lag] == 0: break r = num[lag] / np.sqrt(norm_a[lag] * norm_b[lag]) if abs(r) * np.sqrt(n[lag]) < LAG_SIGNIFICANCE: break rho.append(float(r)) return ChannelNoise(np.divide(power, weight, out=np.ones_like(power), where=weight > 0), np.array(rho))
[docs] def stokes_i(data: np.ndarray, weight: np.ndarray, flag: np.ndarray): """Average the parallel hands ``[0, -1]`` into (vis, w); ``weight`` per row or per channel, ``w`` 0 if flagged.""" w = pair_weight(weight[0], weight[-1]) return 0.5 * (data[0] + data[-1]), np.where(flag[0] | flag[-1], 0.0, np.broadcast_to(w, data.shape[1:]))
[docs] def sky_offset(direction, ref) -> np.ndarray: """(east, north) offset in arcsec of ``direction`` on the tangent plane at ``ref``, both (RA, Dec) in rad.""" (ra, dec), (ra0, dec0) = direction, ref east = np.cos(dec) * np.sin(ra - ra0) north = np.sin(dec) * np.cos(dec0) - np.cos(dec) * np.sin(dec0) * np.cos(ra - ra0) return np.array([east, north]) / ARCSEC
[docs] def sky_direction(offset, ref) -> np.ndarray: """(RA, Dec) in rad of the (east, north) ``offset`` in arcsec from ``ref``, the inverse of :func:`sky_offset`.""" (l, m), (ra0, dec0) = np.asarray(offset, dtype=float) * ARCSEC, ref n = np.sqrt(1 - l**2 - m**2) dec = np.arcsin(m * np.cos(dec0) + n * np.sin(dec0)) ra = ra0 + np.arctan2(l, n * np.cos(dec0) - m * np.sin(dec0)) return np.array([ra % (2 * np.pi), dec])
[docs] @dataclass(frozen=True, eq=False) class Chunk: """Visibilities of one (field, spectral window), channel-major. ``X = w * V`` is zero where ``w`` is (flagged), so a weighted sum over visibilities is a plain sum over ``X``. Attributes ---------- offset : (2,) arcsec, the field's phase centre in the dataset's sky frame freq : (n_chan,) Hz, ascending u, v : (n_row,) m time : (n_row,) s baseline : (n_row,) int, identifies the antenna pair scan : (n_row,) int64, the scan, ``OBSERVATION_ID << 32 | SCAN_NUMBER`` X : (n_chan, n_row) complex, weighted visibilities w : (n_chan, n_row) float32, inverse variance of each visibility, 0 where flagged """ field: int spw: int offset: np.ndarray freq: np.ndarray u: np.ndarray v: np.ndarray time: np.ndarray baseline: np.ndarray scan: np.ndarray X: np.ndarray w: np.ndarray @property def vis(self) -> np.ndarray: """Visibilities in Jy, 0 where flagged.""" return _vis(self.X, self.w) @property def nbytes(self) -> int: return sum(getattr(self, k).nbytes for k in CHUNK_ARRAYS)
[docs] def scaled(self, factor) -> Chunk: """``X`` and ``w`` times ``factor`` (per visibility, or broadcast): weights rescaled, data as they were.""" return dataclasses.replace(self, X=self.X * factor, w=(self.w * factor).astype(np.float32))
[docs] def binned(self, n: int, rho=None) -> tuple[Chunk, np.ndarray]: """The channels averaged in runs of ``n`` (the last shorter unless ``n`` divides them) and the binned channels' correlation. ``X`` and ``w`` add, an inverse-variance average at each run's mean frequency. Under the channels' correlation ``rho`` (``(1, rho_1, ...)``) a run's variance is ``sum_kl rho_|k-l|`` times the independent one: the weights are divided by it, and neighbouring runs correlate by the ``rho`` spanning them. """ rho = np.ones(1) if rho is None else np.asarray(rho, dtype=float) starts = np.arange(0, self.freq.size, n) runs = np.arange(-(-(rho.size - 1) // n) + 1)[:, None, None] # the runs a correlation spans lag = np.abs(np.arange(n)[:, None] - np.arange(n)[None, :] + n * runs) corr = np.where(lag < rho.size, rho[np.minimum(lag, rho.size - 1)], 0.0).sum(axis=(1, 2)) scale = n / corr[0] freq = np.add.reduceat(self.freq, starts) / np.diff(np.r_[starts, self.freq.size]) X, w = np.add.reduceat(self.X, starts, axis=0) * scale, np.add.reduceat(self.w, starts, axis=0) * scale return dataclasses.replace(self, freq=freq, X=X, w=w.astype(np.float32)), corr / corr[0]
[docs] def longer_than(self, length: float) -> Chunk: """Only the rows of baselines at least ``length`` (m) long.""" keep = np.hypot(self.u, self.v) >= length rows = {k: getattr(self, k)[keep] for k in ("u", "v", "time", "baseline", "scan")} return dataclasses.replace(self, **rows, X=self.X[:, keep], w=self.w[:, keep])
[docs] def uv_waves(self) -> tuple[np.ndarray, np.ndarray]: """``u`` and ``v`` in wavelengths, ``(n_chan, n_row)``.""" k = self.freq[:, None] / C return self.u * k, self.v * k
[docs] def phase(self, dra: float, ddec: float) -> np.ndarray: """``exp(-2 pi i (u dra + v ddec))``: moves the phase centre to (dra, ddec) arcsec in the sky frame.""" delay = (self.u * (dra - self.offset[0]) + self.v * (ddec - self.offset[1])) * ARCSEC / C return np.exp(-2j * np.pi * np.outer(self.freq, delay))
[docs] def jackknife(self) -> Chunk: """A signal-free noise realisation: consecutive integrations of every baseline differenced and halved. Each baseline's integrations are paired in time order, its first with its second, third with fourth, ..., whatever the other baselines do, so a mosaic field seen once per visit (or a baseline missing from some integrations) still pairs nearly every row; an odd last integration is left out. A signal constant over a pair cancels; the noise stays, with weight ``pair_weight`` (2w for equal weights). The split is by integration, never random: a random sign flip of individual visibilities destroys the baseline structure and does not give the observation's white-noise level. """ a, b = _pairs(self.time, self.baseline) X, w = _difference(self.X, self.w, a, b) return dataclasses.replace( self, u=0.5 * (self.u[a] + self.u[b]), v=0.5 * (self.v[a] + self.v[b]), time=0.5 * (self.time[a] + self.time[b]), baseline=self.baseline[a], scan=self.scan[a], X=X, w=w.astype(np.float32), )
[docs] @dataclass(frozen=True) class Metadata: """Quantities of the whole dataset, the same whichever fields or windows are loaded. Attributes ---------- dish_diameter : m ref : (RA, Dec) in rad that sky offsets are measured from (the first target field) central_frequency : Hz max_uvwave : longest baseline in wavelengths """ dish_diameter: float ref: tuple[float, float] central_frequency: float max_uvwave: float
[docs] @classmethod @cache def from_ms(cls, msfile: str) -> Metadata: """The measurement set's metadata (read once per process: it scans every row's UVW).""" fields, spws = target_selection(msfile) msmd = tools().msmetadata() msmd.open(msfile) diameters = msmd.antennadiameter() dish = diameters[next(iter(diameters))]["value"] ref = _phase_centre(msmd, fields[0]) freqs = np.concatenate([msmd.chanfreqs(int(s)) for s in spws]) msmd.close() tb = tools().table() tb.open(f"{msfile}/SPECTRAL_WINDOW") check_frame(msfile, tb.getcol("MEAS_FREQ_REF")[[int(s) for s in spws]]) tb.close() tb.open(msfile) sel = tb.query(f"FIELD_ID IN {[int(f) for f in fields]} AND ANTENNA1 != ANTENNA2", columns="UVW") uvw = sel.getcol("UVW") sel.close() tb.close() return cls(dish, ref, float(freqs.mean()), float(np.hypot(uvw[0], uvw[1]).max() * freqs.max() / C))
[docs] def minresolution(self) -> float: """Angular resolution in arcsec from the longest baseline.""" return 1.0 / self.max_uvwave / ARCSEC
#: casacore's MFrequency reference codes (``MEAS_FREQ_REF`` of an MS's SPECTRAL_WINDOW). FREQUENCY_FRAMES = ("REST", "LSRK", "LSRD", "BARY", "GEO", "TOPO", "GALACTO", "LGROUP", "CMB")
[docs] def check_frame(msfile: str, codes) -> None: """Warn unless every window is in LSRK: the search, the catalogue and the fit keep the MS's frame.""" frames = sorted({FREQUENCY_FRAMES[int(c)] for c in np.atleast_1d(codes)}) if frames != ["LSRK"]: warnings.warn( f"{msfile}: channel frequencies are in {', '.join(frames)}, not LSRK, and so will be the catalogue's " "frequencies and redshifts (TOPO differs from LSRK by up to ~30 km/s); regrid to LSRK first " "(CASA mstransform, outframe='LSRK') for published values", stacklevel=2, )
[docs] def target_selection(msfile: str): """Field and spectral-window ids observed with the target intent.""" msmd = tools().msmetadata() msmd.open(msfile) fields = msmd.fieldsforintent("*OBSERVE_TARGET*", False) spws = msmd.spwsforintent("*OBSERVE_TARGET*") msmd.close() return fields, spws
def _phase_centre(msmd, field) -> tuple[float, float]: centre = msmd.phasecenter(int(field)) return centre["m0"]["value"], centre["m1"]["value"]
[docs] def iter_chunks( msfile: str, ref, fields=None, spws=None, reweight: bool = True, scales: dict | None = None ) -> Iterator[Chunk]: """Read target visibilities one spw at a time over all ``fields``, weights corrected, offsets from ``ref``. Autocorrelations, flagged rows and rows with no unflagged channel are dropped. Every window's weights are divided by the noise measured in it (:func:`weight_scale`), so the peak memory is one window over all its fields; ``reweight=False`` keeps the measurement set's weights, for a noiseless simulation, which has no noise to measure. A ``scales`` dict receives each window's :class:`WeightScale` by spw. """ all_fields, all_spws = target_selection(msfile) msmd = tools().msmetadata() msmd.open(msfile) n_ant = msmd.nantennas() plan = [ ( int(s), int(f), sky_offset(_phase_centre(msmd, f), ref), msmd.chanfreqs(int(s)), msmd.datadescids(spw=int(s))[0], ) for s in (all_spws if spws is None else spws) for f in (all_fields if fields is None else fields) ] msmd.close() tb = tools().table() tb.open(msfile) spectrum = "WEIGHT_SPECTRUM" in tb.colnames() try: for _, window in groupby(plan, key=lambda p: p[0]): chunks = [] for spw, field, offset, freq, ddid in window: sel = tb.query( f"FIELD_ID == {field} AND DATA_DESC_ID == {ddid} AND ANTENNA1 != ANTENNA2 AND NOT FLAG_ROW" ) try: chunk = _read_chunk(sel, field, spw, offset, freq, n_ant, spectrum) finally: sel.close() if chunk is not None: chunks.append(chunk) if chunks and reweight: ws = weight_scale(chunks) if scales is not None: scales[chunks[0].spw] = ws _report(ws, chunks[0].spw) while chunks: # popped as yielded: one window's chunks in memory, not two c = chunks.pop(0) yield c.scaled(1 / ws.at(c.scan)) if reweight else c finally: tb.close()
def _read_chunk(sel, field, spw, offset, freq, n_ant, spectrum) -> Chunk | None: order = np.argsort(freq) step = max(1, READ_BLOCK // len(freq)) parts = [] for start in range(0, sel.nrows(), step): def col(name, start=start): return sel.getcol(name, start, step) vis, w = stokes_i(col("DATA"), col("WEIGHT_SPECTRUM" if spectrum else "WEIGHT"), col("FLAG")) keep = w.any(axis=0) w = w[np.ix_(order, keep)] uvw = col("UVW")[:, keep] parts.append( { "u": uvw[0], "v": uvw[1], "time": col("TIME")[keep], "baseline": (col("ANTENNA1") * n_ant + col("ANTENNA2"))[keep], "scan": (col("OBSERVATION_ID").astype(np.int64) << 32 | col("SCAN_NUMBER"))[keep], "X": vis[np.ix_(order, keep)] * w, "w": w.astype(np.float32), } ) if not sum(len(p["u"]) for p in parts): return None arrays = {k: np.concatenate([p[k] for p in parts], axis=-1) for k in parts[0]} return Chunk(field, spw, offset, freq[order], **arrays)
[docs] def write_npz(path: str, metadata: Metadata, chunks: Iterable[Chunk]) -> None: """Write an NPZ one array at a time, so ``chunks`` may be a generator that never sits in memory whole.""" def put(name, value): with zf.open(name + ".npy", "w", force_zip64=True) as fh: np.lib.format.write_array(fh, np.asanyarray(value)) ids = [] with zipfile.ZipFile(path, "w", allowZip64=True) as zf: for name, value in dataclasses.asdict(metadata).items(): put(name, value) for chunk in chunks: for name in CHUNK_ARRAYS: put(f"{chunk.field}_{chunk.spw}_{name}", getattr(chunk, name)) ids.append((chunk.field, chunk.spw)) put("chunks", np.array(ids, dtype=int).reshape(-1, 2))
[docs] class DataHandler: """Visibilities of a measurement set, one :class:`Chunk` per (field, spectral window). Parameters ---------- msfile : str, optional Measurement set to read; ``fields`` selects a subset of its target fields, and ``reweight=False`` keeps its weights (a noiseless simulation; see :func:`iter_chunks`). chunks, metadata : optional Ready-made data, e.g. from :meth:`from_npz` or a test. """ def __init__( self, msfile: str | None = None, chunks: list[Chunk] | None = None, metadata: Metadata | None = None, fields=None, reweight: bool = True, ): if msfile is not None: metadata = Metadata.from_ms(msfile) chunks = list(iter_chunks(msfile, metadata.ref, fields, reweight=reweight)) self.metadata = metadata self.chunks = sorted(chunks, key=lambda c: (c.field, c.freq[0])) @property def fields(self) -> list[int]: return sorted({c.field for c in self.chunks}) @property def freqs(self) -> np.ndarray: """Channel frequencies in Hz, every chunk in order.""" return np.concatenate([c.freq for c in self.chunks]) @property def nbytes(self) -> int: return sum(c.nbytes for c in self.chunks)
[docs] def jackknife(self) -> DataHandler: """Signal-free noise realisation of every chunk (see :meth:`Chunk.jackknife`).""" return DataHandler(chunks=[c.jackknife() for c in self.chunks], metadata=self.metadata)
[docs] def channel_noise(self, jackknife: DataHandler | None = None) -> dict[int, ChannelNoise]: """:func:`channel_noise` per spectral window, pooled over its fields: ``{spw: ChannelNoise}``; on ``jackknife`` if already made (default: :meth:`jackknife`, a chunk at a time).""" chunks = self.chunks if jackknife is None else jackknife.chunks spws = sorted({c.spw for c in chunks}) if jackknife is None: return {s: channel_noise(c.jackknife() for c in chunks if c.spw == s) for s in spws} return {s: channel_noise(c for c in chunks if c.spw == s) for s in spws}
[docs] def spectrum(self, dra: float = 0.0, ddec: float = 0.0, model=None) -> np.ndarray: """Weighted mean visibility per channel, phase-shifted to (dra, ddec) arcsec. With ``model`` (a source component) the mean is over its visibilities on the data's uv coverage and weights instead. """ out = [] for c in self.chunks: vis = c.X if model is None else c.w * model.profile(c) weight = c.w.sum(axis=1) total = (vis * c.phase(dra, ddec)).sum(axis=1) out.append(np.divide(total, weight, out=np.zeros_like(total), where=weight > 0)) return np.concatenate(out)
# --------------------------------------------------------------- NPZ I/O
[docs] def to_npz(self, path: str) -> None: """Save the chunks and metadata to an NPZ (CASA-free format).""" write_npz(path, self.metadata, self.chunks)
[docs] @classmethod def from_npz(cls, path: str, fields=None) -> DataHandler: """Load an NPZ written by :meth:`to_npz` or ``luv-export``; ``fields`` loads only those.""" with np.load(path) as f: metadata = Metadata( float(f["dish_diameter"]), tuple(f["ref"].tolist()), float(f["central_frequency"]), float(f["max_uvwave"]), ) chunks = [ Chunk(int(field), int(spw), **{k: f[f"{field}_{spw}_{k}"] for k in CHUNK_ARRAYS}) for field, spw in f["chunks"] if fields is None or field in fields ] return cls(chunks=chunks, metadata=metadata)
def _is_npz(path) -> bool: return str(path).endswith(".npz")
[docs] def fields_in(path) -> list[int]: """Target fields of a measurement set, an NPZ or a list of NPZs, sorted; an NPZ is not read beyond its index.""" if isinstance(path, list | tuple): return sorted(set().union(*(fields_in(p) for p in path))) if _is_npz(path): with np.load(path) as f: return np.unique(f["chunks"][:, 0]).tolist() return sorted(int(f) for f in target_selection(path)[0])
[docs] def pointing(path, field: int) -> tuple[Metadata, np.ndarray, float]: """A field's dataset metadata, phase centre (arcsec, the data's sky frame) and lowest frequency (Hz), from a measurement set, an NPZ or a list of NPZs, without reading its visibilities.""" if isinstance(path, list | tuple): parts = [pointing(p, field) for p in path if field in fields_in(p)] return parts[0][0], parts[0][1], min(p[2] for p in parts) if _is_npz(path): with np.load(path) as f: spws = [int(s) for fd, s in f["chunks"] if fd == field] offset, low = f[f"{field}_{spws[0]}_offset"], min(float(f[f"{field}_{s}_freq"].min()) for s in spws) return DataHandler.from_npz(path, []).metadata, offset, low metadata = Metadata.from_ms(path) msmd = tools().msmetadata() msmd.open(str(path)) try: offset = sky_offset(_phase_centre(msmd, field), metadata.ref) low = min(float(msmd.chanfreqs(int(s)).min()) for s in target_selection(str(path))[1]) finally: msmd.close() return metadata, offset, low
[docs] def load(path, fields=None) -> DataHandler: """The visibilities of ``fields`` (default: all) from a measurement set, an NPZ or a list of one dataset's NPZs.""" if isinstance(path, list | tuple): parts = [load(p, fields) for p in path] return DataHandler(chunks=[c for d in parts for c in d.chunks], metadata=parts[0].metadata) return DataHandler.from_npz(path, fields) if _is_npz(path) else DataHandler(str(path), fields=fields)