"""Synthetic ALMA observations of spectral-line sources via CASA ``simobserve``."""
from __future__ import annotations
import copy
import glob
import os
import shutil
import astropy.units as u
import numpy as np
import yaml
from astropy.constants import c
from astropy.coordinates import SkyCoord
from astropy.io import fits
from astropy.modeling import models
from matplotlib.figure import Figure
from ._casa import tasks, tools
from .model import FWHM_TO_SIGMA
_REF_RMS_JY = 0.05e-3
_REF_MIN = 27.0
SIMMOS = "alma/simmos"
[docs]
def resolve_antenna_config(name: str) -> str:
"""Absolute path to a CASA antenna configuration, by bare name or path."""
if os.path.sep in name and os.path.exists(name):
return name
repo = tools().ctsys.resolve(SIMMOS)
path = os.path.join(repo, name)
if not os.path.exists(path):
available = sorted(os.path.basename(p) for p in glob.glob(os.path.join(repo, "*cycle*.cfg")))
cycles = sorted({n.split("cycle")[1].split(".")[0] for n in available if "cycle" in n}, key=int)
raise FileNotFoundError(
f"antenna configuration {name!r} not found in {repo}. "
f"Cycles available in this casarundata: {', '.join(cycles) or 'none'}. "
"Update it with: python -c 'from casaconfig import data_update; data_update()'"
)
return path
[docs]
def source_from_grid(grid: dict | str, source: dict) -> dict:
"""``source`` with position, size and line width taken from a targeted grid.
``grid`` is a grid YAML (or its dict) with one value per key.
"""
if not isinstance(grid, dict):
with open(grid) as f:
grid = yaml.safe_load(f)
multi = [k for k, v in grid.items() if not np.isscalar(v)]
if multi:
raise ValueError(f"a mock source needs one value per grid key, got several for {multi}")
return {
**source,
"position": [grid["dra"], grid["ddec"]],
"axis_min": grid["bmin"],
"axis_maj": grid["bmaj"],
"pa": grid.get("pa", 0.0),
"line": {**source["line"], "width": grid["width"]},
}
[docs]
class MockObservation:
"""Build a line cube, observe it with ``simobserve``, image it, and move the products.
Parameters
----------
cube_shape : (nchan, ny, nx)
dv, freq_center, cell : str with units, e.g. "100km/s", "40GHz", "0.25arcsec"
sensitivity : str or float, per-channel rms in Jy (ignored if integration_time given)
integration_time : str with units, e.g. "8min"
sources : list of dict
``position`` (dra, ddec) arcsec east and north; optional ``axis_maj``/``axis_min``, the Gaussian
FWHM in arcsec (the model's ``bmaj``/``bmin``, default a sigma of one cell), and ``pa``, the
major axis' position angle in degrees east of north (default 0);
``line`` {width km/s, mean GHz, snr}; ``continuum`` {snr, index}, flux density
``propto (nu / freq_center)^index`` (default 0, flat), ``snr`` at ``freq_center``.
alma_config : str
Bare name of a CASA antenna configuration, e.g. ``alma.cycle13.3.cfg``,
resolved from CASA's own data directory.
calibrate_snr : bool
Re-simulate once with the flux rescaled so that each line's declared
``snr`` is its achieved matched-filter S/N. Doubles the runtime.
pointings : list of (dra, ddec), optional
Pointing centres in arcsec east and north of ``direction``, observed in turn (a mosaic);
default one pointing at ``direction``.
windows : list of dict, optional
Spectral windows ``{freq_center, nchan}``, each observed by its own ``simobserve`` of the
same sky (its own noise, seed ``seed + k``) and concatenated into one MS; default the cube's
own. The channel width in Hz (``dv`` at ``freq_center``) and the flux scale are shared.
image : bool
``tclean`` the MSs and plot the maps (one window only).
"""
def __init__(
self,
cube_shape=(30, 256, 256),
dv="100km/s",
freq_center="40GHz",
cell="0.25arcsec",
sensitivity=None,
integration_time=None,
fits_filename=None,
sources=None,
direction="J2000 00h00m00.10 -40d00m00.00",
ptg_file=None,
alma_config="alma.cycle13.3.cfg",
output_folder="output/ms_files",
support_folder="support",
seed=242,
calibrate_snr=True,
pointings=None,
windows=None,
image=True,
):
if not sources:
raise ValueError("at least one source is required")
self.cube_shape = tuple(cube_shape)
self.dv = dv
self.freq_center = freq_center
self.reference = freq_center # the continuum's reference frequency, shared by every window
self.cell = cell
self.sources = sources
default = "_".join(
f"{k}{s[key]['snr']}" for s in sources for k, key in (("line", "line"), ("cont", "continuum")) if key in s
)
self.fits_filename = fits_filename or os.path.join(support_folder, default + "_input.fits")
self.direction = direction
self.ptg_file = ptg_file or os.path.join(support_folder, "ptgfile.txt")
self.calibrate_snr = calibrate_snr
self.alma_config = alma_config
self.output_folder = output_folder
self.support_folder = support_folder
self.seed = seed
self.pointings = [tuple(p) for p in pointings] if pointings else [(0.0, 0.0)]
self.windows = windows
self.image = image
if windows and (calibrate_snr or image):
raise ValueError("calibrate_snr and image need one spectral window")
self.cube = None
self.ms_noiseless = None
self.ms_noisy = None
if integration_time is not None:
self.integration_time = u.Quantity(integration_time).to(u.min).value
self.sensitivity = self.estimate_sensitivity(self.integration_time)
elif sensitivity is not None:
self.sensitivity = u.Quantity(sensitivity).to(u.Jy).value if isinstance(sensitivity, str) else sensitivity
self.integration_time = self.compute_totaltime()
else:
raise ValueError("either sensitivity or integration_time must be given")
[docs]
@classmethod
def from_yaml(cls, path: str, grid: dict | str | None = None, **overrides) -> MockObservation:
"""Build from a YAML preset; ``grid`` sets the first source's geometry (see :func:`source_from_grid`)."""
with open(path) as f:
cfg = yaml.safe_load(f)
cfg.update(overrides)
if grid is not None:
cfg["sources"] = [source_from_grid(grid, cfg["sources"][0]), *cfg["sources"][1:]]
return cls(**cfg)
# ------------------------------------------------------------ frequencies
@property
def nchan(self) -> int:
return self.cube_shape[0]
def compute_df(self) -> u.Quantity:
return (u.Quantity(self.freq_center) * u.Quantity(self.dv) / c).to(u.GHz)
def _freq_axis(self):
"""(df [GHz], f_start [GHz], dv [km/s])."""
df = self.compute_df().value
f_start = u.Quantity(self.freq_center).to(u.GHz).value - self.nchan / 2 * df
return df, f_start, u.Quantity(self.dv).to(u.km / u.s).value
def compute_totaltime(self) -> float:
return (_REF_RMS_JY / (self.sensitivity / np.sqrt(self.nchan))) ** 2 * _REF_MIN
def estimate_sensitivity(self, totaltime_min: float) -> float:
return np.sqrt(_REF_MIN * _REF_RMS_JY**2 * self.nchan / totaltime_min)
@property
def project_name(self) -> str:
return os.path.basename(self.fits_filename).split("_input")[0]
def _name_ms(self) -> None:
"""The noiseless and noisy MS that ``simobserve`` (or ``concat``) writes under the project."""
stem = f"{self.project_name}/{self.project_name}." + os.path.basename(self.alma_config)
self.ms_noiseless, self.ms_noisy = stem.replace("cfg", "ms"), stem.replace("cfg", "noisy.ms")
# ------------------------------------------------------------------ cube
def _source_pixels(self, source):
nchan, ny, nx = self.cube_shape
cell = u.Quantity(self.cell).to(u.arcsec).value
dra, ddec = source.get("position", (0.0, 0.0))
pos_y = ny / 2 + ddec / cell
pos_x = nx / 2 - dra / cell
sig_min = source.get("axis_min", cell / FWHM_TO_SIGMA) * FWHM_TO_SIGMA / cell
sig_maj = source.get("axis_maj", cell / FWHM_TO_SIGMA) * FWHM_TO_SIGMA / cell
return pos_y, pos_x, sig_maj, sig_min
def create_cube(self) -> np.ndarray:
nchan, ny, nx = self.cube_shape
self.cube = np.zeros(self.cube_shape)
chans = np.arange(nchan)
Y, X = np.meshgrid(np.arange(ny), np.arange(nx), indexing="ij")
df, f_start, dv = self._freq_axis()
for src in self.sources:
pos_y, pos_x, sig_maj, sig_min = self._source_pixels(src)
spatial = models.Gaussian2D(
amplitude=1 / (2 * np.pi * sig_maj * sig_min),
x_mean=pos_x,
y_mean=pos_y,
x_stddev=sig_maj,
y_stddev=sig_min,
# pixel x runs west and y north, so a major axis at pa east of north is pa + 90 deg from +x
theta=np.deg2rad(src.get("pa", 0.0) + 90.0),
)(X, Y)
if "line" in src:
width_chan = src["line"]["width"] / dv
mean_chan = (src["line"]["mean"] - f_start) / df
flux_line = src["line"]["snr"] * self.sensitivity * np.sqrt(width_chan * FWHM_TO_SIGMA * 4)
amp = flux_line / (width_chan * FWHM_TO_SIGMA * np.sqrt(2 * np.pi))
spec = models.Gaussian1D(amplitude=amp, mean=mean_chan, stddev=width_chan * FWHM_TO_SIGMA)(chans)
self.cube += spec[:, None, None] * spatial[None]
print(f"source {src['position']}: line flux {flux_line * dv:.3e} Jy km/s")
if "continuum" in src:
flux_cont = src["continuum"]["snr"] * self.sensitivity / np.sqrt(nchan)
nu = f_start + df * chans
slope = (nu / u.Quantity(self.reference).to(u.GHz).value) ** src["continuum"].get("index", 0.0)
self.cube += flux_cont * slope[:, None, None] * spatial[None]
print(f"source {src['position']}: continuum {flux_cont:.3e} Jy/chan")
return self.cube
def save_cube(self) -> str:
if self.cube is None:
raise RuntimeError("call create_cube() first")
os.makedirs(os.path.dirname(self.fits_filename), exist_ok=True)
fits.writeto(self.fits_filename, self.cube, overwrite=True)
return self.fits_filename
# -------------------------------------------------------------- simulate
def _pointing_directions(self) -> list[str]:
"""``J2000 ...`` of every pointing, offset from ``direction`` (one ``ptgfile`` line each)."""
frame, ra, dec = self.direction.split()
centre = SkyCoord(ra, dec, frame=frame.lower().replace("j2000", "fk5"))
out = []
for dra, ddec in self.pointings:
p = centre.spherical_offsets_by(dra * u.arcsec, ddec * u.arcsec)
out.append(f"{frame} {p.ra.to_string(u.hour, sep='hms', precision=4)} "
f"{p.dec.to_string(u.deg, sep='dms', precision=3, alwayssign=True)}") # fmt: skip
return out
[docs]
def simulate_observation(self) -> None:
"""``simobserve`` the cube, or every window's cube and ``concat`` them (``windows``)."""
if not self.windows:
return self._simobserve()
concat = tasks().concat
parts = []
for k, window in enumerate(self.windows):
part = copy.copy(self)
part.windows, part.seed = None, self.seed + k
part.freq_center = window["freq_center"]
# the same channel width in Hz in every window, as the correlator gives
scale = (u.Quantity(self.freq_center) / u.Quantity(part.freq_center)).decompose().value
part.dv = str(u.Quantity(self.dv) * scale)
part.cube_shape = (window["nchan"], *self.cube_shape[1:])
part.fits_filename = self.fits_filename.replace("_input", f"_w{k}_input")
part.create_cube()
part.save_cube()
part._simobserve()
parts.append(part)
os.makedirs(self.project_name, exist_ok=True)
self._name_ms()
for name in ("ms_noiseless", "ms_noisy"):
shutil.rmtree(getattr(self, name), ignore_errors=True)
concat(vis=[getattr(part, name) for part in parts], concatvis=getattr(self, name))
for part in parts:
shutil.rmtree(part.project_name)
def _simobserve(self) -> None:
simobserve = tasks().simobserve
os.makedirs(os.path.dirname(self.ptg_file) or ".", exist_ok=True)
with open(self.ptg_file, "w") as f:
f.write("\n".join(self._pointing_directions()))
simobserve(
project=self.project_name,
skymodel=self.fits_filename,
setpointings=False,
ptgfile=self.ptg_file,
overwrite=True,
integration="10s",
totaltime=f"{self.integration_time}min",
inbright="",
comp_nchan=self.nchan,
indirection=self.direction,
incell=self.cell,
incenter=str(self.freq_center),
inwidth=str(self.compute_df()),
antennalist=resolve_antenna_config(self.alma_config),
seed=self.seed,
graphics="none",
)
self._name_ms() # weights stay as simobserve writes them: reading corrects them from the data
def run_imaging(self) -> None:
casatasks = tasks()
tclean, exportfits = casatasks.tclean, casatasks.exportfits
for vis in (self.ms_noiseless, self.ms_noisy):
if vis is None:
raise RuntimeError("call simulate_observation() first")
name = vis.replace(".ms", ".im")
tclean(
vis=vis,
imagename=name,
niter=0,
imsize=self.cube_shape[1],
cell=self.cell,
gridder="standard",
weighting="natural",
specmode="cube",
parallel=False,
)
exportfits(imagename=name + ".image", fitsimage=name + ".fits", overwrite=True)
# ------------------------------------------------------------------ plots
[docs]
def plot_results(self, plots_dir: str = "plots") -> None:
"""Line / continuum maps (noiseless, noisy, SNR) for every source."""
os.makedirs(plots_dir, exist_ok=True)
cube_clean = fits.getdata(self.ms_noiseless.replace(".ms", ".im.fits"))[0]
cube_noisy = fits.getdata(self.ms_noisy.replace(".ms", ".im.fits"))[0]
nchan, ny, nx = cube_clean.shape
df, f_start, dv = self._freq_axis()
def triptych(clean, noisy, tag):
std = np.nanstd(noisy - clean)
snr = clean / std if std else clean
fig = Figure(figsize=(12, 4))
axes = fig.subplots(1, 3)
for ax, img, title in zip(
axes, (clean, noisy, snr), ("noiseless", "noisy", f"SNR (peak {np.nanmax(snr):.1f})"), strict=True
):
im = ax.imshow(img, origin="lower")
ax.set_title(f"{tag} {title}")
fig.colorbar(im, ax=ax)
fig.tight_layout()
fig.savefig(os.path.join(plots_dir, f"{self.project_name}_{tag}.png"))
for i, src in enumerate(self.sources):
if "line" in src:
sig = src["line"]["width"] / dv * FWHM_TO_SIGMA
mean = (src["line"]["mean"] - f_start) / df
lo, hi = int(max(0, mean - 3 * sig)), int(min(nchan, mean + 3 * sig + 1))
triptych(cube_clean[lo:hi].sum(0), cube_noisy[lo:hi].sum(0), f"src{i}_line")
if "continuum" in src:
pos_y, pos_x, sig_maj, sig_min = self._source_pixels(src)
h = int(3 * max(sig_maj, sig_min))
sl = (
slice(max(0, int(pos_y - h)), min(ny, int(pos_y + h))),
slice(max(0, int(pos_x - h)), min(nx, int(pos_x + h))),
)
triptych(cube_clean.mean(0)[sl], cube_noisy.mean(0)[sl], f"src{i}_cont")
# ----------------------------------------------------------------- output
def move_output(self) -> str:
dest = os.path.join(self.output_folder, self.project_name)
if os.path.exists(dest):
shutil.rmtree(dest)
os.makedirs(self.output_folder, exist_ok=True)
shutil.move(self.project_name, dest)
for task in ("simobserve", "concat", "tclean", "exportfits"):
if os.path.exists(f"{task}.last"): # CASA's record of the last call, written to the working directory
os.remove(f"{task}.last")
self.ms_noiseless = os.path.join(self.output_folder, self.ms_noiseless)
self.ms_noisy = os.path.join(self.output_folder, self.ms_noisy)
return dest
[docs]
def achieved_snr(self) -> list[float]:
"""Optimal matched-filter S/N of each line source in the simulated data.
Uses the noiseless visibilities as the signal and the noisy weights as the
noise model, so this is the expected S/N rather than one noisy draw. The
template is the source itself, sqrt(sum_jk w Re[V']^2), which the
filter attains; for a point source it equals the channel-averaged S/N.
"""
from .data import DataHandler
clean = DataHandler(self.ms_noiseless, reweight=False)
noisy = DataHandler(self.ms_noisy)
out = []
for src in self.sources:
if "line" not in src:
continue
dra, ddec = src.get("position", (0.0, 0.0))
snr2 = sum(
np.sum(n.w * (c.vis * c.phase(dra, ddec)).real ** 2)
for c, n in zip(clean.chunks, noisy.chunks, strict=True)
)
out.append(float(np.sqrt(snr2)))
return out
def _calibrate(self) -> list[float]:
"""Rescale the cube so each line's declared ``snr`` is its achieved S/N.
S/N is linear in flux and the noise draw is seeded, so a single
multiplicative correction is exact.
"""
achieved = self.achieved_snr()
wanted = [src["line"]["snr"] for src in self.sources if "line" in src]
if not achieved or min(achieved) <= 0:
return achieved
# one global factor: all line sources share the same noise realisation
factor = float(np.mean([w / a for w, a in zip(wanted, achieved, strict=True)]))
print(f"calibrating: achieved S/N {[round(a, 2) for a in achieved]} -> scaling flux by {factor:.3f}")
self.cube *= factor
self.save_cube()
self.simulate_observation()
return self.achieved_snr()
def run_all(self, plots_dir: str = "plots") -> str:
if not self.windows:
self.create_cube()
self.save_cube()
self.simulate_observation()
if self.calibrate_snr:
final = self._calibrate()
print(f"achieved matched-filter S/N: {[round(a, 2) for a in final]}")
if self.image:
self.run_imaging()
self.plot_results(plots_dir)
return self.move_output()